def read_trigger_log(configuration, vgrid_name, flags):
"""Read-in saved trigger logs for vgrid workflows page. We read in all
rotated logs. If flags don't include verbose we try to filter out all
system trigger lines.
"""
log_content = ''
for i in xrange(workflows_log_cnt - 1, -1, -1):
log_name = '%s.%s' % (configuration.vgrid_triggers, workflows_log_name)
if i > 0:
log_name += '.%d' % i
log_path = os.path.join(configuration.vgrid_home, vgrid_name, log_name)
configuration.logger.debug('read from %s' % log_path)
try:
log_fd = open(log_path)
log_content += log_fd.read()
configuration.logger.debug('read in log lines:\n%s' % log_content)
log_fd.close()
except IOError:
pass
if not verbose(flags):
# Strip system trigger lines containing '.meta/EXT.last_modified'
system_pattern = '[0-9 ,:-]* [A-Z]* .*/\.meta/.*\.last_modified.*\n'
log_content = re.sub(system_pattern, '', log_content)
return log_content
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
name_pattern = accepted['name'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dir_listings.append(dir_listing)
try:
for (root, dirs, files) in os.walk(abs_path):
for filename in fnmatch.filter(files, name_pattern):
# IMPORTANT: this join always yields abs expanded path
abs_path = os.path.join(root, filename)
relative_path = abs_path.replace(base_dir, '')
if not valid_user_path(configuration, abs_path, base_dir,
True):
continue
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'file_with_dir': relative_path,
'flags': flags,
'special': '',
}
entries.append(file_obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
current_dir = accepted['current_dir']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
# NB: Globbing disabled on purpose here
unfiltered_match = [base_dir + pattern]
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: "%s: cannot remove directory '%s': Permission denied"
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
if not os.path.exists(real_path):
output_objects.append({'object_type': 'file_not_found',
'name': relative_path})
continue
try:
if parents(flags):
# Please note that 'rmdir -p X' should give error if X
# doesn't exist contrary to 'mkdir -p X' not giving error
# if X exists.
os.removedirs(real_path)
else:
os.rmdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s failed on '%s'" % (op_name,
relative_path)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
size = int(accepted['size'][-1])
pattern_list = accepted['path']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if size < 0:
output_objects.append({
'object_type': 'error_text',
'text': 'size must be non-negative'
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
if not check_write_access(abs_path):
logger.warning('%s called without write access: %s' % \
(op_name, abs_path))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot truncate "%s": inside a read-only location!' % \
pattern})
status = returnvalues.CLIENT_ERROR
continue
try:
fd = open(abs_path, 'r+')
fd.truncate(size)
fd.close()
logger.info('%s %s %s done' % (op_name, abs_path, size))
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
dst_dir = cache_dir
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = header_item['text'] = page_title
title_entry['skipwidgets'] = not widgets
title_entry['skipuserstyle'] = not userstyle
# Input validation assures target_dir can't escape base_dir
if not os.path.isdir(base_dir):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid client/sharelink id!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
logger.info('parsing upload form in %s' % op_name)
# Now parse and validate files to archive
# ... this includes checking for illegal directory traversal attempts
for name in defaults.keys():
if user_arguments_dict.has_key(name):
del user_arguments_dict[name]
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
dst = accepted['dst'][-1].lstrip(os.sep)
pattern_list = accepted['src']
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
dst = os.path.join(current_dir, dst)
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'ZIP/TAR archiver'
output_objects.append(
{'object_type': 'header', 'text': 'ZIP/TAR archiver'})
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
if 'h' in flags:
usage(output_objects)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(os.path.join(base_dir,
current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
# out of bounds
output_objects.append({'object_type': 'error_text', 'text':
"You're not allowed to work in %s!"
% current_dir})
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append(
{'object_type': 'text', 'text': "working in %s" % current_dir})
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('pack in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
# out of bounds
output_objects.append(
{'object_type': 'error_text', 'text':
"Invalid path! (%s expands to an illegal path)" % dst})
logger.warning('%s tried to %s restricted path %s !(%s)'
% (client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(abs_dest)):
output_objects.append({'object_type': 'error_text', 'text':
"No such destination directory: %s"
% os.path.dirname(relative_dest)})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot pack to "%s": inside a read-only location!' %
relative_dest})
return (output_objects, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
logger.debug("%s: inspecting: %s" % (op_name, server_path))
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
else:
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'error_text', 'text':
"%s: cannot pack '%s': no valid src paths"
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
continue
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
# Generally refuse handling symlinks including root vgrid shares
if os.path.islink(abs_path):
output_objects.append(
{'object_type': 'warning', 'text': """You're not allowed to
pack entire special folders like %s shared folders!""" %
configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
# Additionally refuse operations on inherited subvgrid share roots
elif in_vgrid_share(configuration, abs_path) == relative_path:
output_objects.append(
{'object_type': 'warning', 'text': """You're not allowed to
pack entire %s shared folders!""" % configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_path) == os.path.realpath(base_dir):
logger.error("%s: refusing pack home dir: %s" % (op_name,
abs_path))
output_objects.append(
{'object_type': 'warning', 'text':
"You're not allowed to pack your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_dest) == os.path.realpath(abs_path):
output_objects.append({'object_type': 'warning', 'text':
'overlap in source and destination %s'
% relative_dest})
status = returnvalues.CLIENT_ERROR
continue
if verbose(flags):
output_objects.append(
{'object_type': 'file', 'name': relative_path})
(pack_status, msg) = pack_archive(configuration, client_id,
relative_path, relative_dest)
if not pack_status:
output_objects.append({'object_type': 'error_text',
'text': 'Error: %s' % msg})
status = returnvalues.CLIENT_ERROR
continue
output_objects.append({'object_type': 'text', 'text':
'Added %s to %s' % (relative_path,
relative_dest)})
output_objects.append({'object_type': 'text', 'text':
'Packed archive of %s is now available in %s'
% (', '.join(pattern_list), relative_dest)})
output_objects.append({'object_type': 'link', 'text': 'Download archive',
'destination': os.path.join('..', client_dir,
relative_dest)})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
# We search for the last 'lines' lines by beginning from the end of
# the file and exponetially backtracking until the backtrack
# contains at least 'lines' lines or we're back to the beginning of
# the file.
# At that point we skip any extra lines before printing.
try:
filedes = open(real_path, 'r')
# Go to end of file and backtrack
backstep = 1
newlines = 0
filedes.seek(0, 2)
length = filedes.tell()
while backstep < length and newlines <= lines:
backstep *= 2
if backstep > length:
backstep = length
filedes.seek(-backstep, 2)
newlines = len(filedes.readlines())
if length:
# Go back after reading to end of file
filedes.seek(-backstep, 2)
# Skip any extra lines caused by the exponential backtracking.
# We could probably speed up convergence with binary search...
while newlines > lines:
dummy = filedes.readline()
newlines -= 1
backstep = length - filedes.tell()
# Now we're at the wanted spot - print rest of file
for _ in range(newlines):
output_lines.append(filedes.readline())
filedes.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults, output_objects, allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
iosessionid = accepted['iosessionid'][-1]
if not client_id:
if not iosessionid.strip() or not iosessionid.isalnum():
# deny
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No sessionid or invalid sessionid supplied!'})
return (output_objects, returnvalues.CLIENT_ERROR)
base_dir_no_sessionid = \
os.path.realpath(configuration.webserver_home) + os.sep
base_dir = \
os.path.realpath(os.path.join(configuration.webserver_home,
iosessionid)) + os.sep
if not os.path.isdir(base_dir):
# deny
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid sessionid!'})
return (output_objects, returnvalues.CLIENT_ERROR)
if not valid_user_path(base_dir, base_dir_no_sessionid, True):
# deny
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid sessionid!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
# TODO: this is a hack to allow truncate - fix 'put' empty files
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! ( %s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
# Make it harder to accidentially delete too much - e.g. do not delete
# VGrid files without explicit selection of subdir contents
if real_path == os.path.abspath(base_dir):
output_objects.append({'object_type': 'warning', 'text'
: "You're not allowed to delete your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
if os.path.islink(real_path):
output_objects.append({'object_type': 'warning', 'text'
: "You're not allowed to delete entire %s shared dirs!"
% configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
if os.path.isdir(real_path) and recursive(flags):
# bottom up traversal of the file tree since rmdir is limited to
# empty dirs
for (root, dirs, files) in os.walk(real_path,
topdown=False):
for name in files:
path = os.path.join(root, name)
relative_path = path.replace(base_dir, '')
# Traversal may find additional invisible files to skip
if invisible_file(name):
continue
if verbose(flags):
output_objects.append({'object_type': 'file'
, 'name': relative_path})
try:
os.remove(path)
except Exception, exc:
output_objects.append({'object_type'
: 'error_text', 'text'
: "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s"
% (op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
for name in dirs:
path = os.path.join(root, name)
relative_path = path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file'
, 'name': relative_path})
try:
os.rmdir(path)
except Exception, exc:
output_objects.append({'object_type'
: 'error_text', 'text'
: "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s"
% (op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
# Finally remove base directory
relative_path = real_path.replace(base_dir, '')
try:
os.rmdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
dst = accepted['dst'][-1].lstrip(os.sep)
pattern_list = accepted['src']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Zip/tar archive extractor'
output_objects.append({'object_type': 'header', 'text'
: 'Zip/tar archive extractor'})
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if 'h' in flags:
usage(output_objects)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('unpack in %s' % abs_dest)
# Don't use real dest in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
# out of bounds
logger.error('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' % \
(op_name, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot unpack to "%s": inside a read-only location!' % \
relative_dest})
return (output_objects, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
(unpack_status, msg) = unpack_archive(configuration,
client_id,
relative_path,
relative_dest)
if not unpack_status:
output_objects.append({'object_type': 'error_text',
'text': 'Error: %s' % msg})
status = returnvalues.CLIENT_ERROR
continue
output_objects.append(
{'object_type': 'text', 'text'
: 'The zip/tar archive %s was unpacked in %s'
% (relative_path, relative_dest)})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
dst = accepted['dst'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if dst:
dst_mode = "wb"
real_dst = os.path.join(base_dir, dst)
relative_dst = real_dst.replace(base_dir, '')
if not valid_user_path(real_dst, base_dir, True):
logger.warning('%s tried to %s into restricted path %s ! (%s)'
% (client_id, op_name, real_dst, dst))
output_objects.append({'object_type': 'error_text',
'text': "invalid destination: '%s'" % \
dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
output_lines = []
relative_path = real_path.replace(base_dir, '')
try:
fd = open(real_path, 'r')
# use file directly as iterator for efficiency
for line in fd:
output_lines.append(line)
fd.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
try:
out_fd = open(real_dst, dst_mode)
out_fd.writelines(output_lines)
out_fd.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "write failed: '%s'" % exc})
logger.error("%s: write failed on '%s': %s" % (op_name,
real_dst, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'text',
'text': "wrote %s to %s" % (relative_path,
relative_dst)})
# Prevent truncate after first write
dst_mode = "ab+"
else:
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
# TODO: rip this hack out into real download handler?
# Force download of files when output_format == 'file_format'
# This will only work for the first file matching a glob when
# using file_format.
# And it is supposed to only work for one file.
if user_arguments_dict.has_key('output_format'):
output_format = user_arguments_dict['output_format'][0]
if output_format == 'file':
output_objects.append(
{'object_type': 'start', 'headers':
[('Content-Disposition',
'attachment; filename="%s";' % \
os.path.basename(real_path))]})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
iosessionid = accepted['iosessionid'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if not client_id:
base_dir = os.path.realpath(configuration.webserver_home
+ os.sep + iosessionid) + os.sep
status = returnvalues.OK
real_dest = base_dir + dst
dst_list = glob.glob(real_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(real_dest)):
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal dst path provided!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [real_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text'
: 'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
real_dest = os.path.abspath(dest)
# Don't use real_path in output as it may expose underlying
# fs layout.
relative_dest = real_dest.replace(base_dir, '')
if not valid_user_path(real_dest, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid destination (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
# src must be a file unless recursive is specified
if not recursive(flags) and os.path.isdir(real_path):
logger.warning('skipping directory source %s' % real_path)
output_objects.append({'object_type': 'warning', 'text'
: 'skipping directory src %s!' % relative_path})
continue
# If destination is a directory the src should be copied there
real_target = real_dest
if os.path.isdir(real_target):
real_target = os.path.join(real_target,
os.path.basename(real_path))
if os.path.abspath(real_path) == os.path.abspath(real_target):
logger.warning('%s tried to %s %s to itself! (%s)' % \
(client_id, op_name, real_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot copy '%s' to self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
if os.path.isdir(real_path) and \
real_target.startswith(real_path + os.sep):
logger.warning('%s tried to %s %s to itself! (%s)'
% (client_id, op_name, real_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot copy '%s' to (sub) self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
try:
if os.path.isdir(real_path):
shutil.copytree(real_path, real_target)
else:
shutil.copy(real_path, real_target)
logger.info('%s %s %s done' % (op_name, real_path, real_target))
except Exception, exc:
output_objects.append(
{'object_type': 'error_text',
'text': "%s: failed on '%s' to '%s'" \
% (op_name, relative_path, relative_dest)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
# NOTE: we produce output matching an invocation of:
# du -aL --apparent-size --block-size=1 PATH [PATH ...]
filedus = []
summarize_output = summarize(flags)
for abs_path in match:
if invisible_path(abs_path):
continue
relative_path = abs_path.replace(base_dir, '')
# cache accumulated sub dir sizes - du sums into parent dir size
dir_sizes = {}
try:
# Assume a directory to walk
for (root, dirs, files) in walk(abs_path, topdown=False,
followlinks=True):
if invisible_path(root):
continue
dir_bytes = 0
for name in files:
real_file = os.path.join(root, name)
if invisible_path(real_file):
continue
relative_file = real_file.replace(base_dir, '')
size = os.path.getsize(real_file)
dir_bytes += size
if not summarize_output:
filedus.append({'object_type': 'filedu',
'name': relative_file,
'bytes': size})
for name in dirs:
real_dir = os.path.join(root, name)
if invisible_path(real_dir):
continue
dir_bytes += dir_sizes[real_dir]
relative_root = root.replace(base_dir, '')
dir_bytes += os.path.getsize(root)
dir_sizes[root] = dir_bytes
if root == abs_path or not summarize_output:
filedus.append({'object_type': 'filedu',
'name': relative_root,
'bytes': dir_bytes})
if os.path.isfile(abs_path):
# Fall back to plain file where walk is empty
size = os.path.getsize(abs_path)
filedus.append({'object_type': 'filedu',
'name': relative_path,
'bytes': size})
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path,
exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filedus', 'filedus': filedus})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
name_pattern = accepted['name'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
output_lines = []
relative_path = real_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dir_listings.append(dir_listing)
try:
for (root, dirs, files) in os.walk(real_path):
for filename in fnmatch.filter(files, name_pattern):
real_path = os.path.join(root, filename)
relative_path = real_path.replace(base_dir, '')
if not valid_user_path(real_path, base_dir,
True):
continue
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'file_with_dir': relative_path,
'flags': flags,
'special': '',
}
entries.append(file_obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
abs_dest = base_dir + dst
dst_list = glob.glob(abs_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(abs_dest)):
output_objects.append({
'object_type': 'error_text',
'text': 'Illegal dst path provided!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [abs_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append({
'object_type':
'warning',
'text':
'dst (%s) matches multiple targets - using last: %s' % (dst, dest)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
logger.warning('%s tried to %s to restricted path %s ! (%s)' %
(client_id, op_name, abs_dest, dst))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot move to "%s": inside a read-only location!' % relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: no such file or directory! %s' % (op_name, pattern)
})
status = returnvalues.CLIENT_ERROR
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
# Generally refuse handling symlinks including root vgrid shares
if os.path.islink(abs_path):
output_objects.append({
'object_type':
'warning',
'text':
"""You're not allowed to
move entire special folders like %s shared folders!""" %
configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
# Additionally refuse operations on inherited subvgrid share roots
elif in_vgrid_share(configuration, abs_path) == relative_path:
output_objects.append({
'object_type':
'warning',
'text':
"""You're not allowed to
move entire %s shared folders!""" % configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_path) == os.path.realpath(base_dir):
logger.error("%s: refusing move home dir: %s" %
(op_name, abs_path))
output_objects.append({
'object_type':
'warning',
'text':
"You're not allowed to move your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
if not check_write_access(abs_path):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot move "%s": inside a read-only location!' % pattern
})
status = returnvalues.CLIENT_ERROR
continue
# If destination is a directory the src should be moved in there
# Move with existing directory as target replaces the directory!
abs_target = abs_dest
if os.path.isdir(abs_target):
if os.path.samefile(abs_target, abs_path):
output_objects.append({
'object_type':
'warning',
'text':
"Cannot move '%s' to a subdirectory of itself!" %
relative_path
})
status = returnvalues.CLIENT_ERROR
continue
abs_target = os.path.join(abs_target,
os.path.basename(abs_path))
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'moved', [relative_path, relative_dest])
shutil.move(abs_path, abs_target)
logger.info('%s %s %s done' % (op_name, abs_path, abs_target))
except Exception, exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'moved', [relative_path, relative_dest],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
logger.error("jobstatus input validation failed: %s" % accepted)
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
max_jobs = int(accepted['max_jobs'][-1])
order = 'unsorted '
if sorted(flags):
order = 'sorted '
patterns = accepted['job_id']
project_names = accepted['project_name']
if len(project_names) > 0:
for project_name in project_names:
project_name_job_ids = \
get_job_ids_with_specified_project_name(client_id,
project_name, configuration.mrsl_files_dir, logger)
patterns.extend(project_name_job_ids)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text'
: '%s %s job status' % \
(configuration.short_title, order)})
if not patterns:
output_objects.append({'object_type': 'error_text', 'text'
: 'No job_id specified!'})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if not os.path.isdir(base_dir):
output_objects.append(
{'object_type': 'error_text', 'text'
: ('You have not been created as a user on the %s server! ' \
'Please contact the %s team.') % \
(configuration.short_title, configuration.short_title)})
return (output_objects, returnvalues.CLIENT_ERROR)
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s: You do not have any matching job IDs!' % pattern})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
if sorted(flags):
sort(filelist)
if max_jobs < len(filelist):
output_objects.append(
{'object_type': 'text', 'text'
: 'Only showing first %d of the %d matching jobs as requested'
% (max_jobs, len(filelist))})
filelist = filelist[:max_jobs]
# Iterate through jobs and print details for each
job_list = {'object_type': 'job_list', 'jobs': []}
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No such job: %s (could not load mRSL file %s)' % \
(job_id, filepath)})
continue
# Expand any job variables before use
job_dict = expand_variables(job_dict)
job_obj = {'object_type': 'job', 'job_id': job_id}
job_obj['status'] = job_dict['STATUS']
time_fields = [
'VERIFIED',
'VERIFIED_TIMESTAMP',
'RECEIVED_TIMESTAMP',
'QUEUED_TIMESTAMP',
'SCHEDULE_TIMESTAMP',
'EXECUTING_TIMESTAMP',
'FINISHED_TIMESTAMP',
'FAILED_TIMESTAMP',
'CANCELED_TIMESTAMP',
]
for name in time_fields:
if job_dict.has_key(name):
# time objects cannot be marshalled, asctime if timestamp
try:
job_obj[name.lower()] = time.asctime(job_dict[name])
except Exception, exc:
# not a time object, just add
job_obj[name.lower()] = job_dict[name]
###########################################
# ARC job status retrieval on demand:
# But we should _not_ update the status in the mRSL files, since
# other MiG code might rely on finding only valid "MiG" states.
if configuration.arc_clusters and \
job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \
and job_dict['STATUS'] == 'EXECUTING':
try:
home = os.path.join(configuration.user_home, client_dir)
arcsession = arc.Ui(home)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arc.ARCWrapperError, err:
logger.error('Error retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arc.NoProxyError, err:
logger.error('While retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
lang = accepted['lang'][-1].lower()
mode = accepted['mode'][-1]
output_objects.append({'object_type': 'header', 'text'
: '%s spell check' % configuration.short_title })
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
allowed_modes = ['none', 'url', 'email', 'sgml', 'tex']
# Include both base languages and variants
allowed_langs = [
'da',
'da_dk',
'de',
'en',
'en_gb',
'en_us',
'es',
'fi',
'fr',
'it',
'nl',
'no',
'se',
]
# TODO: use path from settings file
dict_path = '%s/%s' % (base_dir, '.personal_dictionary')
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if not mode in allowed_modes:
output_objects.append({'object_type': 'error_text', 'text'
: 'Unsupported mode: %s' % mode})
status = returnvalues.CLIENT_ERROR
if not lang in allowed_langs:
output_objects.append({'object_type': 'error_text', 'text'
: 'Unsupported lang: %s' % mode})
status = returnvalues.CLIENT_ERROR
# Show all if no flags given
if not flags:
flags = 'blw'
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
try:
(out, err) = spellcheck(real_path, mode, lang,
dict_path)
if err:
output_objects.append({'object_type': 'error_text',
'text': err})
for line in out:
output_lines.append(line + '\n')
except Exception, err:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, err)})
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
htmlform = \
'''
<form method="post" action="editor.py">
<input type="hidden" name="path" value="%s" />
<input type="submit" value="Edit file" />
</form>
'''\
% relative_path
output_objects.append({'object_type': 'html_form', 'text'
: htmlform})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
real_dest = base_dir + dst
dst_list = glob.glob(real_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(real_dest)):
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal dst path provided!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [real_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text'
: 'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
real_dest = os.path.abspath(dest)
# Don't use real_path in output as it may expose underlying
# fs layout.
relative_dest = real_dest.replace(base_dir, '')
if not valid_user_path(real_dest, base_dir, True):
logger.warning('%s tried to %s to restricted path %s ! (%s)'
% (client_id, op_name, real_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'error_text', 'text'
: '%s: no such file or directory! %s'
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
if os.path.islink(real_path):
output_objects.append(
{'object_type': 'warning', 'text'
: "You're not allowed to move entire %s shared dirs!"
% configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
# If destination is a directory the src should be moved in there
# Move with existing directory as target replaces the directory!
real_target = real_dest
if os.path.isdir(real_target):
if os.path.samefile(real_target, real_path):
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot move '%s' to a subdirectory of itself!" % \
relative_path
})
status = returnvalues.CLIENT_ERROR
continue
real_target = os.path.join(real_target,
os.path.basename(real_path))
try:
shutil.move(real_path, real_target)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
algo_list = accepted['hash_algo']
max_chunks = int(accepted['max_chunks'][-1])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
for hash_algo in algo_list:
try:
chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"])
checksum = chksum_helper(abs_path, max_chunks=max_chunks)
line = "%s %s\n" % (checksum, relative_path)
logger.info("%s %s of %s: %s" %
(op_name, hash_algo, abs_path, checksum))
output_lines.append(line)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output', 'lines': output_lines}
output_objects.append(entry)
all_lines += output_lines
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
dst = accepted['dst'][-1].lstrip(os.sep)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
dst_mode = "wb"
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
relative_dst = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
logger.warning('%s tried to %s into restricted path %s ! (%s)' %
(client_id, op_name, abs_dest, dst))
output_objects.append({
'object_type': 'error_text',
'text': "invalid destination: '%s'" % dst
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'accessed', [relative_path])
fd = open(abs_path, 'r')
# use file directly as iterator for efficiency
for line in fd:
output_lines.append(line)
fd.close()
except Exception, exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'accessed', [relative_path],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'modified', [dst])
out_fd = open(abs_dest, dst_mode)
out_fd.writelines(output_lines)
out_fd.close()
logger.info('%s %s %s done' %
(op_name, abs_path, abs_dest))
except Exception, exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'modified', [dst],
error=True,
details=exc)
output_objects.append({
'object_type': 'error_text',
'text': "write failed: '%s'" % exc
})
logger.error("%s: write failed on '%s': %s" %
(op_name, abs_dest, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({
'object_type':
'text',
'text':
"wrote %s to %s" % (relative_path, relative_dst)
})
# Prevent truncate after first write
dst_mode = "ab+"
else:
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
# TODO: rip this hack out into real download handler?
# Force download of files when output_format == 'file_format'
# This will only work for the first file matching a glob when
# using file_format.
# And it is supposed to only work for one file.
if user_arguments_dict.has_key('output_format'):
output_format = user_arguments_dict['output_format'][0]
if output_format == 'file':
output_objects.append({
'object_type':
'start',
'headers': [('Content-Disposition',
'attachment; filename="%s";' %
os.path.basename(abs_path))]
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
try:
filedes = open(abs_path, 'r')
i = 0
for line in filedes:
if i >= lines:
break
output_lines.append(line)
i += 1
filedes.close()
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
# Show all if no type flags given
if not byte_count(flags) and not line_count(flags)\
and not word_count(flags):
flags += 'blw'
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
filewcs = []
for real_path in match:
relative_path = real_path.replace(base_dir, '')
(bytes, words, lines) = (0, 0, 0)
try:
obj = {'object_type': 'filewc', 'name': relative_path}
if os.path.isdir(real_path):
obj = {
'object_type': 'filewc',
'name': '%s: %s: Is a directory' % (op_name,
relative_path),
'lines': 0,
'words': 0,
'bytes': 0,
}
filewcs.append(obj)
continue
fd = open(real_path, 'r')
lines = 0
# use file directly as iterator for efficiency
for line in fd:
lines += 1
bytes += len(line)
words += len(line.split())
if line_count(flags):
obj['lines'] = lines
if word_count(flags):
obj['words'] = words
if byte_count(flags):
obj['bytes'] = bytes
filewcs.append(obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filewcs', 'filewcs'
: filewcs})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# Show all if no type flags given
if not byte_count(flags) and not line_count(flags)\
and not word_count(flags):
flags += 'blw'
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
filewcs = []
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
(bytes, words, lines) = (0, 0, 0)
try:
obj = {'object_type': 'filewc', 'name': relative_path}
if os.path.isdir(abs_path):
obj = {
'object_type': 'filewc',
'name':
'%s: %s: Is a directory' % (op_name, relative_path),
'lines': 0,
'words': 0,
'bytes': 0,
}
filewcs.append(obj)
continue
fd = open(abs_path, 'r')
lines = 0
# use file directly as iterator for efficiency
for line in fd:
lines += 1
bytes += len(line)
words += len(line.split())
if line_count(flags):
obj['lines'] = lines
if word_count(flags):
obj['words'] = words
if byte_count(flags):
obj['bytes'] = bytes
filewcs.append(obj)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filewcs', 'filewcs': filewcs})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# IMPORTANT: the CGI front end forces the input extraction to be delayed
# We must manually extract and parse input here to avoid memory explosion
# for huge files!
# TODO: explosions still happen sometimes!
# Most likely because of Apache SSL renegotiations which have
# no other way of storing input
extract_input = user_arguments_dict.get('__DELAYED_INPUT__', dict)
logger.info('Extracting input in %s' % op_name)
form = extract_input()
logger.info('After extracting input in %s' % op_name)
file_item = None
file_name = ''
user_arguments_dict = {}
if form.has_key('fileupload'):
file_item = form['fileupload']
file_name = file_item.filename
user_arguments_dict['fileupload'] = ['true']
user_arguments_dict['path'] = [file_name]
if form.has_key('path'):
user_arguments_dict['path'] = [form['path'].value]
if form.has_key('restrict'):
user_arguments_dict['restrict'] = [form['restrict'].value]
else:
user_arguments_dict['restrict'] = defaults['restrict']
logger.info('Filtered input is: %s' % user_arguments_dict)
# Now validate parts as usual
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
path = accepted['path'][-1]
restrict = accepted['restrict'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_griddk:
output_objects.append({
'object_type':
'text',
'text':
'''Grid.dk features are disabled on this site.
Please contact the site admins %s if you think they should be enabled.
''' % configuration.admin_email
})
return (output_objects, returnvalues.OK)
logger.info('Filtered input validated with result: %s' % accepted)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
output_objects.append({'object_type': 'header', 'text': 'Uploading file'})
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
real_path = os.path.realpath(os.path.join(base_dir, path))
# Implicit destination
if os.path.isdir(real_path):
real_path = os.path.join(real_path, os.path.basename(file_name))
if not valid_user_path(configuration, real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, real_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid destination (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(real_path)):
output_objects.append({
'object_type':
'error_text',
'text':
"cannot write: no such file or directory: %s)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
# We fork off here and redirect the user to a progress page for user
# friendly output and to avoid cgi timeouts from killing the upload.
# We use something like the Active State python recipe for daemonizing
# to properly detach from the CGI process and continue in the background.
# Please note that we only close stdio file descriptors to avoid closing
# the fileupload.
file_item.file.seek(0, 2)
total_size = file_item.file.tell()
file_item.file.seek(0, 0)
try:
pid = os.fork()
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
os.chdir('/')
os.umask(0)
for fno in range(3):
try:
os.close(fno)
except OSError:
pass
else:
os._exit(0)
except OSError, ose:
output_objects.append({
'object_type':
'error_text',
'text':
'%s upload could not background! (%s)' %
(path, str(ose).replace(base_dir, ''))
})
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = accepted['flags']
patterns = accepted['job_id']
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
mrsl_keywords_dict = get_keywords_dict(configuration)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in patterns:
# Add file extension
pattern += '.mRSL'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
mrsl_dict = unpickle(abs_path, logger)
if not mrsl_dict:
raise Exception('could not load job mRSL')
for (key, val) in mrsl_dict.items():
if not key in mrsl_keywords_dict.keys():
continue
if not val:
continue
output_lines.append('::%s::\n' % key)
if 'multiplestrings' == mrsl_keywords_dict[key]['Type']:
for line in val:
output_lines.append('%s\n' % line)
elif 'multiplekeyvalues' == mrsl_keywords_dict[key][
'Type']:
for (left, right) in val:
output_lines.append('%s=%s\n' % (left, right))
else:
output_lines.append('%s\n' % val)
output_lines.append('\n')
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({
'object_type': 'file_output',
'path': relative_path,
'lines': output_lines
})
else:
output_objects.append({
'object_type': 'file_output',
'lines': output_lines
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
submitstatuslist = []
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
submitstatus = {
'object_type': 'submitstatus',
'name': relative_path
}
try:
(job_status, newmsg, job_id) = new_job(abs_path, client_id,
configuration, False,
True)
except Exception, exc:
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
job_status = False
newmsg = "%s failed on '%s' (is it a valid mRSL file?)"\
% (op_name, relative_path)
job_id = None
if not job_status:
submitstatus['status'] = False
submitstatus['message'] = newmsg
status = returnvalues.CLIENT_ERROR
else:
submitstatus['status'] = True
submitstatus['job_id'] = job_id
submitstatuslist.append(submitstatus)
output_objects.append({
'object_type': 'submitstatuslist',
'submitstatuslist': submitstatuslist
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# IMPORTANT: the CGI front end forces the input extraction to be delayed
# We must manually extract and parse input here to avoid memory explosion
# for huge files!
# TODO: explosions still happen sometimes!
# Most likely because of Apache SSL renegotiations which have
# no other way of storing input
extract_input = user_arguments_dict["__DELAYED_INPUT__"]
logger.info("Extracting input in %s" % op_name)
form = extract_input()
logger.info("After extracting input in %s" % op_name)
file_item = None
file_name = ""
user_arguments_dict = {}
if form.has_key("fileupload"):
file_item = form["fileupload"]
file_name = file_item.filename
user_arguments_dict["fileupload"] = ["true"]
user_arguments_dict["path"] = [file_name]
if form.has_key("path"):
user_arguments_dict["path"] = [form["path"].value]
if form.has_key("restrict"):
user_arguments_dict["restrict"] = [form["restrict"].value]
else:
user_arguments_dict["restrict"] = defaults["restrict"]
logger.info("Filtered input is: %s" % user_arguments_dict)
# Now validate parts as usual
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler("POST"):
output_objects.append(
{"object_type": "error_text", "text": "Only accepting POST requests to prevent unintended updates"}
)
return (output_objects, returnvalues.CLIENT_ERROR)
flags = "".join(accepted["flags"])
path = accepted["path"][-1]
restrict = accepted["restrict"][-1]
if not configuration.site_enable_griddk:
output_objects.append(
{
"object_type": "text",
"text": """Grid.dk features are disabled on this site.
Please contact the Grid admins %s if you think they should be enabled.
"""
% configuration.admin_email,
}
)
return (output_objects, returnvalues.OK)
logger.info("Filtered input validated with result: %s" % accepted)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({"object_type": "text", "text": "%s using flag: %s" % (op_name, flag)})
output_objects.append({"object_type": "header", "text": "Uploading file"})
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
real_path = os.path.realpath(os.path.join(base_dir, path))
# Implicit destination
if os.path.isdir(real_path):
real_path = os.path.join(real_path, os.path.basename(file_name))
if not valid_user_path(real_path, base_dir, True):
logger.warning("%s tried to %s restricted path %s ! (%s)" % (client_id, op_name, real_path, path))
output_objects.append(
{"object_type": "error_text", "text": "Invalid destination (%s expands to an illegal path)" % path}
)
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(real_path)):
output_objects.append(
{"object_type": "error_text", "text": "cannot write: no such file or directory: %s)" % path}
)
return (output_objects, returnvalues.CLIENT_ERROR)
# We fork off here and redirect the user to a progress page for user
# friendly output and to avoid cgi timeouts from killing the upload.
# We use something like the Active State python recipe for daemonizing
# to properly detach from the CGI process and continue in the background.
# Please note that we only close stdio file descriptors to avoid closing
# the fileupload.
file_item.file.seek(0, 2)
total_size = file_item.file.tell()
file_item.file.seek(0, 0)
try:
pid = os.fork()
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
os.chdir("/")
os.umask(0)
for fno in range(3):
try:
os.close(fno)
except OSError:
pass
else:
os._exit(0)
except OSError, ose:
output_objects.append(
{
"object_type": "error_text",
"text": "%s upload could not background! (%s)" % (path, str(ose).replace(base_dir, "")),
}
)
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
try:
filedes = open(real_path, 'r')
i = 0
for line in filedes:
if i >= lines:
break
output_lines.append(line)
i += 1
filedes.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = accepted['flags']
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
stats = []
for real_path in match:
relative_path = real_path.replace(base_dir, '')
try:
(stat_status, stat) = stat_path(real_path, logger)
if stat_status:
if verbose(flags):
stat['name'] = relative_path
stat['object_type'] = 'stat'
stats.append(stat)
else:
output_objects.append({'object_type': 'error_text',
'text': stat})
status = returnvalues.SYSTEM_ERROR
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'stats', 'stats'
: stats})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
title_entry = find_entry(output_objects, 'title')
label = "%s" % configuration.site_vgrid_label
title_entry['text'] = '%s Workflows' % label
# NOTE: Delay header entry here to include vgrid_name
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
vgrid_name = accepted['vgrid_name'][-1]
operation = accepted['operation'][-1]
flags = ''.join(accepted['flags'][-1])
if not vgrid_is_owner_or_member(vgrid_name, client_id, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
'''You must be an owner or member of %s vgrid to
access the workflows.''' % vgrid_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not operation in allowed_operations:
output_objects.append({
'object_type':
'error_text',
'text':
'''Operation must be one of %s.''' % ', '.join(allowed_operations)
})
return (output_objects, returnvalues.OK)
if operation in show_operations:
# jquery support for tablesorter (and unused confirmation dialog)
# table initially sorted by 0 (last update / date)
refresh_call = 'ajax_workflowjobs("%s", "%s")' % (vgrid_name, flags)
table_spec = {
'table_id': 'workflowstable',
'sort_order': '[[0,1]]',
'refresh_call': refresh_call
}
(add_import, add_init,
add_ready) = man_base_js(configuration, [table_spec])
if operation == "show":
add_ready += '%s;' % refresh_call
add_ready += '''
/* Init variables helper as foldable but closed and with individual
heights */
$(".variables-accordion").accordion({
collapsible: true,
active: false,
heightStyle: "content"
});
/* fix and reduce accordion spacing */
$(".ui-accordion-header").css("padding-top", 0)
.css("padding-bottom", 0).css("margin", 0);
/* NOTE: requires managers CSS fix for proper tab bar height */
$(".workflow-tabs").tabs();
$("#logarea").scrollTop($("#logarea")[0].scrollHeight);
'''
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
output_objects.append({
'object_type': 'html_form',
'text': man_base_html(configuration)
})
output_objects.append({
'object_type':
'header',
'text':
'%s Workflows for %s' % (label, vgrid_name)
})
logger.info('vgridworkflows %s %s' % (vgrid_name, operation))
# Iterate through jobs and list details for each
trigger_jobs = []
log_content = ''
if operation in list_operations:
trigger_job_dir = os.path.join(
configuration.vgrid_home,
os.path.join(vgrid_name,
'.%s.jobs' % configuration.vgrid_triggers))
trigger_job_pending_dir = os.path.join(trigger_job_dir,
'pending_states')
trigger_job_final_dir = os.path.join(trigger_job_dir, 'final_states')
if makedirs_rec(trigger_job_pending_dir, configuration) \
and makedirs_rec(trigger_job_final_dir, configuration):
abs_vgrid_dir = '%s/' \
% os.path.abspath(os.path.join(configuration.vgrid_files_home,
vgrid_name))
for filename in os.listdir(trigger_job_pending_dir):
trigger_job_filepath = \
os.path.join(trigger_job_pending_dir, filename)
trigger_job = unpickle(trigger_job_filepath, logger)
serverjob_filepath = \
os.path.join(configuration.mrsl_files_dir,
os.path.join(
client_id_dir(trigger_job['owner']),
'%s.mRSL' % trigger_job['jobid']))
serverjob = unpickle(serverjob_filepath, logger)
if serverjob:
if serverjob['STATUS'] in pending_states:
trigger_event = trigger_job['event']
trigger_rule = trigger_job['rule']
trigger_action = trigger_event['event_type']
trigger_time = time.ctime(trigger_event['time_stamp'])
trigger_path = '%s %s' % \
(trigger_event['src_path'].replace(
abs_vgrid_dir, ''),
trigger_event['dest_path'].replace(
abs_vgrid_dir, ''))
job = {
'object_type': 'trigger_job',
'job_id': trigger_job['jobid'],
'rule_id': trigger_rule['rule_id'],
'path': trigger_path,
'action': trigger_action,
'time': trigger_time,
'status': serverjob['STATUS']
}
if not job['rule_id'].startswith(img_trigger_prefix) \
or verbose(flags):
trigger_jobs.append(job)
elif serverjob['STATUS'] in final_states:
src_path = os.path.join(trigger_job_pending_dir,
filename)
dest_path = os.path.join(trigger_job_final_dir,
filename)
move_file(src_path, dest_path, configuration)
else:
logger.error(
'Trigger job: %s, unknown state: %s' %
(trigger_job['jobid'], serverjob['STATUS']))
log_content = read_trigger_log(configuration, vgrid_name, flags)
if operation in show_operations:
# Always run as rule creator to avoid users being able to act on behalf
# of ANY other user using triggers (=exploit)
extra_fields = [
('path', None),
('match_dirs', ['False', 'True']),
('match_recursive', ['False', 'True']),
('changes', [keyword_all] + valid_trigger_changes),
('action', [keyword_auto] + valid_trigger_actions),
('arguments', None),
('run_as', client_id),
]
# NOTE: we do NOT show saved template contents - see addvgridtriggers
optional_fields = [('rate_limit', None), ('settle_time', None)]
# Only include system triggers in verbose mode
if verbose(flags):
system_filter = []
else:
system_filter = [('rule_id', '%s_.*' % img_trigger_prefix)]
(init_status,
oobjs) = vgrid_add_remove_table(client_id,
vgrid_name,
'trigger',
'vgridtrigger',
configuration,
extra_fields + optional_fields,
filter_items=system_filter)
if not init_status:
output_objects.append({
'object_type': 'error_text',
'text': 'failed to load triggers: %s' % oobjs
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Generate variable helper values for a few concrete samples for help
# text
vars_html = ''
dummy_rule = {'run_as': client_id, 'vgrid_name': vgrid_name}
samples = [('input.txt', 'modified'), ('input/image42.raw', 'changed')]
for (path, change) in samples:
vgrid_path = os.path.join(vgrid_name, path)
vars_html += "<b>Expanded variables when %s is %s:</b><br/>" % \
(vgrid_path, change)
expanded = get_path_expand_map(vgrid_path, dummy_rule, change)
for (key, val) in expanded.items():
vars_html += " %s: %s<br/>" % (key, val)
commands_html = ''
commands = get_usage_map(configuration)
for usage in commands.values():
commands_html += " %s<br/>" % usage
helper_html = """
<div class='variables-accordion'>
<h4>Help on available trigger variable names and values</h4>
<p>
Triggers can use a number of helper variables on the form +TRIGGERXYZ+ to
dynamically act on targets. Some of the values are bound to the rule owner the
%s while the remaining ones are automatically expanded for the particular
trigger target as shown in the following examples:<br/>
%s
</p>
<h4>Help on available trigger commands and arguments</h4>
<p>
It is possible to set up trigger rules that basically run any operation with a
side effect you could manually do on %s. I.e. like submitting/cancelling
a job, creating/moving/deleting a file or directory and so on. When you select
'command' as the action for a trigger rule, you have the following commands at
your disposal:<br/>
%s
</p>
</div>
""" % (label, vars_html, configuration.short_title, commands_html)
# Make page with manage triggers tab and active jobs and log tab
output_objects.append({
'object_type':
'html_form',
'text':
'''
<div id="wrap-tabs" class="workflow-tabs">
<ul>
<li><a href="#manage-tab">Manage Triggers</a></li>
<li><a href="#jobs-tab">Active Trigger Jobs</a></li>
</ul>
'''
})
# Display existing triggers and form to add new ones
output_objects.append({
'object_type': 'html_form',
'text': '''
<div id="manage-tab">
'''
})
output_objects.append({
'object_type': 'sectionheader',
'text': 'Manage Triggers'
})
output_objects.extend(oobjs)
output_objects.append({
'object_type': 'html_form',
'text': helper_html
})
if configuration.site_enable_crontab:
output_objects.append({
'object_type':
'html_form',
'text':
'''
<p>You can combine these workflows with the personal '''
})
output_objects.append({
'object_type': 'link',
'destination': 'crontab.py',
'class': 'crontablink iconspace',
'text': 'schedule task'
})
output_objects.append({
'object_type':
'html_form',
'text':
'''
facilities in case you want to trigger flows at given times rather than only
in reaction to file system events.</p>
'''
})
output_objects.append({
'object_type': 'html_form',
'text': '''
</div>
'''
})
# Display active trigger jobs and recent logs for this vgrid
output_objects.append({
'object_type': 'html_form',
'text': '''
<div id="jobs-tab">
'''
})
output_objects.append({
'object_type': 'sectionheader',
'text': 'Active Trigger Jobs'
})
output_objects.append({
'object_type': 'table_pager',
'entry_name': 'job',
'default_entries': default_pager_entries
})
output_objects.append({
'object_type': 'trigger_job_list',
'trigger_jobs': trigger_jobs
})
if operation in show_operations:
output_objects.append({
'object_type': 'sectionheader',
'text': 'Trigger Log'
})
output_objects.append({
'object_type': 'trigger_log',
'log_content': log_content
})
if operation in show_operations:
output_objects.append({
'object_type': 'html_form',
'text': '''
</div>
'''
})
output_objects.append({
'object_type': 'html_form',
'text': '''
</div>
'''
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
logger.error("jobstatus input validation failed: %s" % accepted)
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
max_jobs = int(accepted['max_jobs'][-1])
order = 'unsorted '
if sorted(flags):
order = 'sorted '
patterns = accepted['job_id']
project_names = accepted['project_name']
if len(project_names) > 0:
for project_name in project_names:
project_name_job_ids = \
get_job_ids_with_specified_project_name(client_id,
project_name, configuration.mrsl_files_dir, logger)
patterns.extend(project_name_job_ids)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text'
: '%s %s job status' % \
(configuration.short_title, order)})
if not patterns:
output_objects.append({
'object_type': 'error_text',
'text': 'No job_id specified!'
})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if not os.path.isdir(base_dir):
output_objects.append(
{'object_type': 'error_text', 'text'
: ('You have not been created as a user on the %s server! ' \
'Please contact the %s team.') % \
(configuration.short_title, configuration.short_title)})
return (output_objects, returnvalues.CLIENT_ERROR)
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
if sorted(flags):
sort(filelist)
if max_jobs > 0 and max_jobs < len(filelist):
output_objects.append({
'object_type':
'text',
'text':
'Only showing first %d of the %d matching jobs as requested' %
(max_jobs, len(filelist))
})
filelist = filelist[:max_jobs]
# Iterate through jobs and list details for each
job_list = {'object_type': 'job_list', 'jobs': []}
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No such job: %s (could not load mRSL file %s)' % \
(job_id, filepath)})
continue
# Expand any job variables before use
job_dict = expand_variables(job_dict)
job_obj = {'object_type': 'job', 'job_id': job_id}
job_obj['status'] = job_dict['STATUS']
time_fields = [
'VERIFIED',
'VERIFIED_TIMESTAMP',
'RECEIVED_TIMESTAMP',
'QUEUED_TIMESTAMP',
'SCHEDULE_TIMESTAMP',
'EXECUTING_TIMESTAMP',
'FINISHED_TIMESTAMP',
'FAILED_TIMESTAMP',
'CANCELED_TIMESTAMP',
]
for name in time_fields:
if job_dict.has_key(name):
# time objects cannot be marshalled, asctime if timestamp
try:
job_obj[name.lower()] = time.asctime(job_dict[name])
except Exception, exc:
# not a time object, just add
job_obj[name.lower()] = job_dict[name]
###########################################
# ARC job status retrieval on demand:
# But we should _not_ update the status in the mRSL files, since
# other MiG code might rely on finding only valid "MiG" states.
if configuration.arc_clusters and \
job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \
and job_dict['STATUS'] == 'EXECUTING':
try:
home = os.path.join(configuration.user_home, client_dir)
arcsession = arc.Ui(home)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arc.ARCWrapperError, err:
logger.error('Error retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arc.NoProxyError, err:
logger.error('While retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
# We search for the last 'lines' lines by beginning from the end of
# the file and exponetially backtracking until the backtrack
# contains at least 'lines' lines or we're back to the beginning of
# the file.
# At that point we skip any extra lines before printing.
try:
filedes = open(abs_path, 'r')
# Go to end of file and backtrack
backstep = 1
newlines = 0
filedes.seek(0, 2)
length = filedes.tell()
while backstep < length and newlines <= lines:
backstep *= 2
if backstep > length:
backstep = length
filedes.seek(-backstep, 2)
newlines = len(filedes.readlines())
if length:
# Go back after reading to end of file
filedes.seek(-backstep, 2)
# Skip any extra lines caused by the exponential backtracking.
# We could probably speed up convergence with binary search...
while newlines > lines:
dummy = filedes.readline()
newlines -= 1
backstep = length - filedes.tell()
# Now we're at the wanted spot - print rest of file
for _ in range(newlines):
output_lines.append(filedes.readline())
filedes.close()
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arc.ARCWrapperError, err:
logger.error('Error retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arc.NoProxyError, err:
logger.error('While retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except Exception, err:
logger.error('Error retrieving ARC job status: %s' % err)
job_obj['status'] += '(Error during retrieval)'
exec_histories = []
if verbose(flags):
if job_dict.has_key('EXECUTE'):
command_line = '; '.join(job_dict['EXECUTE'])
if len(command_line) > 256:
job_obj['execute'] = '%s ...' % command_line[:252]
else:
job_obj['execute'] = command_line
res_conf = job_dict.get('RESOURCE_CONFIG', {})
if res_conf.has_key('RESOURCE_ID'):
public_id = res_conf['RESOURCE_ID']
if res_conf.get('ANONYMOUS', True):
public_id = anon_resource_id(public_id)
job_obj['resource'] = public_id
if job_dict.get('PUBLICNAME', False):
job_obj['resource'] += ' (alias %(PUBLICNAME)s)' % job_dict
if job_dict.has_key('RESOURCE_VGRID'):
