def is_owner(
client_id,
unique_config_name,
config_home,
logger,
):
"""Check that client_id is listed in pickled owners file"""
config_path = os.path.abspath(
os.path.join(config_home, unique_config_name, 'owners'))
# Check validity of unique_config_name
# Automatic configuration extraction
configuration = None
if not valid_user_path(configuration, config_path, config_home):
# Extract caller information
from traceback import format_stack
caller = ''.join(format_stack()[:-1]).strip()
logger.warning(
"""is_owner caught possible illegal directory traversal attempt
by client: '%s'
unique name: '%s'
caller: %s""" % (client_id, unique_config_name, caller))
return False
return is_item_in_pickled_list(config_path, client_id, logger)
def is_owner(
client_id,
unique_config_name,
config_home,
logger,
):
"""Check that client_id is listed in pickled owners file"""
config_path = os.path.abspath(os.path.join(config_home,
unique_config_name, 'owners'))
# Check validity of unique_config_name
if not valid_user_path(config_path, config_home):
# Extract caller information
from traceback import format_stack
caller = ''.join(format_stack()[:-1]).strip()
logger.warning("""is_owner caught possible illegal directory traversal attempt
by client: '%s'
unique name: '%s'
caller: %s"""
% (client_id, unique_config_name, caller))
return False
return is_item_in_pickled_list(config_path, client_id, logger)\
or is_item_in_pickled_list(config_path,
old_id_format(client_id), logger)
def _parse_form_xfer(xfer, user_args, client_id, configuration):
"""Parse xfer request (i.e. copy, move or upload) file/dir entries from
user_args.
"""
_logger = configuration.logger
files, rejected = [], []
i = 0
client_dir = client_id_dir(client_id)
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
xfer_pattern = 'freeze_%s_%%d' % xfer
for i in xrange(max_freeze_files):
if user_args.has_key(xfer_pattern % i):
source_path = user_args[xfer_pattern % i][-1].strip()
source_path = os.path.normpath(source_path).lstrip(os.sep)
_logger.debug('found %s entry: %s' % (xfer, source_path))
if not source_path:
continue
try:
valid_path(source_path)
except Exception, exc:
rejected.append('invalid path: %s (%s)' % (source_path,
exc))
continue
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(
os.path.join(base_dir, source_path))
# Prevent out-of-bounds, and restrict some greedy targets
if not valid_user_path(configuration, abs_path, base_dir, True):
_logger.error('found illegal directory traversal %s entry: %s'
% (xfer, source_path))
rejected.append('invalid path: %s (%s)' %
(source_path, 'illegal path!'))
continue
elif os.path.exists(abs_path) and os.path.samefile(abs_path,
base_dir):
_logger.warning('refusing archival of entire user home %s: %s'
% (xfer, source_path))
rejected.append('invalid path: %s (%s)' %
(source_path, 'entire home not allowed!'))
continue
elif in_vgrid_share(configuration, abs_path) == source_path:
_logger.warning(
'refusing archival of entire vgrid shared folder %s: %s' %
(xfer, source_path))
rejected.append('invalid path: %s (%s)' %
(source_path, 'entire %s share not allowed!'
% configuration.site_vgrid_label))
continue
# expand any dirs recursively
if os.path.isdir(abs_path):
for (root, dirnames, filenames) in os.walk(abs_path):
for subname in filenames:
abs_sub = os.path.join(root, subname)
sub_base = root.replace(abs_path, source_path)
sub_path = os.path.join(sub_base, subname)
files.append((abs_sub, sub_path.lstrip(os.sep)))
else:
files.append((abs_path, source_path.lstrip(os.sep)))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
vgrid_name = accepted['vgrid_name'][-1]
path = accepted['path'][-1]
if not vgrid_is_owner_or_member(vgrid_name, client_id,
configuration):
output_objects.append({'object_type': 'error_text', 'text':
'''You must be an owner or member of %s %s to
access the private files.''' % (vgrid_name, configuration.site_vgrid_label)})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.vgrid_private_base,
vgrid_name)) + os.sep
# Strip leading slashes to avoid join() throwing away prefix
rel_path = path.lstrip(os.sep)
real_path = os.path.abspath(os.path.join(base_dir, rel_path))
if not valid_user_path(real_path, base_dir, True):
output_objects.append({'object_type': 'error_text', 'text':
'''You are not allowed to use paths outside %s
private files dir.''' % configuration.site_vgrid_label})
return (output_objects, returnvalues.CLIENT_ERROR)
try:
private_fd = open(real_path, 'rb')
entry = {'object_type': 'binary',
'data': private_fd.read()}
# Cut away all the usual web page formatting to show only contents
output_objects = [{'object_type': 'start', 'headers': []}, entry,
{'object_type': 'script_status'},
{'object_type': 'end'}]
private_fd.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text', 'text'
: 'Error reading %s private file (%s)'
% (configuration.site_vgrid_label, exc)})
return (output_objects, returnvalues.SYSTEM_ERROR)
def parse_form_upload(user_args, client_id, configuration, base_dir):
"""Parse upload file and chunk entries from user_args. Chunk limits are
extracted from content-range http header in environment.
Files are considered to be inside uplad tmp dir inside base_dir.
"""
files, rejected = [], []
logger = configuration.logger
cache_dir = os.path.join(base_dir, upload_tmp_dir) + os.sep
# TODO: we only support single filename and chunk for now; extend?
#for name_index in xrange(max_upload_files):
# if user_args.has_key(filename_field) and \
# len(user_args[filename_field]) > name_index:
for name_index in [0]:
if user_args.has_key(filename_field):
if isinstance(user_args[filename_field], basestring):
filename = user_args[filename_field]
else:
filename = user_args[filename_field][name_index]
logger.info('found name: %s' % filename)
else:
# No more files
break
if not filename.strip():
continue
try:
filename = strip_dir(filename)
valid_path(filename)
except Exception, exc:
logger.error('invalid filename: %s' % filename)
rejected.append((filename, 'invalid filename: %s (%s)' % \
(filename, exc)))
continue
rel_path = os.path.join(upload_tmp_dir, filename)
real_path = os.path.abspath(os.path.join(base_dir, rel_path))
if not valid_user_path(real_path, cache_dir, True):
logger.error('%s tried to access restricted path %s ! (%s)'
% (client_id, real_path, cache_dir))
rejected.append("Invalid path (%s expands to an illegal path)" \
% filename)
continue
#for chunk_index in xrange(max_upload_chunks):
# if user_args.has_key(files_field) and \
# len(user_args[files_field]) > chunk_index:
for chunk_index in [0]:
if user_args.has_key(files_field):
chunk = user_args[files_field][chunk_index]
else:
break
configuration.logger.debug('find chunk range: %s' % filename)
(chunk_first, chunk_last) = extract_chunk_region(configuration)
if len(chunk) > upload_block_size:
configuration.logger.error('skip bigger than allowed chunk')
continue
elif chunk_last < 0:
chunk_last = len(chunk) - 1
files.append((rel_path, (chunk, chunk_first, chunk_last)))
def get_fs_path(configuration, abs_path, root, chroot_exceptions):
"""Internal helper to translate path with chroot and invisible files
in mind. Also assures general path character restrictions are applied.
Automatically expands to abs path to avoid traversal issues with e.g.
MYVGRID/../bla that would expand to vgrid_files_home/bla instead of bla
in user home if left as is.
"""
try:
valid_path(abs_path)
except:
raise ValueError("Invalid path characters")
if not valid_user_path(configuration, abs_path, root, True,
chroot_exceptions):
raise ValueError("Illegal path access attempt")
return abs_path
def _parse_form_xfer(xfer, user_args, client_id, configuration):
"""Parse xfer request (i.e. copy, move or upload) file/dir entries from
user_args.
"""
files, rejected = [], []
i = 0
client_dir = client_id_dir(client_id)
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
xfer_pattern = 'freeze_%s_%%d' % xfer
for i in xrange(max_freeze_files):
if user_args.has_key(xfer_pattern % i):
source_path = user_args[xfer_pattern % i][-1].strip()
configuration.logger.debug('found %s entry: %s' % (xfer,
source_path))
if not source_path:
continue
try:
valid_path(source_path)
except Exception, exc:
rejected.append('invalid path: %s (%s)' % (source_path,
exc))
continue
source_path = os.path.normpath(source_path).lstrip(os.sep)
real_path = os.path.abspath(os.path.join(base_dir, source_path))
if not valid_user_path(real_path, base_dir, True):
rejected.append('invalid path: %s (%s)' % \
(source_path, 'illegal path!'))
continue
# expand any dirs recursively
if os.path.isdir(real_path):
for (root, dirnames, filenames) in os.walk(real_path):
for subname in filenames:
real_sub = os.path.join(root, subname)
sub_base = root.replace(real_path, source_path)
sub_path = os.path.join(sub_base, subname)
files.append((real_sub, sub_path))
else:
files.append((real_path, source_path))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
name_pattern = accepted['name'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
output_lines = []
relative_path = real_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dir_listings.append(dir_listing)
try:
for (root, dirs, files) in os.walk(real_path):
for filename in fnmatch.filter(files, name_pattern):
real_path = os.path.join(root, filename)
relative_path = real_path.replace(base_dir, '')
if not valid_user_path(real_path, base_dir,
True):
continue
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'file_with_dir': relative_path,
'flags': flags,
'special': '',
}
entries.append(file_obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not patterns:
output_objects.append({
'object_type': 'error_text',
'text': 'No job_id specified!'
})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
keywords_dict = mrslkeywords.get_keywords_dict(configuration)
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# resubmit is hard on the server
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
resubmitobjs = []
status = returnvalues.OK
for filepath in filelist:
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
# ("Resubmitting job with job_id: %s" % job_id)
resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id}
mrsl_dict = unpickle(filepath, logger)
if not mrsl_dict:
resubmitobj['message'] = "No such job: %s (%s)" % (job_id,
mrsl_file)
status = returnvalues.CLIENT_ERROR
resubmitobjs.append(resubmitobj)
continue
resubmit_items = keywords_dict.keys()
# loop selected keywords and create mRSL string
resubmit_job_string = ''
for dict_elem in resubmit_items:
value = ''
# Extract job value with fallback to default to support optional
# fields
job_value = mrsl_dict.get(dict_elem,
keywords_dict[dict_elem]['Value'])
if keywords_dict[dict_elem]['Type'].startswith(
'multiplekeyvalues'):
for (elem_key, elem_val) in job_value:
if elem_key:
value += '%s=%s\n' % (str(elem_key).strip(),
str(elem_val).strip())
elif keywords_dict[dict_elem]['Type'].startswith('multiple'):
for elem in job_value:
if elem:
value += '%s\n' % str(elem).rstrip()
else:
if str(job_value):
value += '%s\n' % str(job_value).rstrip()
# Only insert keywords with an associated value
if value:
if value.rstrip() != '':
resubmit_job_string += '''::%s::
%s
''' % (dict_elem, value.rstrip())
# save tempfile
(filehandle, tempfilename) = \
tempfile.mkstemp(dir=configuration.mig_system_files,
text=True)
os.write(filehandle, resubmit_job_string)
os.close(filehandle)
# submit job the usual way
(new_job_status, msg, new_job_id) = new_job(tempfilename, client_id,
configuration, False, True)
if not new_job_status:
resubmitobj['status'] = False
resubmitobj['message'] = msg
status = returnvalues.SYSTEM_ERROR
resubmitobjs.append(resubmitobj)
continue
# o.out("Resubmit failed: %s" % msg)
# o.reply_and_exit(o.ERROR)
resubmitobj['status'] = True
resubmitobj['new_job_id'] = new_job_id
resubmitobjs.append(resubmitobj)
# o.out("Resubmit successful: %s" % msg)
# o.out("%s" % msg)
output_objects.append({
'object_type': 'resubmitobjs',
'resubmitobjs': resubmitobjs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults, output_objects, allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
queue = accepted['queue'][-1]
action = accepted['action'][-1]
iosessionid = accepted['iosessionid'][-1]
msg = accepted['msg'][-1]
msg_id = accepted['msg_id'][-1]
# Web format for cert access and no header for SID access
if client_id:
output_objects.append({'object_type': 'header', 'text'
: 'Message queue %s' % action})
else:
output_objects.append({'object_type': 'start'})
# Always return at least a basic file_output entry
file_entry = {'object_type': 'file_output',
'lines': [],
'wrap_binary': True,
'wrap_targets': ['lines']}
if not action in valid_actions:
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid action "%s" (supported: %s)' % \
(action, ', '.join(valid_actions))})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions and not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
# Find user home from session or certificate
if iosessionid:
client_home = os.path.realpath(os.path.join(configuration.webserver_home,
iosessionid))
client_dir = os.path.basename(client_home)
elif client_id:
client_dir = client_id_dir(client_id)
else:
output_objects.append({'object_type': 'error_text', 'text'
: 'Either certificate or session ID is required'
})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if not os.path.isdir(base_dir):
output_objects.append({'object_type': 'error_text', 'text'
: 'No matching session or user home!'})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep
default_queue_dir = os.path.join(mqueue_base, default_mqueue)
# Create mqueue base and default queue dir if missing
if not os.path.exists(default_queue_dir):
try:
os.makedirs(default_queue_dir)
except:
pass
queue_path = os.path.abspath(os.path.join(mqueue_base, queue))
if not valid_user_path(queue_path, mqueue_base):
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid queue name: "%s"' % queue})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
lock_path = os.path.join(mqueue_base, lock_name)
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
status = returnvalues.OK
if action == "interactive":
output_objects.append({'object_type': 'text', 'text'
: '''
Fill in the fields below to control and access your personal message queues.
Jobs can receive from and send to the message queues during execution, and use
them as a means of job inter-communication. Expect message queue operations to
take several seconds on the resources, however. That is, use it for tasks like
orchestrating long running jobs, and not for low latency communication.
'''})
html = '''
<form name="mqueueform" method="post" action="mqueue.py">
<table class="mqueue">
<tr><td class=centertext>
</td></tr>
<tr><td>
Action:<br />
<input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue
<input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue
<input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue
<input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue
<input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues
<input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages
<input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message
</td></tr>
<tr><td>
Queue:<br />
<input type=text size=60 name=queue value="%s" />
</td></tr>
<tr><td>
<div id="msgfieldf">
<input type=text size=60 name=msg value="%s" /><br />
</div>
</td></tr>
<tr><td>
<input type="submit" value="Apply" />
</td></tr>
</table>
</form>
''' % (queue, msg)
output_objects.append({'object_type': 'html_form', 'text'
: html})
output_objects.append({'object_type': 'text', 'text': '''
Further live job control is avalable through the live I/O interface.
They provide a basic interface for centrally managing input and output files
for active jobs.
'''
})
output_objects.append({'object_type': 'link', 'destination':
'liveio.py',
'text': 'Live I/O interface'})
return (output_objects, returnvalues.OK)
elif action == 'create':
try:
os.mkdir(queue_path)
output_objects.append({'object_type': 'text', 'text':
'New "%s" queue created' % queue})
except Exception, err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
real_dest = base_dir + dst
dst_list = glob.glob(real_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(real_dest)):
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal dst path provided!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [real_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text'
: 'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
real_dest = os.path.abspath(dest)
# Don't use real_path in output as it may expose underlying
# fs layout.
relative_dest = real_dest.replace(base_dir, '')
if not valid_user_path(real_dest, base_dir, True):
logger.warning('%s tried to %s to restricted path %s ! (%s)'
% (client_id, op_name, real_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'error_text', 'text'
: '%s: no such file or directory! %s'
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
if os.path.islink(real_path):
output_objects.append(
{'object_type': 'warning', 'text'
: "You're not allowed to move entire %s shared dirs!"
% configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
# If destination is a directory the src should be moved in there
# Move with existing directory as target replaces the directory!
real_target = real_dest
if os.path.isdir(real_target):
if os.path.samefile(real_target, real_path):
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot move '%s' to a subdirectory of itself!" % \
relative_path
})
status = returnvalues.CLIENT_ERROR
continue
real_target = os.path.join(real_target,
os.path.basename(real_path))
try:
shutil.move(real_path, real_target)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not patterns:
output_objects.append({'object_type': 'error_text', 'text'
: 'No job_id specified!'})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
keywords_dict = mrslkeywords.get_keywords_dict(configuration)
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s: You do not have any matching job IDs!' % pattern})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# resubmit is hard on the server
if len(filelist) > 100:
output_objects.append({'object_type': 'error_text', 'text'
: 'Too many matching jobs (%s)!'
% len(filelist)})
return (output_objects, returnvalues.CLIENT_ERROR)
resubmitobjs = []
status = returnvalues.OK
for filepath in filelist:
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
# ("Resubmitting job with job_id: %s" % job_id)
resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id}
mrsl_dict = unpickle(filepath, logger)
if not mrsl_dict:
resubmitobj['message'] = "No such job: %s (%s)" % (job_id,
mrsl_file)
status = returnvalues.CLIENT_ERROR
resubmitobjs.append(resubmitobj)
continue
resubmit_items = keywords_dict.keys()
# loop selected keywords and create mRSL string
resubmit_job_string = ''
for dict_elem in resubmit_items:
value = ''
# Extract job value with fallback to default to support optional
# fields
job_value = mrsl_dict.get(dict_elem,
keywords_dict[dict_elem]['Value'])
if keywords_dict[dict_elem]['Type'].startswith('multiplekeyvalues'):
for (elem_key, elem_val) in job_value:
if elem_key:
value += '%s=%s\n' % (str(elem_key).strip(),
str(elem_val).strip())
elif keywords_dict[dict_elem]['Type'].startswith('multiple'):
for elem in job_value:
if elem:
value += '%s\n' % str(elem).rstrip()
else:
if str(job_value):
value += '%s\n' % str(job_value).rstrip()
# Only insert keywords with an associated value
if value:
if value.rstrip() != '':
resubmit_job_string += '''::%s::
%s
''' % (dict_elem, value.rstrip())
# save tempfile
(filehandle, tempfilename) = \
tempfile.mkstemp(dir=configuration.mig_system_files,
text=True)
os.write(filehandle, resubmit_job_string)
os.close(filehandle)
# submit job the usual way
(new_job_status, msg, new_job_id) = new_job(tempfilename,
client_id, configuration, False, True)
if not new_job_status:
resubmitobj['status'] = False
resubmitobj['message'] = msg
status = returnvalues.SYSTEM_ERROR
resubmitobjs.append(resubmitobj)
continue
# o.out("Resubmit failed: %s" % msg)
# o.reply_and_exit(o.ERROR)
resubmitobj['status'] = True
resubmitobj['new_job_id'] = new_job_id
resubmitobjs.append(resubmitobj)
# o.out("Resubmit successful: %s" % msg)
# o.out("%s" % msg)
output_objects.append({'object_type': 'resubmitobjs', 'resubmitobjs'
: resubmitobjs})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
dst = accepted['dst'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if dst:
dst_mode = "wb"
real_dst = os.path.join(base_dir, dst)
relative_dst = real_dst.replace(base_dir, '')
if not valid_user_path(real_dst, base_dir, True):
logger.warning('%s tried to %s into restricted path %s ! (%s)'
% (client_id, op_name, real_dst, dst))
output_objects.append({'object_type': 'error_text',
'text': "invalid destination: '%s'" % \
dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
output_lines = []
relative_path = real_path.replace(base_dir, '')
try:
fd = open(real_path, 'r')
# use file directly as iterator for efficiency
for line in fd:
output_lines.append(line)
fd.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
try:
out_fd = open(real_dst, dst_mode)
out_fd.writelines(output_lines)
out_fd.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "write failed: '%s'" % exc})
logger.error("%s: write failed on '%s': %s" % (op_name,
real_dst, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'text',
'text': "wrote %s to %s" % (relative_path,
relative_dst)})
# Prevent truncate after first write
dst_mode = "ab+"
else:
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
# TODO: rip this hack out into real download handler?
# Force download of files when output_format == 'file_format'
# This will only work for the first file matching a glob when
# using file_format.
# And it is supposed to only work for one file.
if user_arguments_dict.has_key('output_format'):
output_format = user_arguments_dict['output_format'][0]
if output_format == 'file':
output_objects.append(
{'object_type': 'start', 'headers':
[('Content-Disposition',
'attachment; filename="%s";' % \
os.path.basename(real_path))]})
def parse_form_upload(user_args,
user_id,
configuration,
base_dir,
dst_dir,
reject_write=False):
"""Parse upload file and chunk entries from user_args. Chunk limits are
extracted from content-range http header in environment.
Existing files are automatically taken from upload_tmp_dir and uploads go
into dst_dir inside base_dir.
The optional reject_write argument is used for delayed refusal if someone
tries to upload to a read-only sharelink.
"""
files, rejected = [], []
logger = configuration.logger
rel_dst_dir = dst_dir.replace(base_dir, '')
# TODO: we only support single filename and chunk for now; extend?
# for name_index in xrange(max_upload_files):
# if user_args.has_key(filename_field) and \
# len(user_args[filename_field]) > name_index:
for name_index in [0]:
if user_args.has_key(filename_field):
if isinstance(user_args[filename_field], basestring):
filename = user_args[filename_field]
else:
filename = user_args[filename_field][name_index]
logger.info('found name: %s' % filename)
else:
# No more files
break
if not filename.strip():
continue
if reject_write:
rejected.append((filename, 'read-only share: upload refused!'))
continue
try:
filename = strip_dir(filename)
valid_path(filename)
except Exception, exc:
logger.error('invalid filename: %s' % filename)
rejected.append(
(filename, 'invalid filename: %s (%s)' % (filename, exc)))
continue
rel_path = os.path.join(rel_dst_dir, filename)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, rel_path))
if not valid_user_path(configuration, abs_path, dst_dir, True):
logger.error('%s tried to access restricted path %s ! (%s)' %
(user_id, abs_path, dst_dir))
rejected.append("Invalid path (%s expands to an illegal path)" %
filename)
continue
# for chunk_index in xrange(max_upload_chunks):
# if user_args.has_key(files_field) and \
# len(user_args[files_field]) > chunk_index:
for chunk_index in [0]:
if user_args.has_key(files_field):
chunk = user_args[files_field][chunk_index]
else:
break
configuration.logger.debug('find chunk range: %s' % filename)
(chunk_first, chunk_last) = extract_chunk_region(configuration)
if len(chunk) > upload_block_size:
configuration.logger.error('skip bigger than allowed chunk')
continue
elif chunk_last < 0:
chunk_last = len(chunk) - 1
files.append((rel_path, (chunk, chunk_first, chunk_last)))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
chosen_newline = accepted['newline'][-1]
submitjob = accepted['submitjob'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs and submitjob:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# HTML spec dictates newlines in forms to be MS style (\r\n)
# rather than un*x style (\n): change if requested.
form_newline = '\r\n'
allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'}
output_objects.append({
'object_type': 'header',
'text': 'Saving changes to edited file'
})
if not chosen_newline in allowed_newline.keys():
output_objects.append({
'object_type':
'error_text',
'text':
'Unsupported newline style supplied: %s (must be one of %s)' %
(chosen_newline, ', '.join(allowed_newline.keys()))
})
return (output_objects, returnvalues.CLIENT_ERROR)
saved_newline = allowed_newline[chosen_newline]
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
abs_path = ''
unfiltered_match = glob.glob(base_dir + path)
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if abs_path == '':
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, path.lstrip(os.sep)))
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_path, parent_dir=True):
logger.warning('%s called without write access: %s' % \
(op_name, abs_path))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot edit "%s": inside a read-only location!' % \
path})
status = returnvalues.CLIENT_ERROR
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(abs_path, client_id)
if owner != client_id:
output_objects.append({
'object_type': 'error_text',
'text': "You don't have the lock for %s!" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
try:
fh = open(abs_path, 'w+')
fh.write(user_arguments_dict['editarea'][0].replace(
form_newline, saved_newline))
fh.close()
# everything ok
output_objects.append({
'object_type': 'text',
'text': 'Saved changes to %s.' % path
})
logger.info('saved changes to %s' % path)
release_edit_lock(abs_path, client_id)
except Exception, exc:
# Don't give away information about actual fs layout
output_objects.append({
'object_type':
'error_text',
'text':
'%s could not be written! (%s)' %
(path, str(exc).replace(base_dir, ''))
})
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
algo_list = accepted['hash_algo']
max_chunks = int(accepted['max_chunks'][-1])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
for hash_algo in algo_list:
try:
chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"])
checksum = chksum_helper(abs_path, max_chunks=max_chunks)
line = "%s %s\n" % (checksum, relative_path)
logger.info("%s %s of %s: %s" %
(op_name, hash_algo, abs_path, checksum))
output_lines.append(line)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output', 'lines': output_lines}
output_objects.append(entry)
all_lines += output_lines
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s: You do not have any matching job IDs!' % pattern})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job schedule is hard on the server, limit
if len(filelist) > 100:
output_objects.append({'object_type': 'error_text', 'text'
: 'Too many matching jobs (%s)!'
% len(filelist)})
return (output_objects, returnvalues.CLIENT_ERROR)
saveschedulejobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
saveschedulejob = {'object_type': 'saveschedulejob',
'job_id': job_id}
dict = unpickle(filepath, logger)
if not dict:
saveschedulejob['message'] = \
('The file containing the information' \
' for job id %s could not be opened!' \
' You can only read schedule for ' \
'your own jobs!') % job_id
saveschedulejobs.append(saveschedulejob)
status = returnvalues.CLIENT_ERROR
continue
saveschedulejob['oldstatus'] = dict['STATUS']
# Is the job status pending?
possible_schedule_states = ['QUEUED', 'RETRY', 'FROZEN']
if not dict['STATUS'] in possible_schedule_states:
saveschedulejob['message'] = \
'You can only read schedule for jobs with status: %s.'\
% ' or '.join(possible_schedule_states)
saveschedulejobs.append(saveschedulejob)
continue
# notify queue
if not send_message_to_grid_script('JOBSCHEDULE ' + job_id
+ '\n', logger, configuration):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Error sending message to grid_script, update may fail.'
})
status = returnvalues.SYSTEM_ERROR
continue
saveschedulejobs.append(saveschedulejob)
savescheduleinfo = """Please find any available job schedule status in
verbose job status output."""
output_objects.append({'object_type': 'saveschedulejobs',
'saveschedulejobs': saveschedulejobs,
'savescheduleinfo': savescheduleinfo})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
lang = accepted['lang'][-1].lower()
mode = accepted['mode'][-1]
output_objects.append({'object_type': 'header', 'text'
: '%s spell check' % configuration.short_title })
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
allowed_modes = ['none', 'url', 'email', 'sgml', 'tex']
# Include both base languages and variants
allowed_langs = [
'da',
'da_dk',
'de',
'en',
'en_gb',
'en_us',
'es',
'fi',
'fr',
'it',
'nl',
'no',
'se',
]
# TODO: use path from settings file
dict_path = '%s/%s' % (base_dir, '.personal_dictionary')
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if not mode in allowed_modes:
output_objects.append({'object_type': 'error_text', 'text'
: 'Unsupported mode: %s' % mode})
status = returnvalues.CLIENT_ERROR
if not lang in allowed_langs:
output_objects.append({'object_type': 'error_text', 'text'
: 'Unsupported lang: %s' % mode})
status = returnvalues.CLIENT_ERROR
# Show all if no flags given
if not flags:
flags = 'blw'
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
try:
(out, err) = spellcheck(real_path, mode, lang,
dict_path)
if err:
output_objects.append({'object_type': 'error_text',
'text': err})
for line in out:
output_lines.append(line + '\n')
except Exception, err:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, err)})
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
htmlform = \
'''
<form method="post" action="editor.py">
<input type="hidden" name="path" value="%s" />
<input type="submit" value="Edit file" />
</form>
'''\
% relative_path
output_objects.append({'object_type': 'html_form', 'text'
: htmlform})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
dst = accepted['dst'][-1].lstrip(os.sep)
pattern_list = accepted['src']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Zip/tar archive extractor'
output_objects.append({'object_type': 'header', 'text'
: 'Zip/tar archive extractor'})
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if 'h' in flags:
usage(output_objects)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('unpack in %s' % abs_dest)
# Don't use real dest in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
# out of bounds
logger.error('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' % \
(op_name, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot unpack to "%s": inside a read-only location!' % \
relative_dest})
return (output_objects, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
(unpack_status, msg) = unpack_archive(configuration,
client_id,
relative_path,
relative_dest)
if not unpack_status:
output_objects.append({'object_type': 'error_text',
'text': 'Error: %s' % msg})
status = returnvalues.CLIENT_ERROR
continue
output_objects.append(
{'object_type': 'text', 'text'
: 'The zip/tar archive %s was unpacked in %s'
% (relative_path, relative_dest)})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
chosen_newline = accepted['newline'][-1]
submitjob = accepted['submitjob'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# HTML spec dictates newlines in forms to be MS style (\r\n)
# rather than un*x style (\n): change if requested.
form_newline = '\r\n'
allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'}
output_objects.append({'object_type': 'header', 'text'
: 'Saving changes to edited file'})
if not chosen_newline in allowed_newline.keys():
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Unsupported newline style supplied: %s (must be one of %s)'
% (chosen_newline, ', '.join(allowed_newline.keys()))})
return (output_objects, returnvalues.CLIENT_ERROR)
saved_newline = allowed_newline[chosen_newline]
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
real_path = ''
unfiltered_match = glob.glob(base_dir + path)
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, path))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % path})
return (output_objects, returnvalues.CLIENT_ERROR)
if real_path == '':
real_path = base_dir + path
if not valid_user_path(real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, path))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % path})
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(real_path, client_id)
if owner != client_id:
output_objects.append({'object_type': 'error_text', 'text'
: "You don't have the lock for %s!"
% path})
return (output_objects, returnvalues.CLIENT_ERROR)
try:
fh = open(real_path, 'w+')
fh.write(user_arguments_dict['editarea'
][0].replace(form_newline, saved_newline))
fh.close()
# everything ok
output_objects.append({'object_type': 'text', 'text'
: 'Saved changes to %s.' % path})
release_edit_lock(real_path, client_id)
except Exception, exc:
# Don't give away information about actual fs layout
output_objects.append({'object_type': 'error_text', 'text'
: '%s could not be written! (%s)'
% (path, str(exc).replace(base_dir, ''
))})
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
output_objects.append({
'object_type': 'header',
'text': 'Reject Resource Request'
})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
unique_resource_name = accepted['unique_resource_name'][-1].strip()
request_name = unhexlify(accepted['request_name'][-1])
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not is_owner(client_id, unique_resource_name,
configuration.resource_home, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'You must be an owner of %s to reject requests!' %
unique_resource_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# resource dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.resource_home,
unique_resource_name)) + os.sep
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, request_name))
if not valid_user_path(
configuration, abs_path, base_dir, allow_equal=False):
logger.warning('%s tried to access restricted path %s ! (%s)' % \
(client_id, abs_path, request_name))
output_objects.append({
'object_type':
'error_text',
'text':
'''Illegal request name "%s":
you can only reject requests to your own resources.''' % request_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
if request_name:
request_dir = os.path.join(configuration.resource_home,
unique_resource_name)
req = load_access_request(configuration, request_dir, request_name)
if not req or not delete_access_request(configuration, request_dir,
request_name):
logger.error("failed to delete owner request for %s in %s" % \
(unique_resource_name, request_name))
output_objects.append({
'object_type': 'error_text', 'text':
'Failed to remove saved resource request for %s in %s!'\
% (unique_resource_name, request_name)})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'text',
'text':
'''
Deleted %(request_type)s access request to %(target)s for %(entity)s .
''' % req
})
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'protocol': any_protocol,
'unique_resource_name': unique_resource_name,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
fill_helpers.update(req)
target_op = 'sendrequestaction'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
output_objects.append({
'object_type':
'html_form',
'text':
"""
<p>
You can use the reply form below if you want to additionally send an
explanation for rejecting the request.
</p>
<form method='%(form_method)s' action='%(target_op)s.py'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<input type=hidden name=request_type value='resourcereject' />
<input type=hidden name=unique_resource_name value='%(target)s' />
<input type=hidden name=cert_id value='%(entity)s' />
<input type=hidden name=protocol value='%(protocol)s' />
<table>
<tr>
<td class='title'>Optional reject message to requestor(s)</td>
</tr><tr>
<td><textarea name=request_text cols=72 rows=10>
We have decided to reject your %(request_type)s request to our %(target)s
resource.
Regards, the %(target)s resource owners
</textarea></td>
</tr>
<tr>
<td><input type='submit' value='Inform requestor(s)' /></td>
</tr>
</table>
</form>
<br />
""" % fill_helpers
})
output_objects.append({
'object_type':
'link',
'destination':
'resadmin.py?unique_resource_name=%s' % unique_resource_name,
'text':
'Back to administration for %s' % unique_resource_name
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults, output_objects, allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
iosessionid = accepted['iosessionid'][-1]
if not client_id:
if not iosessionid.strip() or not iosessionid.isalnum():
# deny
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No sessionid or invalid sessionid supplied!'})
return (output_objects, returnvalues.CLIENT_ERROR)
base_dir_no_sessionid = \
os.path.realpath(configuration.webserver_home) + os.sep
base_dir = \
os.path.realpath(os.path.join(configuration.webserver_home,
iosessionid)) + os.sep
if not os.path.isdir(base_dir):
# deny
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid sessionid!'})
return (output_objects, returnvalues.CLIENT_ERROR)
if not valid_user_path(base_dir, base_dir_no_sessionid, True):
# deny
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid sessionid!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
# TODO: this is a hack to allow truncate - fix 'put' empty files
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! ( %s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
# Make it harder to accidentially delete too much - e.g. do not delete
# VGrid files without explicit selection of subdir contents
if real_path == os.path.abspath(base_dir):
output_objects.append({'object_type': 'warning', 'text'
: "You're not allowed to delete your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
if os.path.islink(real_path):
output_objects.append({'object_type': 'warning', 'text'
: "You're not allowed to delete entire %s shared dirs!"
% configuration.site_vgrid_label
})
status = returnvalues.CLIENT_ERROR
continue
if os.path.isdir(real_path) and recursive(flags):
# bottom up traversal of the file tree since rmdir is limited to
# empty dirs
for (root, dirs, files) in os.walk(real_path,
topdown=False):
for name in files:
path = os.path.join(root, name)
relative_path = path.replace(base_dir, '')
# Traversal may find additional invisible files to skip
if invisible_file(name):
continue
if verbose(flags):
output_objects.append({'object_type': 'file'
, 'name': relative_path})
try:
os.remove(path)
except Exception, exc:
output_objects.append({'object_type'
: 'error_text', 'text'
: "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s"
% (op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
for name in dirs:
path = os.path.join(root, name)
relative_path = path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file'
, 'name': relative_path})
try:
os.rmdir(path)
except Exception, exc:
output_objects.append({'object_type'
: 'error_text', 'text'
: "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s"
% (op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
# Finally remove base directory
relative_path = real_path.replace(base_dir, '')
try:
os.rmdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
try:
filedes = open(abs_path, 'r')
i = 0
for line in filedes:
if i >= lines:
break
output_lines.append(line)
i += 1
filedes.close()
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
name_pattern = accepted['name'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dir_listings.append(dir_listing)
try:
for (root, dirs, files) in os.walk(abs_path):
for filename in fnmatch.filter(files, name_pattern):
# IMPORTANT: this join always yields abs expanded path
abs_path = os.path.join(root, filename)
relative_path = abs_path.replace(base_dir, '')
if not valid_user_path(configuration, abs_path, base_dir,
True):
continue
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'file_with_dir': relative_path,
'flags': flags,
'special': '',
}
entries.append(file_obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({'object_type': 'file_output',
'path': relative_path, 'lines': output_lines})
else:
output_objects.append({'object_type': 'file_output',
'lines': output_lines})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# Show all if no type flags given
if not byte_count(flags) and not line_count(flags)\
and not word_count(flags):
flags += 'blw'
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
filewcs = []
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
(bytes, words, lines) = (0, 0, 0)
try:
obj = {'object_type': 'filewc', 'name': relative_path}
if os.path.isdir(abs_path):
obj = {
'object_type': 'filewc',
'name':
'%s: %s: Is a directory' % (op_name, relative_path),
'lines': 0,
'words': 0,
'bytes': 0,
}
filewcs.append(obj)
continue
fd = open(abs_path, 'r')
lines = 0
# use file directly as iterator for efficiency
for line in fd:
lines += 1
bytes += len(line)
words += len(line.split())
if line_count(flags):
obj['lines'] = lines
if word_count(flags):
obj['words'] = words
if byte_count(flags):
obj['bytes'] = bytes
filewcs.append(obj)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filewcs', 'filewcs': filewcs})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
try:
filedes = open(real_path, 'r')
i = 0
for line in filedes:
if i >= lines:
break
output_lines.append(line)
i += 1
filedes.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = accepted['flags']
patterns = accepted['job_id']
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
mrsl_keywords_dict = get_keywords_dict(configuration)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in patterns:
# Add file extension
pattern += '.mRSL'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
mrsl_dict = unpickle(abs_path, logger)
if not mrsl_dict:
raise Exception('could not load job mRSL')
for (key, val) in mrsl_dict.items():
if not key in mrsl_keywords_dict.keys():
continue
if not val:
continue
output_lines.append('::%s::\n' % key)
if 'multiplestrings' == mrsl_keywords_dict[key]['Type']:
for line in val:
output_lines.append('%s\n' % line)
elif 'multiplekeyvalues' == mrsl_keywords_dict[key][
'Type']:
for (left, right) in val:
output_lines.append('%s=%s\n' % (left, right))
else:
output_lines.append('%s\n' % val)
output_lines.append('\n')
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if verbose(flags):
output_objects.append({
'object_type': 'file_output',
'path': relative_path,
'lines': output_lines
})
else:
output_objects.append({
'object_type': 'file_output',
'lines': output_lines
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
logger.error("jobstatus input validation failed: %s" % accepted)
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
max_jobs = int(accepted['max_jobs'][-1])
order = 'unsorted '
if sorted(flags):
order = 'sorted '
patterns = accepted['job_id']
project_names = accepted['project_name']
if len(project_names) > 0:
for project_name in project_names:
project_name_job_ids = \
get_job_ids_with_specified_project_name(client_id,
project_name, configuration.mrsl_files_dir, logger)
patterns.extend(project_name_job_ids)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text'
: '%s %s job status' % \
(configuration.short_title, order)})
if not patterns:
output_objects.append({'object_type': 'error_text', 'text'
: 'No job_id specified!'})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
if not os.path.isdir(base_dir):
output_objects.append(
{'object_type': 'error_text', 'text'
: ('You have not been created as a user on the %s server! ' \
'Please contact the %s team.') % \
(configuration.short_title, configuration.short_title)})
return (output_objects, returnvalues.CLIENT_ERROR)
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s: You do not have any matching job IDs!' % pattern})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
if sorted(flags):
sort(filelist)
if max_jobs < len(filelist):
output_objects.append(
{'object_type': 'text', 'text'
: 'Only showing first %d of the %d matching jobs as requested'
% (max_jobs, len(filelist))})
filelist = filelist[:max_jobs]
# Iterate through jobs and print details for each
job_list = {'object_type': 'job_list', 'jobs': []}
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No such job: %s (could not load mRSL file %s)' % \
(job_id, filepath)})
continue
# Expand any job variables before use
job_dict = expand_variables(job_dict)
job_obj = {'object_type': 'job', 'job_id': job_id}
job_obj['status'] = job_dict['STATUS']
time_fields = [
'VERIFIED',
'VERIFIED_TIMESTAMP',
'RECEIVED_TIMESTAMP',
'QUEUED_TIMESTAMP',
'SCHEDULE_TIMESTAMP',
'EXECUTING_TIMESTAMP',
'FINISHED_TIMESTAMP',
'FAILED_TIMESTAMP',
'CANCELED_TIMESTAMP',
]
for name in time_fields:
if job_dict.has_key(name):
# time objects cannot be marshalled, asctime if timestamp
try:
job_obj[name.lower()] = time.asctime(job_dict[name])
except Exception, exc:
# not a time object, just add
job_obj[name.lower()] = job_dict[name]
###########################################
# ARC job status retrieval on demand:
# But we should _not_ update the status in the mRSL files, since
# other MiG code might rely on finding only valid "MiG" states.
if configuration.arc_clusters and \
job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \
and job_dict['STATUS'] == 'EXECUTING':
try:
home = os.path.join(configuration.user_home, client_dir)
arcsession = arc.Ui(home)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arc.ARCWrapperError, err:
logger.error('Error retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arc.NoProxyError, err:
logger.error('While retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path_list = accepted['path']
size_list = [int(size) for size in accepted['size']]
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s Upload Progress Monitor' % configuration.short_title
if not configuration.site_enable_griddk:
output_objects.append({'object_type': 'text', 'text':
'''Grid.dk features are disabled on this site.
Please contact the Grid admins %s if you think they should be enabled.
''' % configuration.admin_email})
return (output_objects, returnvalues.OK)
refresh_secs = 5
meta = '<meta http-equiv="refresh" content="%s" />' % refresh_secs
title_entry['meta'] = meta
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text'
: 'Upload progress'})
done_list = [False for _ in path_list]
progress_items = []
index = -1
for path in path_list:
index += 1
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
real_path = os.path.abspath(os.path.join(base_dir, path))
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, path))
output_objects.append({'object_type': 'file_not_found',
'name': path})
status = returnvalues.FILE_NOT_FOUND
continue
if not os.path.isfile(real_path):
output_objects.append({'object_type': 'error_text', 'text'
: "no such upload: %s" % path})
status = returnvalues.CLIENT_ERROR
continue
relative_path = real_path.replace(base_dir, '')
try:
logger.info('Checking size of upload %s' % real_path)
cur_size = os.path.getsize(real_path)
total_size = size_list[index]
percent = round(100.0 * cur_size / total_size, 1)
done_list[index] = (cur_size == total_size)
progress_items.append({'object_type': 'progress', 'path': path,
'cur_size': cur_size, 'total_size':
total_size, 'percent': percent,
'done': done_list[index], 'progress_type':
'upload'})
except Exception, exc:
# Don't give away information about actual fs layout
output_objects.append({'object_type': 'error_text', 'text'
: '%s upload could not be checked! (%s)'
% (path, str(exc).replace(base_dir, ''
))})
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
dst = accepted['dst'][-1].lstrip(os.sep)
pattern_list = accepted['src']
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
dst = os.path.join(current_dir, dst)
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'ZIP/TAR archiver'
output_objects.append(
{'object_type': 'header', 'text': 'ZIP/TAR archiver'})
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
if 'h' in flags:
usage(output_objects)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(os.path.join(base_dir,
current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
# out of bounds
output_objects.append({'object_type': 'error_text', 'text':
"You're not allowed to work in %s!"
% current_dir})
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append(
{'object_type': 'text', 'text': "working in %s" % current_dir})
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('pack in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
# out of bounds
output_objects.append(
{'object_type': 'error_text', 'text':
"Invalid path! (%s expands to an illegal path)" % dst})
logger.warning('%s tried to %s restricted path %s !(%s)'
% (client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(abs_dest)):
output_objects.append({'object_type': 'error_text', 'text':
"No such destination directory: %s"
% os.path.dirname(relative_dest)})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot pack to "%s": inside a read-only location!' %
relative_dest})
return (output_objects, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
logger.debug("%s: inspecting: %s" % (op_name, server_path))
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
else:
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'error_text', 'text':
"%s: cannot pack '%s': no valid src paths"
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
continue
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
# Generally refuse handling symlinks including root vgrid shares
if os.path.islink(abs_path):
output_objects.append(
{'object_type': 'warning', 'text': """You're not allowed to
pack entire special folders like %s shared folders!""" %
configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
# Additionally refuse operations on inherited subvgrid share roots
elif in_vgrid_share(configuration, abs_path) == relative_path:
output_objects.append(
{'object_type': 'warning', 'text': """You're not allowed to
pack entire %s shared folders!""" % configuration.site_vgrid_label})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_path) == os.path.realpath(base_dir):
logger.error("%s: refusing pack home dir: %s" % (op_name,
abs_path))
output_objects.append(
{'object_type': 'warning', 'text':
"You're not allowed to pack your entire home directory!"
})
status = returnvalues.CLIENT_ERROR
continue
elif os.path.realpath(abs_dest) == os.path.realpath(abs_path):
output_objects.append({'object_type': 'warning', 'text':
'overlap in source and destination %s'
% relative_dest})
status = returnvalues.CLIENT_ERROR
continue
if verbose(flags):
output_objects.append(
{'object_type': 'file', 'name': relative_path})
(pack_status, msg) = pack_archive(configuration, client_id,
relative_path, relative_dest)
if not pack_status:
output_objects.append({'object_type': 'error_text',
'text': 'Error: %s' % msg})
status = returnvalues.CLIENT_ERROR
continue
output_objects.append({'object_type': 'text', 'text':
'Added %s to %s' % (relative_path,
relative_dest)})
output_objects.append({'object_type': 'text', 'text':
'Packed archive of %s is now available in %s'
% (', '.join(pattern_list), relative_dest)})
output_objects.append({'object_type': 'link', 'text': 'Download archive',
'destination': os.path.join('..', client_dir,
relative_dest)})
return (output_objects, status)
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text':
'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(dst_base, '')
if not valid_user_path(configuration, abs_dest, dst_base, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text':
"Invalid destination (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
# We must make sure target dir exists if called in import X mode
if (share_id or freeze_id) and not makedirs_rec(abs_dest, configuration):
logger.error('could not create import destination dir: %s' % abs_dest)
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot import to "%s" : file in the way?' % relative_dest})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s'
def main(client_id, user_arguments_dict, environ=None):
"""Main function used by front end"""
if environ is None:
environ = os.environ
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
dst = accepted['dst'][-1].lstrip(os.sep)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
dst_mode = "wb"
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
relative_dst = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
logger.warning('%s tried to %s into restricted path %s ! (%s)' %
(client_id, op_name, abs_dest, dst))
output_objects.append({
'object_type': 'error_text',
'text': "invalid destination: '%s'" % dst
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'accessed', [relative_path])
fd = open(abs_path, 'r')
# use file directly as iterator for efficiency
for line in fd:
output_lines.append(line)
fd.close()
except Exception, exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'accessed', [relative_path],
failed=True,
details=exc)
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
try:
gdp_iolog(configuration, client_id, environ['REMOTE_ADDR'],
'modified', [dst])
out_fd = open(abs_dest, dst_mode)
out_fd.writelines(output_lines)
out_fd.close()
logger.info('%s %s %s done' %
(op_name, abs_path, abs_dest))
except Exception, exc:
if not isinstance(exc, GDPIOLogError):
gdp_iolog(configuration,
client_id,
environ['REMOTE_ADDR'],
'modified', [dst],
error=True,
details=exc)
output_objects.append({
'object_type': 'error_text',
'text': "write failed: '%s'" % exc
})
logger.error("%s: write failed on '%s': %s" %
(op_name, abs_dest, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({
'object_type':
'text',
'text':
"wrote %s to %s" % (relative_path, relative_dst)
})
# Prevent truncate after first write
dst_mode = "ab+"
else:
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
# TODO: rip this hack out into real download handler?
# Force download of files when output_format == 'file_format'
# This will only work for the first file matching a glob when
# using file_format.
# And it is supposed to only work for one file.
if user_arguments_dict.has_key('output_format'):
output_format = user_arguments_dict['output_format'][0]
if output_format == 'file':
output_objects.append({
'object_type':
'start',
'headers': [('Content-Disposition',
'attachment; filename="%s";' %
os.path.basename(abs_path))]
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job schedule is hard on the server, limit
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
saveschedulejobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
saveschedulejob = {'object_type': 'saveschedulejob', 'job_id': job_id}
dict = unpickle(filepath, logger)
if not dict:
saveschedulejob['message'] = \
('The file containing the information' \
' for job id %s could not be opened!' \
' You can only read schedule for ' \
'your own jobs!') % job_id
saveschedulejobs.append(saveschedulejob)
status = returnvalues.CLIENT_ERROR
continue
saveschedulejob['oldstatus'] = dict['STATUS']
# Is the job status pending?
possible_schedule_states = ['QUEUED', 'RETRY', 'FROZEN']
if not dict['STATUS'] in possible_schedule_states:
saveschedulejob['message'] = \
'You can only read schedule for jobs with status: %s.'\
% ' or '.join(possible_schedule_states)
saveschedulejobs.append(saveschedulejob)
continue
# notify queue
if not send_message_to_grid_script('JOBSCHEDULE ' + job_id + '\n',
logger, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
'Error sending message to grid_script, update may fail.'
})
status = returnvalues.SYSTEM_ERROR
continue
saveschedulejobs.append(saveschedulejob)
savescheduleinfo = """Please find any available job schedule status in
verbose job status output."""
output_objects.append({
'object_type': 'saveschedulejobs',
'saveschedulejobs': saveschedulejobs,
'savescheduleinfo': savescheduleinfo
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
output_lines = []
# We search for the last 'lines' lines by beginning from the end of
# the file and exponetially backtracking until the backtrack
# contains at least 'lines' lines or we're back to the beginning of
# the file.
# At that point we skip any extra lines before printing.
try:
filedes = open(real_path, 'r')
# Go to end of file and backtrack
backstep = 1
newlines = 0
filedes.seek(0, 2)
length = filedes.tell()
while backstep < length and newlines <= lines:
backstep *= 2
if backstep > length:
backstep = length
filedes.seek(-backstep, 2)
newlines = len(filedes.readlines())
if length:
# Go back after reading to end of file
filedes.seek(-backstep, 2)
# Skip any extra lines caused by the exponential backtracking.
# We could probably speed up convergence with binary search...
while newlines > lines:
dummy = filedes.readline()
newlines -= 1
backstep = length - filedes.tell()
# Now we're at the wanted spot - print rest of file
for _ in range(newlines):
output_lines.append(filedes.readline())
filedes.close()
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
# Show all if no type flags given
if not byte_count(flags) and not line_count(flags)\
and not word_count(flags):
flags += 'blw'
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
filewcs = []
for real_path in match:
relative_path = real_path.replace(base_dir, '')
(bytes, words, lines) = (0, 0, 0)
try:
obj = {'object_type': 'filewc', 'name': relative_path}
if os.path.isdir(real_path):
obj = {
'object_type': 'filewc',
'name': '%s: %s: Is a directory' % (op_name,
relative_path),
'lines': 0,
'words': 0,
'bytes': 0,
}
filewcs.append(obj)
continue
fd = open(real_path, 'r')
lines = 0
# use file directly as iterator for efficiency
for line in fd:
lines += 1
bytes += len(line)
words += len(line.split())
if line_count(flags):
obj['lines'] = lines
if word_count(flags):
obj['words'] = words
if byte_count(flags):
obj['bytes'] = bytes
filewcs.append(obj)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filewcs', 'filewcs'
: filewcs})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
# TODO: if validator is too tight we should accept rejects here
# and then make sure that such rejected fields are never printed
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
current_dir = accepted['current_dir'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# the client can choose to specify the path of the target directory with
# current_dir + "/" + path, instead of specifying the complete path in
# subdirs. This is usefull from ls.py where a hidden html control makes it
# possible to target the directory from the current dir.
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s file web editor' % configuration.short_title
title_entry['style'] = advanced_editor_css_deps()
title_entry['javascript'] = advanced_editor_js_deps()
title_entry['javascript'] += lock_info('this file', -1)
output_objects.append({'object_type': 'header', 'text'
: 'Editing file in %s home directory' % \
configuration.short_title })
if not path:
now = time.gmtime()
path = 'noname-%s.txt' % time.strftime('%d%m%y-%H%M%S', now)
output_objects.append({'object_type': 'text', 'text'
: 'No path supplied - creating new file in %s'
% path})
path = os.path.normpath(current_dir + path)
real_path = os.path.abspath(base_dir + current_dir + path)
if not valid_user_path(real_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, path))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid path! (%s expands to an illegal path)" % path})
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(real_path, client_id)
if owner == client_id:
javascript = \
'''<script type="text/javascript">
setTimeout("newcountdown('%s', %d)", 1)
</script>
'''\
% (path, time_left / 60)
output_objects.append({'object_type': 'html_form', 'text'
: javascript})
html = edit_file(path, real_path)
output_objects.append({'object_type': 'html_form', 'text'
: html})
else:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s acquired the editing lock for %s! (timeout in %d seconds)'
% (owner, path, time_left)})
return (output_objects, returnvalues.CLIENT_ERROR)
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# IMPORTANT: the CGI front end forces the input extraction to be delayed
# We must manually extract and parse input here to avoid memory explosion
# for huge files!
# TODO: explosions still happen sometimes!
# Most likely because of Apache SSL renegotiations which have
# no other way of storing input
extract_input = user_arguments_dict.get('__DELAYED_INPUT__', dict)
logger.info('Extracting input in %s' % op_name)
form = extract_input()
logger.info('After extracting input in %s' % op_name)
file_item = None
file_name = ''
user_arguments_dict = {}
if form.has_key('fileupload'):
file_item = form['fileupload']
file_name = file_item.filename
user_arguments_dict['fileupload'] = ['true']
user_arguments_dict['path'] = [file_name]
if form.has_key('path'):
user_arguments_dict['path'] = [form['path'].value]
if form.has_key('restrict'):
user_arguments_dict['restrict'] = [form['restrict'].value]
else:
user_arguments_dict['restrict'] = defaults['restrict']
logger.info('Filtered input is: %s' % user_arguments_dict)
# Now validate parts as usual
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
path = accepted['path'][-1]
restrict = accepted['restrict'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_griddk:
output_objects.append({
'object_type':
'text',
'text':
'''Grid.dk features are disabled on this site.
Please contact the site admins %s if you think they should be enabled.
''' % configuration.admin_email
})
return (output_objects, returnvalues.OK)
logger.info('Filtered input validated with result: %s' % accepted)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
output_objects.append({'object_type': 'header', 'text': 'Uploading file'})
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
real_path = os.path.realpath(os.path.join(base_dir, path))
# Implicit destination
if os.path.isdir(real_path):
real_path = os.path.join(real_path, os.path.basename(file_name))
if not valid_user_path(configuration, real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, real_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid destination (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(real_path)):
output_objects.append({
'object_type':
'error_text',
'text':
"cannot write: no such file or directory: %s)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
# We fork off here and redirect the user to a progress page for user
# friendly output and to avoid cgi timeouts from killing the upload.
# We use something like the Active State python recipe for daemonizing
# to properly detach from the CGI process and continue in the background.
# Please note that we only close stdio file descriptors to avoid closing
# the fileupload.
file_item.file.seek(0, 2)
total_size = file_item.file.tell()
file_item.file.seek(0, 0)
try:
pid = os.fork()
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
os.chdir('/')
os.umask(0)
for fno in range(3):
try:
os.close(fno)
except OSError:
pass
else:
os._exit(0)
except OSError, ose:
output_objects.append({
'object_type':
'error_text',
'text':
'%s upload could not background! (%s)' %
(path, str(ose).replace(base_dir, ''))
})
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# IMPORTANT: the CGI front end forces the input extraction to be delayed
# We must manually extract and parse input here to avoid memory explosion
# for huge files!
# TODO: explosions still happen sometimes!
# Most likely because of Apache SSL renegotiations which have
# no other way of storing input
extract_input = user_arguments_dict["__DELAYED_INPUT__"]
logger.info("Extracting input in %s" % op_name)
form = extract_input()
logger.info("After extracting input in %s" % op_name)
file_item = None
file_name = ""
user_arguments_dict = {}
if form.has_key("fileupload"):
file_item = form["fileupload"]
file_name = file_item.filename
user_arguments_dict["fileupload"] = ["true"]
user_arguments_dict["path"] = [file_name]
if form.has_key("path"):
user_arguments_dict["path"] = [form["path"].value]
if form.has_key("restrict"):
user_arguments_dict["restrict"] = [form["restrict"].value]
else:
user_arguments_dict["restrict"] = defaults["restrict"]
logger.info("Filtered input is: %s" % user_arguments_dict)
# Now validate parts as usual
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler("POST"):
output_objects.append(
{"object_type": "error_text", "text": "Only accepting POST requests to prevent unintended updates"}
)
return (output_objects, returnvalues.CLIENT_ERROR)
flags = "".join(accepted["flags"])
path = accepted["path"][-1]
restrict = accepted["restrict"][-1]
if not configuration.site_enable_griddk:
output_objects.append(
{
"object_type": "text",
"text": """Grid.dk features are disabled on this site.
Please contact the Grid admins %s if you think they should be enabled.
"""
% configuration.admin_email,
}
)
return (output_objects, returnvalues.OK)
logger.info("Filtered input validated with result: %s" % accepted)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({"object_type": "text", "text": "%s using flag: %s" % (op_name, flag)})
output_objects.append({"object_type": "header", "text": "Uploading file"})
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
real_path = os.path.realpath(os.path.join(base_dir, path))
# Implicit destination
if os.path.isdir(real_path):
real_path = os.path.join(real_path, os.path.basename(file_name))
if not valid_user_path(real_path, base_dir, True):
logger.warning("%s tried to %s restricted path %s ! (%s)" % (client_id, op_name, real_path, path))
output_objects.append(
{"object_type": "error_text", "text": "Invalid destination (%s expands to an illegal path)" % path}
)
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(real_path)):
output_objects.append(
{"object_type": "error_text", "text": "cannot write: no such file or directory: %s)" % path}
)
return (output_objects, returnvalues.CLIENT_ERROR)
# We fork off here and redirect the user to a progress page for user
# friendly output and to avoid cgi timeouts from killing the upload.
# We use something like the Active State python recipe for daemonizing
# to properly detach from the CGI process and continue in the background.
# Please note that we only close stdio file descriptors to avoid closing
# the fileupload.
file_item.file.seek(0, 2)
total_size = file_item.file.tell()
file_item.file.seek(0, 0)
try:
pid = os.fork()
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
os.chdir("/")
os.umask(0)
for fno in range(3):
try:
os.close(fno)
except OSError:
pass
else:
os._exit(0)
except OSError, ose:
output_objects.append(
{
"object_type": "error_text",
"text": "%s upload could not background! (%s)" % (path, str(ose).replace(base_dir, "")),
}
)
return (output_objects, returnvalues.SYSTEM_ERROR)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
src_list = accepted['src']
dst = accepted['dst'][-1]
iosessionid = accepted['iosessionid'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if not client_id:
base_dir = os.path.realpath(configuration.webserver_home
+ os.sep + iosessionid) + os.sep
status = returnvalues.OK
real_dest = base_dir + dst
dst_list = glob.glob(real_dest)
if not dst_list:
# New destination?
if not glob.glob(os.path.dirname(real_dest)):
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal dst path provided!'})
return (output_objects, returnvalues.CLIENT_ERROR)
else:
dst_list = [real_dest]
# Use last match in case of multiple matches
dest = dst_list[-1]
if len(dst_list) > 1:
output_objects.append(
{'object_type': 'warning', 'text'
: 'dst (%s) matches multiple targets - using last: %s'
% (dst, dest)})
real_dest = os.path.abspath(dest)
# Don't use real_path in output as it may expose underlying
# fs layout.
relative_dest = real_dest.replace(base_dir, '')
if not valid_user_path(real_dest, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_dest, dst))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid destination (%s expands to an illegal path)" % dst})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in src_list:
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
# src must be a file unless recursive is specified
if not recursive(flags) and os.path.isdir(real_path):
logger.warning('skipping directory source %s' % real_path)
output_objects.append({'object_type': 'warning', 'text'
: 'skipping directory src %s!' % relative_path})
continue
# If destination is a directory the src should be copied there
real_target = real_dest
if os.path.isdir(real_target):
real_target = os.path.join(real_target,
os.path.basename(real_path))
if os.path.abspath(real_path) == os.path.abspath(real_target):
logger.warning('%s tried to %s %s to itself! (%s)' % \
(client_id, op_name, real_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot copy '%s' to self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
if os.path.isdir(real_path) and \
real_target.startswith(real_path + os.sep):
logger.warning('%s tried to %s %s to itself! (%s)'
% (client_id, op_name, real_path, pattern))
output_objects.append(
{'object_type': 'warning', 'text'
: "Cannot copy '%s' to (sub) self!" % relative_path})
status = returnvalues.CLIENT_ERROR
continue
try:
if os.path.isdir(real_path):
shutil.copytree(real_path, real_target)
else:
shutil.copy(real_path, real_target)
logger.info('%s %s %s done' % (op_name, real_path, real_target))
except Exception, exc:
output_objects.append(
{'object_type': 'error_text',
'text': "%s: failed on '%s' to '%s'" \
% (op_name, relative_path, relative_dest)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
action = accepted['action'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not action in valid_actions.keys():
output_objects.append({
'object_type':
'error_text',
'text':
'Invalid job action "%s" (only %s supported)' %
(action, ', '.join(valid_actions.keys()))
})
return (output_objects, returnvalues.CLIENT_ERROR)
new_state = valid_actions[action]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.error(
'%s tried to use %s %s outside own home! (pattern %s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job state change is hard on the server, limit
if len(filelist) > 500:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
changedstatusjobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
changedstatusjob = {
'object_type': 'changedstatusjob',
'job_id': job_id
}
job_dict = unpickle(filepath, logger)
if not job_dict:
changedstatusjob['message'] = '''The file containing the
information for job id %s could not be opened! You can only %s your own
jobs!''' % (job_id, action)
changedstatusjobs.append(changedstatusjob)
status = returnvalues.CLIENT_ERROR
continue
changedstatusjob['oldstatus'] = job_dict['STATUS']
# Is the job status compatible with action?
possible_cancel_states = [
'PARSE', 'QUEUED', 'RETRY', 'EXECUTING', 'FROZEN'
]
if action == 'cancel' and \
not job_dict['STATUS'] in possible_cancel_states:
changedstatusjob['message'] = \
'You can only cancel jobs with status: %s.'\
% ' or '.join(possible_cancel_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
possible_freeze_states = ['QUEUED', 'RETRY']
if action == 'freeze' and \
not job_dict['STATUS'] in possible_freeze_states:
changedstatusjob['message'] = \
'You can only freeze jobs with status: %s.'\
% ' or '.join(possible_freeze_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
possible_thaw_states = ['FROZEN']
if action == 'thaw' and \
not job_dict['STATUS'] in possible_thaw_states:
changedstatusjob['message'] = \
'You can only thaw jobs with status: %s.'\
% ' or '.join(possible_thaw_states)
status = returnvalues.CLIENT_ERROR
changedstatusjobs.append(changedstatusjob)
continue
# job action is handled by changing the STATUS field, notifying the
# job queue and making sure the server never submits jobs with status
# FROZEN or CANCELED.
# file is repickled to ensure newest information is used, job_dict
# might be old if another script has modified the file.
if not unpickle_and_change_status(filepath, new_state, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'Job status could not be changed to %s!' % new_state
})
status = returnvalues.SYSTEM_ERROR
# Avoid key error and make sure grid_script gets expected number of
# arguments
if not job_dict.has_key('UNIQUE_RESOURCE_NAME'):
job_dict['UNIQUE_RESOURCE_NAME'] = \
'UNIQUE_RESOURCE_NAME_NOT_FOUND'
if not job_dict.has_key('EXE'):
job_dict['EXE'] = 'EXE_NAME_NOT_FOUND'
# notify queue
if not send_message_to_grid_script(
'JOBACTION ' + job_id + ' ' + job_dict['STATUS'] + ' ' +
new_state + ' ' + job_dict['UNIQUE_RESOURCE_NAME'] + ' ' +
job_dict['EXE'] + '\n', logger, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
'''Error sending message to grid_script,
job may still be in the job queue.'''
})
status = returnvalues.SYSTEM_ERROR
continue
changedstatusjob['newstatus'] = new_state
changedstatusjobs.append(changedstatusjob)
output_objects.append({
'object_type': 'changedstatusjobs',
'changedstatusjobs': changedstatusjobs
})
return (output_objects, status)
def run_transfer(configuration, client_id, transfer_dict):
"""Actual data transfer built from transfer_dict on behalf of client_id"""
logger.debug('run transfer for %s: %s' %
(client_id, blind_pw(transfer_dict)))
transfer_id = transfer_dict['transfer_id']
action = transfer_dict['action']
protocol = transfer_dict['protocol']
status_dir = get_status_dir(configuration, client_id, transfer_id)
cmd_map = get_cmd_map()
if not protocol in cmd_map[action]:
raise ValueError('unsupported protocol: %s' % protocol)
client_dir = client_id_dir(client_id)
makedirs_rec(status_dir, configuration)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# TODO: we should refactor to move command extraction into one function
command_pattern = cmd_map[action][protocol]
target_helper_list = []
key_path = transfer_dict.get("key", "")
if key_path:
# Use key with given name from settings dir
settings_base_dir = os.path.abspath(
os.path.join(configuration.user_settings, client_dir)) + os.sep
key_path = os.path.join(settings_base_dir, user_keys_dir,
key_path.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
key_path = os.path.abspath(key_path)
if not valid_user_path(configuration, key_path, settings_base_dir):
logger.error('rejecting illegal directory traversal for %s (%s)' %
(key_path, blind_pw(transfer_dict)))
raise ValueError("user provided a key outside own settings!")
rel_src_list = transfer_dict['src']
rel_dst = transfer_dict['dst']
compress = transfer_dict.get("compress", False)
exclude = transfer_dict.get("exclude", [])
if transfer_dict['action'] in ('import', ):
logger.debug('setting abs dst for action %(action)s' % transfer_dict)
src_path_list = transfer_dict['src']
dst_path = os.path.join(base_dir, rel_dst.lstrip(os.sep))
dst_path = os.path.abspath(dst_path)
for src in rel_src_list:
abs_dst = os.path.join(dst_path, src.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dst = os.path.abspath(abs_dst)
# Reject illegal directory traversal and hidden files
if not valid_user_path(configuration, abs_dst, base_dir, True):
logger.error(
'rejecting illegal directory traversal for %s (%s)' %
(abs_dst, blind_pw(transfer_dict)))
raise ValueError("user provided a destination outside home!")
if src.endswith(os.sep):
target_helper_list.append(
(get_lftp_target(True, False, exclude),
get_rsync_target(True, False, exclude, compress)))
else:
target_helper_list.append(
(get_lftp_target(True, True, exclude),
get_rsync_target(True, True, exclude, compress)))
makedirs_rec(dst_path, configuration)
elif transfer_dict['action'] in ('export', ):
logger.debug('setting abs src for action %(action)s' % transfer_dict)
dst_path = transfer_dict['dst']
src_path_list = []
for src in rel_src_list:
src_path = os.path.join(base_dir, src.lstrip(os.sep))
# IMPORTANT: path must be expanded to abs for proper chrooting
src_path = os.path.abspath(src_path)
# Reject illegal directory traversal and hidden files
if not valid_user_path(configuration, src_path, base_dir, True):
logger.error(
'rejecting illegal directory traversal for %s (%s)' %
(src, blind_pw(transfer_dict)))
raise ValueError("user provided a source outside home!")
src_path_list.append(src_path)
if src.endswith(os.sep) or os.path.isdir(src):
target_helper_list.append(
(get_lftp_target(False, False, exclude),
get_rsync_target(False, False, exclude, compress)))
else:
target_helper_list.append(
(get_lftp_target(False, True, exclude),
get_rsync_target(False, True, exclude, compress)))
else:
raise ValueError('unsupported action for %(transfer_id)s: %(action)s' %
transfer_dict)
run_dict = transfer_dict.copy()
run_dict['log_path'] = os.path.join(status_dir, 'transfer.log')
# Use private known hosts file for ssh transfers as explained above
# NOTE: known_hosts containing '=' silently leads to rest getting ignored!
# use /dev/null to skip host key verification completely for now.
#run_dict['known_hosts'] = os.path.join(base_dir, '.ssh', 'known_hosts')
run_dict['known_hosts'] = '/dev/null'
# Make sure password is set to empty string as default
run_dict['password'] = run_dict.get('password', '')
# TODO: this is a bogus cert path for now - we don't support ssl certs
run_dict['cert'] = run_dict.get('cert', '')
# IMPORTANT: must be implicit proto or 'ftp://' (not ftps://) and similarly
# webdav(s) must use explicit http(s) instead. In both cases we
# replace protocol between cmd selection and lftp path expansion
if run_dict['protocol'] == 'ftps':
run_dict['orig_proto'] = run_dict['protocol']
run_dict['protocol'] = 'ftp'
logger.info(
'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
run_dict)
elif run_dict['protocol'].startswith('webdav'):
run_dict['orig_proto'] = run_dict['protocol']
run_dict['protocol'] = run_dict['protocol'].replace('webdav', 'http')
logger.info(
'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
run_dict)
if key_path:
rel_key = run_dict['key']
rel_cert = run_dict['cert']
run_dict['key'] = key_path
run_dict['cert'] = key_path.replace(rel_key, rel_cert)
run_dict['ssh_auth'] = get_ssh_auth(True, run_dict)
run_dict['ssl_auth'] = get_ssl_auth(True, run_dict)
else:
# Extract encrypted password
password_digest = run_dict.get('password_digest', '')
if password_digest:
_, _, _, payload = password_digest.split("$")
unscrambled = unscramble_digest(configuration.site_digest_salt,
payload)
_, _, password = unscrambled.split(":")
run_dict['password'] = password
run_dict['ssh_auth'] = get_ssh_auth(False, run_dict)
run_dict['ssl_auth'] = get_ssl_auth(False, run_dict)
run_dict['rel_dst'] = rel_dst
run_dict['dst'] = dst_path
run_dict['lftp_buf_size'] = run_dict.get('lftp_buf_size',
lftp_buffer_bytes)
run_dict['lftp_sftp_block_size'] = run_dict.get('sftp_sftp_block_size',
lftp_sftp_block_bytes)
status = 0
for (src, rel_src, target_helper) in zip(src_path_list, rel_src_list,
target_helper_list):
(lftp_target, rsync_target) = target_helper
logger.debug('setting up %(action)s for %(src)s' % run_dict)
if run_dict['protocol'] == 'sftp' and not os.path.isabs(src):
# NOTE: lftp interprets sftp://FQDN/SRC as absolute path /SRC
# We force relative paths into user home with a tilde.
# The resulting sftp://FQDN/~/SRC looks funky but works.
run_dict['src'] = "~/%s" % src
else:
# All other paths are probably absolute or auto-chrooted anyway
run_dict['src'] = src
run_dict['rel_src'] = rel_src
run_dict['lftp_args'] = ' '.join(lftp_target[0]) % run_dict
run_dict['lftp_excludes'] = ' '.join(lftp_target[1])
# src and dst may actually be reversed for lftp, but for symmetry ...
run_dict['lftp_src'] = lftp_target[2][0] % run_dict
run_dict['lftp_dst'] = lftp_target[2][1] % run_dict
run_dict['rsync_args'] = ' '.join(rsync_target[0]) % run_dict
# Preserve excludes on list form for rsync, where it matters
run_dict[RSYNC_EXCLUDES_LIST] = rsync_target[1]
run_dict['rsync_src'] = rsync_target[2][0] % run_dict
run_dict['rsync_dst'] = rsync_target[2][1] % run_dict
blind_dict = blind_pw(run_dict)
logger.debug('expanded vars to %s' % blind_dict)
# NOTE: Make sure NOT to break rsync excludes on list form as they
# won't work if concatenated to a single string in command_list!
command_list, blind_list = [], []
for i in command_pattern:
if i == RSYNC_EXCLUDES_LIST:
command_list += run_dict[RSYNC_EXCLUDES_LIST]
blind_list += run_dict[RSYNC_EXCLUDES_LIST]
else:
command_list.append(i % run_dict)
blind_list.append(i % blind_dict)
command_str = ' '.join(command_list)
blind_str = ' '.join(blind_list)
logger.info('run %s on behalf of %s' % (blind_str, client_id))
transfer_proc = subprocess_popen(command_list,
stdout=subprocess_pipe,
stderr=subprocess_pipe)
# Save transfer_proc.pid for use in clean up during shutdown
# in that way we can resume pretty smoothly in next run.
sub_pid = transfer_proc.pid
logger.info('%s %s running transfer process %s' %
(client_id, transfer_id, sub_pid))
add_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
sub_pid)
out, err = transfer_proc.communicate()
exit_code = transfer_proc.wait()
status |= exit_code
del_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
sub_pid)
logger.info('done running transfer %s: %s' % (transfer_id, blind_str))
logger.debug('raw output is: %s' % out)
logger.debug('raw error is: %s' % err)
logger.debug('result was %s' % exit_code)
if not transfer_result(configuration, client_id, run_dict, exit_code,
out.replace(base_dir, ''),
err.replace(base_dir, '')):
logger.error('writing transfer status for %s failed' % transfer_id)
logger.debug('done handling transfers in %(transfer_id)s' % transfer_dict)
transfer_dict['exit_code'] = status
if status == 0:
transfer_dict['status'] = 'DONE'
else:
transfer_dict['status'] = 'FAILED'
valid_job_id(pattern, min_length, max_length, extra_chars)
def valid_user_path_name(safe_path, path, home_dir, allow_equal=False):
"""Wrap valid_user_path and valid_path name checks in one to check both
destination dir and filename characters. Returns error using safe_path if
validation fails.
"""
(status, msg) = (True, "")
try:
valid_path(path)
except InputException, iex:
status = False
msg = "Invalid path! (%s: %s)" % (safe_path, iex)
if not valid_user_path(path, home_dir, allow_equal):
status = False
msg = "Invalid path! (%s expands to illegal path)" % safe_path
return (status, html_escape(msg))
def valid_email_address(addr):
"""Email check from
http://www.secureprogramming.com/?action=view&feature=recipes&recipeid=1
"""
rfc822_specials = '()<>@,;:\\"[]'
# First we validate the name portion (name@domain)
c = 0
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
job_ids = accepted['job_id']
action = accepted['action'][-1]
src = accepted['src']
dst = accepted['dst'][-1]
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s live I/O' % configuration.short_title
output_objects.append({'object_type': 'header', 'text'
: 'Request live communication with jobs'})
if not action in valid_actions:
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid action "%s" (supported: %s)' % \
(action, ', '.join(valid_actions))})
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions and not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
if not job_ids or action in interactive_actions:
job_id = ''
if job_ids:
job_id = job_ids[-1]
output_objects.append({'object_type': 'text', 'text'
: '''
Fill in the live I/O details below to request communication with a running
job.
Job ID can be a full ID or a wild card pattern using "*" and "?" to match one
or more of your job IDs.
Use send output without source and destination paths to request upload of the
default stdio files from the job on the resource to the associated job_output
directory in your MiG home.
Destination is a always handled as a directory path to put source files into.
Source and destination paths are always taken relative to the job execution
directory on the resource and your MiG home respectively.
'''})
html = '''
<table class="liveio">
<tr>
<td>
<form method="post" action="liveio.py">
<table class="liveio">
<tr><td class=centertext>
</td></tr>
<tr><td>
Action:<br />
<input type=radio name=action checked value="send" />send output
<input type=radio name=action value="get" />get input
</td></tr>
<tr><td>
Job ID:<br />
<input type=text size=60 name=job_id value="%s" />
</td></tr>
<tr><td>
Source path(s):<br />
<div id="srcfields">
<input type=text size=60 name=src value="" /><br />
</div>
</td></tr>
<tr><td>
Destination path:<br />
<input type=text size=60 name=dst value="" />
</td></tr>
<tr><td>
<input type="submit" value="Send request" />
</td></tr>
</table>
</form>
</td>
<td>
<script type="text/javascript">
fields = 1;
max_fields = 64;
function addInput() {
if (fields < max_fields) {
document.getElementById("srcfields").innerHTML += "<input type=text size=60 name=src value='' /><br />";
fields += 1;
} else {
alert("Maximum " + max_fields + " source fields allowed!");
document.form.add.disabled=true;
}
}
</script>
<form name="addsrcform">
<input type="button" onclick="addInput(); return false;" name="add" value="Add another source field" />
</form>
</td>
</tr>
</table>
''' % job_id
output_objects.append({'object_type': 'html_form', 'text'
: html})
output_objects.append({'object_type': 'text', 'text': '''
Further live job control is avalable through your personal message queues.
They provide a basic interface for centrally storing messages under your grid
account and can be used to pass messages between jobs or for orchestrating
jobs before and during execution.
'''
})
output_objects.append({'object_type': 'link', 'destination':
'mqueue.py',
'text': 'Message queue interface'})
return (output_objects, returnvalues.OK)
elif action in ['get', 'receive', 'input']:
action = 'get'
action_desc = 'will be downloaded to the job on the resource'
elif action in ['put', 'send', 'output']:
action = 'send'
action_desc = 'will be uploaded from the job on the resource'
else:
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid live io action: %s' % action})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({'object_type': 'text', 'text'
: 'Requesting live I/O for %s'
% ', '.join(job_ids)})
if action == 'get' and (not src or not dst):
output_objects.append(
{'object_type': 'error_text',
'text': 'src and dst parameters required for live input'})
return (output_objects, returnvalues.CLIENT_ERROR)
# Automatic fall back to stdio files if output with no path provided
if src:
src_text = 'The files ' + ' '.join(src)
else:
src_text = 'The job stdio files'
if dst:
dst_text = 'the ' + dst + ' directory'
else:
dst_text = 'the corresponding job_output directory'
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
for job_id in job_ids:
job_id = job_id.strip()
# is job currently being executed?
# Backward compatibility - all_jobs keyword should match all jobs
if job_id == all_jobs:
job_id = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + job_id + '.mRSL')
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning("%s tried to %s restricted path %s ! (%s)" % \
(client_id, op_name, real_path, job_id))
continue
# Insert valid job files in filelist for later treatment
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: '%s: You do not have any matching job IDs!' % job_id})
else:
filelist += match
for filepath in filelist:
# Extract jo_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text'
: ('You can only list status of your own jobs. '
'Please verify that you submitted the mRSL file '
'with job id "%s" (Could not unpickle mRSL file %s)'
) % (job_id, filepath)})
continue
if job_dict['STATUS'] != 'EXECUTING':
output_objects.append(
{'object_type': 'text', 'text'
: 'Job %s is not currently being executed! Job status: %s'
% (job_id, job_dict['STATUS'])})
continue
if job_dict['UNIQUE_RESOURCE_NAME'] == 'ARC':
output_objects.append(
{'object_type': 'text', 'text'
: 'Job %s is submitted to ARC, details are not available!'
% job_id })
continue
last_live_update_dict = {}
last_live_update_file = configuration.mig_system_files + os.sep\
+ job_id + '.last_live_update'
if os.path.isfile(last_live_update_file):
last_live_update_dict_unpickled = \
unpickle(last_live_update_file, logger)
if not last_live_update_dict_unpickled:
output_objects.append({'object_type': 'error_text',
'text'
: 'Could not unpickle %s - skipping request!'
% last_live_update_file})
continue
if not last_live_update_dict_unpickled.has_key(
'LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'):
output_objects.append(
{'object_type': 'error_text',
'text': 'Could not find needed key in %s.'
% last_live_update_file})
continue
last_live_update_request = \
last_live_update_dict_unpickled['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'
]
difference = datetime.datetime.now()- last_live_update_request
try:
min_delay = \
int(configuration.min_seconds_between_live_update_requests)
except:
min_delay = 30
if difference.seconds < min_delay:
output_objects.append(
{'object_type': 'error_text',
'text': ('Request not allowed, you must wait at least ' \
'%s seconds between live update requests!'
) % min_delay})
continue
# save this request to file to avoid DoS from a client request loop.
last_live_update_dict['LAST_LIVE_UPDATE_REQUEST_TIMESTAMP'] = \
datetime.datetime.now()
pickle_ret = pickle(last_live_update_dict,
last_live_update_file, logger)
if not pickle_ret:
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Error saving live io request timestamp to last_live_update '
'file, request not sent!'})
continue
# #
# ## job is being executed right now, send live io request to frontend
# #
# get resource_config, needed by scp_file_to_resource
#(status, resource_config) = get_resource_configuration(
# resource_home, unique_resource_name, logger)
resource_config = job_dict['RESOURCE_CONFIG']
(status, exe) = get_resource_exe(resource_config, job_dict['EXE'],
logger)
if not status:
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Could not get exe configuration for job %s' % job_id})
continue
local_file = '%s.%supdate' % (job_dict['LOCALJOBNAME'], action)
if not os.path.exists(local_file):
# create
try:
filehandle = open(local_file, 'w')
filehandle.write('job_id '
+ job_dict['JOB_ID'] + '\n')
filehandle.write('localjobname '
+ job_dict['LOCALJOBNAME'] + '\n')
filehandle.write('execution_user '
+ exe['execution_user'] + '\n')
filehandle.write('execution_node '
+ exe['execution_node'] + '\n')
filehandle.write('execution_dir ' + exe['execution_dir']
+ '\n')
filehandle.write('target liveio\n')
# Leave defaults src and dst to FE script if not provided
if src:
filehandle.write('source ' + ' '.join(src) + '\n')
if dst:
filehandle.write('destination ' + dst + '\n')
# Backward compatible test for shared_fs - fall back to scp
if exe.has_key('shared_fs') and exe['shared_fs']:
filehandle.write('copy_command cp\n')
filehandle.write('copy_frontend_prefix \n')
filehandle.write('copy_execution_prefix \n')
else:
filehandle.write('copy_command scp -B\n')
filehandle.write('copy_frontend_prefix ${frontend_user}@${frontend_node}:\n'
)
filehandle.write('copy_execution_prefix ${execution_user}@${execution_node}:\n'
)
filehandle.write('### END OF SCRIPT ###\n')
filehandle.close()
except Exception, exc:
pass
if not os.path.exists(local_file):
output_objects.append(
{'object_type': 'error_text', 'text'
: '.%supdate file not available on %s server' % \
(action, configuration.short_title)})
continue
scpstatus = copy_file_to_resource(local_file, '%s.%supdate'
% (job_dict['LOCALJOBNAME'], action), resource_config, logger)
if not scpstatus:
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Error sending request for live io to resource!'})
continue
else:
output_objects.append(
{'object_type': 'text', 'text'
: 'Request for live io was successfully sent to the resource!'
})
output_objects.append(
{'object_type': 'text', 'text'
: '%s %s and should become available in %s in a minute.' % \
(src_text, action_desc, dst_text)
})
if action == 'send':
if not dst:
target_path = '%s/%s/*' % (job_output_dir, job_id)
else:
target_path = dst
output_objects.append({'object_type': 'link', 'destination'
: 'ls.py?path=%s' % target_path,
'text': 'View uploaded files'})
else:
output_objects.append({'object_type': 'link', 'destination'
: 'ls.py?path=%s' % ';path='.join(src),
'text': 'View files for download'})
try:
os.remove(local_file)
except Exception, exc:
pass
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
size = int(accepted['size'][-1])
pattern_list = accepted['path']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if size < 0:
output_objects.append({
'object_type': 'error_text',
'text': 'size must be non-negative'
})
return (output_objects, returnvalues.CLIENT_ERROR)
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({
'object_type': 'file',
'name': relative_path
})
if not check_write_access(abs_path):
logger.warning('%s called without write access: %s' % \
(op_name, abs_path))
output_objects.append(
{'object_type': 'error_text', 'text':
'cannot truncate "%s": inside a read-only location!' % \
pattern})
status = returnvalues.CLIENT_ERROR
continue
try:
fd = open(abs_path, 'r+')
fd.truncate(size)
fd.close()
logger.info('%s %s %s done' % (op_name, abs_path, size))
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False,
op_menu=client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults, output_objects, allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
queue = accepted['queue'][-1]
action = accepted['action'][-1]
iosessionid = accepted['iosessionid'][-1]
msg = accepted['msg'][-1]
msg_id = accepted['msg_id'][-1]
# Web format for cert access and no header for SID access
if client_id:
output_objects.append({'object_type': 'header', 'text'
: 'Message queue %s' % action})
else:
output_objects.append({'object_type': 'start'})
# Always return at least a basic file_output entry
file_entry = {'object_type': 'file_output',
'lines': [],
'wrap_binary': True,
'wrap_targets': ['lines']}
if not action in valid_actions:
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid action "%s" (supported: %s)' % \
(action, ', '.join(valid_actions))})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Find user home from session or certificate
if iosessionid:
client_home = os.path.realpath(os.path.join(configuration.webserver_home,
iosessionid))
client_dir = os.path.basename(client_home)
elif client_id:
client_dir = client_id_dir(client_id)
else:
output_objects.append({'object_type': 'error_text', 'text'
: 'Either certificate or session ID is required'
})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if not os.path.isdir(base_dir):
output_objects.append({'object_type': 'error_text', 'text'
: 'No matching session or user home!'})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
mqueue_base = os.path.join(base_dir, mqueue_prefix) + os.sep
default_queue_dir = os.path.join(mqueue_base, default_mqueue)
# Create mqueue base and default queue dir if missing
if not os.path.exists(default_queue_dir):
try:
os.makedirs(default_queue_dir)
except:
pass
# IMPORTANT: path must be expanded to abs for proper chrooting
queue_path = os.path.abspath(os.path.join(mqueue_base, queue))
if not valid_user_path(configuration, queue_path, mqueue_base):
output_objects.append({'object_type': 'error_text', 'text'
: 'Invalid queue name: "%s"' % queue})
output_objects.append(file_entry)
return (output_objects, returnvalues.CLIENT_ERROR)
lock_path = os.path.join(mqueue_base, lock_name)
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
status = returnvalues.OK
if action == "interactive":
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {'queue': queue,
'msg': msg,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit, }
target_op = 'mqueue'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
output_objects.append({'object_type': 'text', 'text': '''
Fill in the fields below to control and access your personal message queues.
Jobs can receive from and send to the message queues during execution, and use
them as a means of job inter-communication. Expect message queue operations to
take several seconds on the resources, however. That is, use it for tasks like
orchestrating long running jobs, and not for low latency communication.
'''})
html = '''
<form name="mqueueform" method="%(form_method)s" action="%(target_op)s.py">
<table class="mqueue">
<tr><td class=centertext>
</td></tr>
<tr><td>
Action:<br />
<input type="hidden" name="%(csrf_field)s" value="%(csrf_token)s" />
<input type=radio name=action value="create" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />create queue
<input type=radio name=action checked value="send" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=false;" />send message to queue
<input type=radio name=action value="receive" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />receive message from queue
<input type=radio name=action value="remove" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />remove queue
<input type=radio name=action value="listqueues" onclick="javascript: document.mqueueform.queue.disabled=true; document.mqueueform.msg.disabled=true;" />list queues
<input type=radio name=action value="listmessages" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />list messages
<input type=radio name=action value="show" onclick="javascript: document.mqueueform.queue.disabled=false; document.mqueueform.msg.disabled=true;" />show message
</td></tr>
<tr><td>
Queue:<br />
<input class="fillwidth" type=text name=queue value="%(queue)s" />
</td></tr>
<tr><td>
<div id="msgfieldf">
<input class="fillwidth" type=text name=msg value="%(msg)s" /><br />
</div>
</td></tr>
<tr><td>
<input type="submit" value="Apply" />
</td></tr>
</table>
</form>
''' % fill_helpers
output_objects.append({'object_type': 'html_form', 'text'
: html})
output_objects.append({'object_type': 'text', 'text': '''
Further live job control is avalable through the live I/O interface.
They provide a basic interface for centrally managing input and output files
for active jobs.
'''
})
output_objects.append({'object_type': 'link', 'destination':
'liveio.py',
'text': 'Live I/O interface'})
return (output_objects, returnvalues.OK)
elif action == 'create':
try:
os.mkdir(queue_path)
output_objects.append({'object_type': 'text', 'text':
'New "%s" queue created' % queue})
except Exception, err:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create "%s" queue: "%s"' % \
(queue, err)})
status = returnvalues.CLIENT_ERROR
filename_field,
os.path.basename(rel_path),
files_field, "dummy", csrf_field,
csrf_token)
uploaded.append(file_entry)
logger.info('status done: %s' % ' '.join([i[0] for i in upload_files]))
return (output_objects, status)
elif action == 'move':
# Move automatically takes place relative to dst_dir and current_dir
for (rel_path, chunk_tuple) in upload_files:
abs_src_path = os.path.abspath(os.path.join(base_dir, rel_path))
# IMPORTANT: path must be expanded to abs for proper chrooting
dest_dir = os.path.abspath(os.path.join(base_dir, current_dir))
dest_path = os.path.join(dest_dir, os.path.basename(rel_path))
rel_dst = dest_path.replace(base_dir, '')
if not valid_user_path(configuration, dest_path, base_dir, True):
logger.error(
'%s tried to %s move to restricted path %s ! (%s)' %
(user_id, op_name, dest_path, current_dir))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid destination "
"(%s expands to an illegal path)" % current_dir
})
moved = False
elif not check_write_access(dest_path, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, dest_path))
output_objects.append({
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = accepted['flags']
patterns = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
stats = []
for real_path in match:
relative_path = real_path.replace(base_dir, '')
try:
(stat_status, stat) = stat_path(real_path, logger)
if stat_status:
if verbose(flags):
stat['name'] = relative_path
stat['object_type'] = 'stat'
stats.append(stat)
else:
output_objects.append({'object_type': 'error_text',
'text': stat})
status = returnvalues.SYSTEM_ERROR
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'stats', 'stats'
: stats})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
submitstatuslist = []
for abs_path in match:
output_lines = []
relative_path = abs_path.replace(base_dir, '')
submitstatus = {
'object_type': 'submitstatus',
'name': relative_path
}
try:
(job_status, newmsg, job_id) = new_job(abs_path, client_id,
configuration, False,
True)
except Exception, exc:
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
job_status = False
newmsg = "%s failed on '%s' (is it a valid mRSL file?)"\
% (op_name, relative_path)
job_id = None
if not job_status:
submitstatus['status'] = False
submitstatus['message'] = newmsg
status = returnvalues.CLIENT_ERROR
else:
submitstatus['status'] = True
submitstatus['job_id'] = job_id
submitstatuslist.append(submitstatus)
output_objects.append({
'object_type': 'submitstatuslist',
'submitstatuslist': submitstatuslist
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
logger.error("jobstatus input validation failed: %s" % accepted)
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
max_jobs = int(accepted['max_jobs'][-1])
order = 'unsorted '
if sorted(flags):
order = 'sorted '
patterns = accepted['job_id']
project_names = accepted['project_name']
if len(project_names) > 0:
for project_name in project_names:
project_name_job_ids = \
get_job_ids_with_specified_project_name(client_id,
project_name, configuration.mrsl_files_dir, logger)
patterns.extend(project_name_job_ids)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text'
: '%s %s job status' % \
(configuration.short_title, order)})
if not patterns:
output_objects.append({
'object_type': 'error_text',
'text': 'No job_id specified!'
})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
if not os.path.isdir(base_dir):
output_objects.append(
{'object_type': 'error_text', 'text'
: ('You have not been created as a user on the %s server! ' \
'Please contact the %s team.') % \
(configuration.short_title, configuration.short_title)})
return (output_objects, returnvalues.CLIENT_ERROR)
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match....
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
if sorted(flags):
sort(filelist)
if max_jobs > 0 and max_jobs < len(filelist):
output_objects.append({
'object_type':
'text',
'text':
'Only showing first %d of the %d matching jobs as requested' %
(max_jobs, len(filelist))
})
filelist = filelist[:max_jobs]
# Iterate through jobs and list details for each
job_list = {'object_type': 'job_list', 'jobs': []}
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
job_dict = unpickle(filepath, logger)
if not job_dict:
status = returnvalues.CLIENT_ERROR
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No such job: %s (could not load mRSL file %s)' % \
(job_id, filepath)})
continue
# Expand any job variables before use
job_dict = expand_variables(job_dict)
job_obj = {'object_type': 'job', 'job_id': job_id}
job_obj['status'] = job_dict['STATUS']
time_fields = [
'VERIFIED',
'VERIFIED_TIMESTAMP',
'RECEIVED_TIMESTAMP',
'QUEUED_TIMESTAMP',
'SCHEDULE_TIMESTAMP',
'EXECUTING_TIMESTAMP',
'FINISHED_TIMESTAMP',
'FAILED_TIMESTAMP',
'CANCELED_TIMESTAMP',
]
for name in time_fields:
if job_dict.has_key(name):
# time objects cannot be marshalled, asctime if timestamp
try:
job_obj[name.lower()] = time.asctime(job_dict[name])
except Exception, exc:
# not a time object, just add
job_obj[name.lower()] = job_dict[name]
###########################################
# ARC job status retrieval on demand:
# But we should _not_ update the status in the mRSL files, since
# other MiG code might rely on finding only valid "MiG" states.
if configuration.arc_clusters and \
job_dict.get('UNIQUE_RESOURCE_NAME', 'unset') == 'ARC' \
and job_dict['STATUS'] == 'EXECUTING':
try:
home = os.path.join(configuration.user_home, client_dir)
arcsession = arc.Ui(home)
arcstatus = arcsession.jobStatus(job_dict['EXE'])
job_obj['status'] = arcstatus['status']
except arc.ARCWrapperError, err:
logger.error('Error retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
except arc.NoProxyError, err:
logger.error('While retrieving ARC job status: %s' % \
err.what())
job_obj['status'] += '(Error: ' + err.what() + ')'
if current_dir == upload_tmp_dir:
file_entry["deleteType"] = "POST"
file_entry["deleteUrl"] = del_url % \
(output_format, filename_field,
os.path.basename(rel_path),
files_field, "dummy")
uploaded.append(file_entry)
logger.info('status done: %s' % ' '.join([i[0] for i in upload_files]))
return (output_objects, status)
elif action == 'move':
# Move automatically takes place relative to upload tmp dir
for (rel_path, chunk_tuple) in upload_files:
real_path = os.path.abspath(os.path.join(base_dir, rel_path))
dest_path = os.path.abspath(os.path.join(
base_dir, current_dir, os.path.basename(rel_path)))
if not valid_user_path(dest_path, base_dir, True):
logger.error('%s tried to %s move to restricted path %s ! (%s)'
% (client_id, op_name, dest_path, current_dir))
output_objects.append(
{'object_type': 'error_text', 'text'
: "Invalid destination (%s expands to an illegal path)" \
% current_dir})
moved = False
else:
try:
move(real_path, dest_path)
moved = True
except Exception, exc:
logger.error('could not move %s to %s: %s' % (real_path,
dest_path, exc))
moved = False
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path_list = accepted['path']
size_list = [int(size) for size in accepted['size']]
title_entry = find_entry(output_objects, 'title')
title_entry[
'text'] = '%s Upload Progress Monitor' % configuration.short_title
if not configuration.site_enable_griddk:
output_objects.append({
'object_type':
'text',
'text':
'''Grid.dk features are disabled on this site.
Please contact the site admins %s if you think they should be enabled.
''' % configuration.admin_email
})
return (output_objects, returnvalues.OK)
refresh_secs = 5
meta = '<meta http-equiv="refresh" content="%s" />' % refresh_secs
title_entry['meta'] = meta
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
output_objects.append({'object_type': 'header', 'text': 'Upload progress'})
done_list = [False for _ in path_list]
progress_items = []
index = -1
for path in path_list:
index += 1
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, path))
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type': 'file_not_found',
'name': path
})
status = returnvalues.FILE_NOT_FOUND
continue
if not os.path.isfile(abs_path):
output_objects.append({
'object_type': 'error_text',
'text': "no such upload: %s" % path
})
status = returnvalues.CLIENT_ERROR
continue
relative_path = abs_path.replace(base_dir, '')
try:
logger.info('Checking size of upload %s' % abs_path)
cur_size = os.path.getsize(abs_path)
total_size = size_list[index]
percent = round(100.0 * cur_size / total_size, 1)
done_list[index] = (cur_size == total_size)
progress_items.append({
'object_type': 'progress',
'path': path,
'cur_size': cur_size,
'total_size': total_size,
'percent': percent,
'done': done_list[index],
'progress_type': 'upload'
})
except Exception, exc:
# Don't give away information about actual fs layout
output_objects.append({
'object_type':
'error_text',
'text':
'%s upload could not be checked! (%s)' %
(path, str(exc).replace(base_dir, ''))
})
status = returnvalues.SYSTEM_ERROR
continue
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
title_entry = find_entry(output_objects, 'title')
label = "%s" % configuration.site_vgrid_label
title_entry['text'] = "Add/Update %s Trigger" % label
output_objects.append({
'object_type': 'header',
'text': 'Add/Update %s Trigger' % label
})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
# NOTE: strip leftmost slashes from all fields used in file paths to avoid
# interference with os.path.join calls. Furthermore we strip and normalize
# the path variable first to make sure it does not point outside the vgrid.
# In practice any such directory traversal attempts will generally be moot
# since the grid_events daemon only starts a listener for each top-level
# vgrid and in there only reacts to events that match trigger rules from
# that particular vgrid. Thus only subvgrid access to parent vgrids might
# be a concern and still of limited consequence.
# NOTE: merge multi args into one string and split again to get flat array
rule_id = accepted['rule_id'][-1].strip()
vgrid_name = accepted['vgrid_name'][-1].strip().lstrip(os.sep)
path = os.path.normpath(accepted['path'][-1].strip()).lstrip(os.sep)
changes = [i.strip() for i in ' '.join(accepted['changes']).split()]
action = accepted['action'][-1].strip()
arguments = [
i.strip() for i in shlex.split(' '.join(accepted['arguments']))
]
rate_limit = accepted['rate_limit'][-1].strip()
settle_time = accepted['settle_time'][-1].strip()
match_files = accepted['match_files'][-1].strip() == 'True'
match_dirs = accepted['match_dirs'][-1].strip() == 'True'
match_recursive = accepted['match_recursive'][-1].strip() == 'True'
rank_str = accepted['rank'][-1]
try:
rank = int(rank_str)
except ValueError:
rank = None
logger.debug("addvgridtrigger with args: %s" % user_arguments_dict)
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# we just use a high res timestamp as automatic rule_id
if rule_id == keyword_auto:
rule_id = "%d" % (time.time() * 1E8)
if action == keyword_auto:
action = valid_trigger_actions[0]
if any_state in changes:
changes = valid_trigger_changes
logger.info("addvgridtrigger %s" % vgrid_name)
# Validity of user and vgrid names is checked in this init function so
# no need to worry about illegal directory traversal through variables
(ret_val, msg, ret_variables) = \
init_vgrid_script_add_rem(vgrid_name, client_id,
rule_id, 'trigger',
configuration)
if not ret_val:
output_objects.append({'object_type': 'error_text', 'text': msg})
return (output_objects, returnvalues.CLIENT_ERROR)
elif msg:
# In case of warnings, msg is non-empty while ret_val remains True
output_objects.append({'object_type': 'warning', 'text': msg})
# if we get here user is either vgrid owner or allowed to add rule
# don't add if already in vgrid or parent vgrid - but update if owner
update_id = None
if vgrid_is_trigger(vgrid_name, rule_id, configuration):
if vgrid_is_trigger_owner(vgrid_name, rule_id, client_id,
configuration):
update_id = 'rule_id'
else:
output_objects.append({
'object_type':
'error_text',
'text':
'%s is already a trigger owned by somebody else in the %s' %
(rule_id, label)
})
return (output_objects, returnvalues.CLIENT_ERROR)
# don't add if already in subvgrid
(list_status, subvgrids) = vgrid_list_subvgrids(vgrid_name, configuration)
if not list_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Error getting list of sub%ss: %s' % (label, subvgrids)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
for subvgrid in subvgrids:
if vgrid_is_trigger(subvgrid, rule_id, configuration, recursive=False):
output_objects.append({
'object_type': 'error_text',
'text': '''%(rule_id)s is already in a
sub-%(vgrid_label)s (%(subvgrid)s). Please remove the trigger from the
sub-%(vgrid_label)s and try again''' % {
'rule_id': rule_id,
'subvgrid': subvgrid,
'vgrid_label': label
}
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not action in valid_trigger_actions:
output_objects.append({
'object_type': 'error_text',
'text': "invalid action value %s" % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
if keyword_all in changes:
changes = valid_trigger_changes
for change in changes:
if not change in valid_trigger_changes:
output_objects.append({
'object_type':
'error_text',
'text':
"found invalid change value %s" % change
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Check if we should load saved trigger for rank change or update
rule_dict = None
if rank is not None or update_id is not None:
(load_status, all_triggers) = vgrid_triggers(vgrid_name, configuration)
if not load_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to load triggers for %s: %s' %
(vgrid_name, all_triggers)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
for saved_dict in all_triggers:
if saved_dict['rule_id'] == rule_id:
rule_dict = saved_dict
break
if rule_dict is None:
output_objects.append({
'object_type':
'error_text',
'text':
'No such trigger %s for %s: %s' %
(rule_id, vgrid_name, all_triggers)
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif not path:
# New trigger with missing path
output_objects.append({
'object_type': 'error_text',
'text': '''Either path or rank must
be set.'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif action == "submit" and not arguments:
# New submit trigger with missing mrsl arguments
output_objects.append({
'object_type':
'error_text',
'text':
'''Submit triggers must give
a job description file path as argument.'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Handle create and update (i.e. new, update all or just refresh mRSL)
if rank is None:
# IMPORTANT: we save the job template contents to avoid potential abuse
# Otherwise someone else in the VGrid could tamper with the template
# and make the next trigger execute arbitrary code on behalf of the
# rule owner.
templates = []
# Merge current and saved values
req_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': path,
'changes': changes,
'run_as': client_id,
'action': action,
'arguments': arguments,
'rate_limit': rate_limit,
'settle_time': settle_time,
'match_files': match_files,
'match_dirs': match_dirs,
'match_recursive': match_recursive,
'templates': templates
}
if rule_dict is None:
rule_dict = req_dict
else:
for field in user_arguments_dict:
if req_dict.has_key(field):
rule_dict[field] = req_dict[field]
# Now refresh template contents
if rule_dict['action'] == "submit":
for rel_path in rule_dict['arguments']:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, rel_path))
try:
if not valid_user_path(configuration, abs_path, base_dir,
True):
logger.warning(
'%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, rel_path))
raise ValueError('invalid submit path argument: %s' %
rel_path)
temp_fd = open(abs_path)
templates.append(temp_fd.read())
temp_fd.close()
except Exception, err:
logger.error("read submit argument file failed: %s" % err)
output_objects.append({
'object_type':
'error_text',
'text':
'failed to read submit argument file "%s"' % rel_path
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Save updated template contents here
rule_dict['templates'] = templates
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
action = accepted['action'][-1]
share_id = accepted['share_id'][-1]
path = accepted['path'][-1]
read_access = accepted['read_access'][-1].lower() in enabled_strings
write_access = accepted['write_access'][-1].lower() in enabled_strings
expire = accepted['expire'][-1]
# Merge and split invite to make sure 'a@b, c@d' entries are handled
invite_list = ','.join(accepted['invite']).split(',')
invite_list = [i for i in invite_list if i]
invite_msg = accepted['msg']
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Share Link'
# jquery support for tablesorter and confirmation on delete/redo:
# table initially sorted by 5, 4 reversed (active first and in growing age)
table_spec = {'table_id': 'sharelinkstable', 'sort_order': '[[5,1],[4,1]]'}
(add_import, add_init, add_ready) = man_base_js(configuration,
[table_spec],
{'width': 600})
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
output_objects.append({
'object_type': 'html_form',
'text': man_base_html(configuration)
})
header_entry = {'object_type': 'header', 'text': 'Manage share links'}
output_objects.append(header_entry)
if not configuration.site_enable_sharelinks:
output_objects.append({
'object_type':
'text',
'text':
'''
Share links are disabled on this site.
Please contact the site admins %s if you think they should be enabled.
''' % configuration.admin_email
})
return (output_objects, returnvalues.OK)
logger.info('sharelink %s from %s' % (action, client_id))
logger.debug('sharelink from %s: %s' % (client_id, accepted))
if not action in valid_actions:
output_objects.append({
'object_type':
'error_text',
'text':
'Invalid action "%s" (supported: %s)' %
(action, ', '.join(valid_actions))
})
return (output_objects, returnvalues.CLIENT_ERROR)
if action in post_actions:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
(load_status, share_map) = load_share_links(configuration, client_id)
if not load_status:
share_map = {}
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
target_op = 'sharelink'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
if action in get_actions:
if action == "show":
# Table columns to skip
skip_list = ['owner', 'single_file', 'expire']
sharelinks = []
for (saved_id, share_dict) in share_map.items():
share_item = build_sharelinkitem_object(
configuration, share_dict)
js_name = 'delete%s' % hexlify(saved_id)
helper = html_post_helper(
js_name, '%s.py' % target_op, {
'share_id': saved_id,
'action': 'delete',
csrf_field: csrf_token
})
output_objects.append({
'object_type': 'html_form',
'text': helper
})
share_item['delsharelink'] = {
'object_type':
'link',
'destination':
"javascript: confirmDialog(%s, '%s');" %
(js_name, 'Really remove %s?' % saved_id),
'class':
'removelink iconspace',
'title':
'Remove share link %s' % saved_id,
'text':
''
}
sharelinks.append(share_item)
# Display share links and form to add new ones
output_objects.append({
'object_type': 'sectionheader',
'text': 'Share Links'
})
output_objects.append({
'object_type': 'table_pager',
'entry_name': 'share links',
'default_entries': default_pager_entries
})
output_objects.append({
'object_type': 'sharelinks',
'sharelinks': sharelinks,
'skip_list': skip_list
})
output_objects.append({
'object_type': 'html_form',
'text': '<br/>'
})
output_objects.append({
'object_type': 'sectionheader',
'text': 'Create Share Link'
})
submit_button = '''<span>
<input type=submit value="Create share link" />
</span>'''
sharelink_html = create_share_link_form(configuration, client_id,
'html', submit_button,
csrf_token)
output_objects.append({
'object_type': 'html_form',
'text': sharelink_html
})
elif action == "edit":
header_entry['text'] = 'Edit Share Link'
share_dict = share_map.get(share_id, {})
if not share_dict:
output_objects.append({
'object_type':
'error_text',
'text':
'existing share link is required for edit'
})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'html_form',
'text':
'''
<p>
Here you can send invitations for your share link %(share_id)s to one or more
comma-separated recipients.
</p>
''' % share_dict
})
sharelinks = []
share_item = build_sharelinkitem_object(configuration, share_dict)
saved_id = share_item['share_id']
js_name = 'delete%s' % hexlify(saved_id)
helper = html_post_helper(js_name, '%s.py' % target_op, {
'share_id': saved_id,
'action': 'delete',
csrf_field: csrf_token
})
output_objects.append({'object_type': 'html_form', 'text': helper})
# Hide link to self
del share_item['editsharelink']
share_item['delsharelink'] = {
'object_type':
'link',
'destination':
"javascript: confirmDialog(%s, '%s');" %
(js_name, 'Really remove %s?' % saved_id),
'class':
'removelink iconspace',
'title':
'Remove share link %s' % saved_id,
'text':
''
}
sharelinks.append(share_item)
output_objects.append({
'object_type': 'sharelinks',
'sharelinks': sharelinks
})
submit_button = '''<span>
<input type=submit value="Send invitation(s)" />
</span>'''
sharelink_html = invite_share_link_form(configuration, client_id,
share_dict, 'html',
submit_button, csrf_token)
output_objects.append({
'object_type': 'html_form',
'text': sharelink_html
})
output_objects.append({
'object_type': 'link',
'destination': 'sharelink.py',
'text': 'Return to share link overview'
})
return (output_objects, returnvalues.OK)
elif action in post_actions:
share_dict = share_map.get(share_id, {})
if not share_dict and action != 'create':
logger.warning('%s tried to %s missing or not owned link %s!' %
(client_id, action, share_id))
output_objects.append({
'object_type':
'error_text',
'text':
'%s requires existing share link' % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
share_path = share_dict.get('path', path)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
rel_share_path = share_path.lstrip(os.sep)
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, rel_share_path))
relative_path = abs_path.replace(base_dir, '')
real_path = os.path.realpath(abs_path)
single_file = os.path.isfile(real_path)
vgrid_name = in_vgrid_share(configuration, abs_path)
if action == 'delete':
header_entry['text'] = 'Delete Share Link'
(save_status, _) = delete_share_link(share_id, client_id,
configuration, share_map)
if save_status and vgrid_name:
logger.debug("del vgrid sharelink pointer %s" % share_id)
(del_status,
del_msg) = vgrid_remove_sharelinks(configuration, vgrid_name,
[share_id], 'share_id')
if not del_status:
logger.error("del vgrid sharelink pointer %s failed: %s" %
(share_id, del_msg))
return (False, share_map)
desc = "delete"
elif action == "update":
header_entry['text'] = 'Update Share Link'
# Try to point replies to client_id email
client_email = extract_field(client_id, 'email')
if invite_list:
invites = share_dict.get('invites', []) + invite_list
invites_uniq = list(set([i for i in invites if i]))
invites_uniq.sort()
share_dict['invites'] = invites_uniq
auto_msg = invite_share_link_message(configuration, client_id,
share_dict, 'html')
msg = '\n'.join(invite_msg)
# Now send request to all targets in turn
threads = []
for target in invite_list:
job_dict = {
'NOTIFY': [target.strip()],
'JOB_ID': 'NOJOBID',
'USER_CERT': client_id,
'EMAIL_SENDER': client_email
}
logger.debug('invite %s to %s' % (target, share_id))
threads.append(
notify_user_thread(
job_dict,
[auto_msg, msg],
'INVITESHARE',
logger,
'',
configuration,
))
# Try finishing delivery but do not block forever on one message
notify_done = [False for _ in threads]
for _ in range(3):
for i in range(len(invite_list)):
if not notify_done[i]:
logger.debug('check done %s' % invite_list[i])
notify = threads[i]
notify.join(3)
notify_done[i] = not notify.isAlive()
notify_sent, notify_failed = [], []
for i in range(len(invite_list)):
if notify_done[i]:
notify_sent.append(invite_list[i])
else:
notify_failed.append(invite_list[i])
logger.debug('notify sent %s, failed %s' %
(notify_sent, notify_failed))
if notify_failed:
output_objects.append({
'object_type':
'html_form',
'text':
'''
<p>Failed to send invitation to %s</p>''' % ', '.join(notify_failed)
})
if notify_sent:
output_objects.append({
'object_type':
'html_form',
'text':
'''<p>Invitation sent to %s</p>
<textarea class="fillwidth padspace" rows="%d" readonly="readonly">
%s
%s
</textarea>
''' %
(', '.join(notify_sent),
(auto_msg + msg).count('\n') + 3, auto_msg, msg)
})
if expire:
share_dict['expire'] = expire
(save_status, _) = update_share_link(share_dict, client_id,
configuration, share_map)
desc = "update"
elif action == "create":
header_entry['text'] = 'Create Share Link'
if not read_access and not write_access:
output_objects.append({
'object_type':
'error_text',
'text':
'No access set - please select read, write or both'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: check path here as relative_path is empty for path='/'
if not path:
output_objects.append({
'object_type': 'error_text',
'text': 'No path provided!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# We refuse sharing of entire home for security reasons
elif not valid_user_path(
configuration, abs_path, base_dir, allow_equal=False):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, action, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
'''Illegal path "%s":
you can only share your own data, and not your entire home direcory.''' % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif not os.path.exists(abs_path):
output_objects.append({
'object_type':
'error_text',
'text':
'Provided path "%s" does not exist!' % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Refuse sharing of (mainly auth) dot dirs in root of user home
elif real_path.startswith(os.path.join(base_dir, '.')):
output_objects.append({
'object_type':
'error_text',
'text':
'Provided path "%s" cannot be shared for security reasons'
% path
})
return (output_objects, returnvalues.CLIENT_ERROR)
elif single_file and write_access:
output_objects.append({
'object_type':
'error_text',
'text':
'''Individual files
cannot be shared with write access - please share a directory with the file in
it or only share with read access.
'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# We check if abs_path is in vgrid share, but do not worry about
# private_base or public_base since they are only available to
# owners, who can always share anyway.
if vgrid_name is not None and \
not vgrid_is_owner(vgrid_name, client_id, configuration):
# share is inside vgrid share so we must check that user is
# permitted to create sharelinks there.
(load_status, settings_dict) = vgrid_settings(vgrid_name,
configuration,
recursive=True,
as_dict=True)
if not load_status:
# Probably owners just never saved settings, use defaults
settings_dict = {'vgrid_name': vgrid_name}
allowed = settings_dict.get('create_sharelink', keyword_owners)
if allowed != keyword_members:
output_objects.append({
'object_type': 'error_text',
'text': '''The settings
for the %(vgrid_name)s %(vgrid_label)s do not permit you to re-share
%(vgrid_label)s shared folders. Please contact the %(vgrid_name)s owners if you
think you should be allowed to do that.
''' % {
'vgrid_name': vgrid_name,
'vgrid_label': configuration.site_vgrid_label
}
})
return (output_objects, returnvalues.CLIENT_ERROR)
access_list = []
if read_access:
access_list.append('read')
if write_access:
access_list.append('write')
share_mode = '-'.join((access_list + ['only'])[:2])
# TODO: more validity checks here
if share_dict:
desc = "update"
else:
desc = "create"
# IMPORTANT: always use expanded path
share_dict.update({
'path': relative_path,
'access': access_list,
'expire': expire,
'invites': invite_list,
'single_file': single_file
})
attempts = 1
generate_share_id = False
if not share_id:
attempts = 3
generate_share_id = True
for i in range(attempts):
if generate_share_id:
share_id = generate_sharelink_id(configuration, share_mode)
share_dict['share_id'] = share_id
(save_status,
save_msg) = create_share_link(share_dict, client_id,
configuration, share_map)
if save_status:
logger.info('created sharelink: %s' % share_dict)
break
else:
# ID Collision?
logger.warning('could not create sharelink: %s' % save_msg)
if save_status and vgrid_name:
logger.debug("add vgrid sharelink pointer %s" % share_id)
(add_status,
add_msg) = vgrid_add_sharelinks(configuration, vgrid_name,
[share_dict])
if not add_status:
logger.error(
"save vgrid sharelink pointer %s failed: %s " %
(share_id, add_msg))
return (False, share_map)
else:
output_objects.append({
'object_type': 'error_text',
'text': 'No such action %s' % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not save_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Error in %s share link %s: ' % (desc, share_id) +
'save updated share links failed!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'text',
'text':
'%sd share link %s on %s .' %
(desc.title(), share_id, relative_path)
})
if action in ['create', 'update']:
sharelinks = []
share_item = build_sharelinkitem_object(configuration, share_dict)
saved_id = share_item['share_id']
js_name = 'delete%s' % hexlify(saved_id)
helper = html_post_helper(js_name, '%s.py' % target_op, {
'share_id': saved_id,
'action': 'delete',
csrf_field: csrf_token
})
output_objects.append({'object_type': 'html_form', 'text': helper})
share_item['delsharelink'] = {
'object_type':
'link',
'destination':
"javascript: confirmDialog(%s, '%s');" %
(js_name, 'Really remove %s?' % saved_id),
'class':
'removelink iconspace',
'title':
'Remove share link %s' % saved_id,
'text':
''
}
sharelinks.append(share_item)
output_objects.append({
'object_type': 'sharelinks',
'sharelinks': sharelinks
})
if action == 'create':
# NOTE: Leave editsharelink here for use in fileman overlay
#del share_item['editsharelink']
output_objects.append({
'object_type': 'html_form',
'text': '<br />'
})
submit_button = '''<span>
<input type=submit value="Send invitation(s)" />
</span>'''
invite_html = invite_share_link_form(configuration, client_id,
share_dict, 'html',
submit_button, csrf_token)
output_objects.append({
'object_type': 'html_form',
'text': invite_html
})
else:
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid share link action: %s' % action
})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({'object_type': 'html_form', 'text': '<br />'})
output_objects.append({
'object_type': 'link',
'destination': 'sharelink.py',
'text': 'Return to share link overview'
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
current_dir = accepted['current_dir']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text'
: '%s using flag: %s' % (op_name,
flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
# NB: Globbing disabled on purpose here
unfiltered_match = [base_dir + pattern]
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append(
{'object_type': 'error_text', 'text'
: "%s: cannot remove directory '%s': Permission denied"
% (op_name, pattern)})
status = returnvalues.CLIENT_ERROR
for real_path in match:
relative_path = real_path.replace(base_dir, '')
if verbose(flags):
output_objects.append({'object_type': 'file', 'name'
: relative_path})
if not os.path.exists(real_path):
output_objects.append({'object_type': 'file_not_found',
'name': relative_path})
continue
try:
if parents(flags):
# Please note that 'rmdir -p X' should give error if X
# doesn't exist contrary to 'mkdir -p X' not giving error
# if X exists.
os.removedirs(real_path)
else:
os.rmdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s failed on '%s'" % (op_name,
relative_path)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
# outside home, which is checked later.
path = os.path.abspath(path)
if not path.startswith(home_path):
logger.error("got path from %s outside user home: %s" %
(client_ip, raw_path))
print INVALID_MARKER
continue
real_path = os.path.realpath(path)
logger.debug("check path %s in home %s or chroot" %
(path, home_path))
# Exact match to user home does not make sense as we expect a file
# IMPORTANT: use path and not real_path here in order to test both
if not valid_user_path(configuration,
path,
home_path,
allow_equal=False,
apache_scripts=True):
logger.error("path from %s outside user chroot %s: %s (%s)" %
(client_ip, home_path, raw_path, real_path))
print INVALID_MARKER
continue
logger.info("found valid user chroot path from %s: %s" %
(client_ip, real_path))
print real_path
# Throttle down a bit to yield
time.sleep(0.01)
except KeyboardInterrupt:
keep_running = False
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
lines = int(accepted['lines'][-1])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
# We search for the last 'lines' lines by beginning from the end of
# the file and exponetially backtracking until the backtrack
# contains at least 'lines' lines or we're back to the beginning of
# the file.
# At that point we skip any extra lines before printing.
try:
filedes = open(abs_path, 'r')
# Go to end of file and backtrack
backstep = 1
newlines = 0
filedes.seek(0, 2)
length = filedes.tell()
while backstep < length and newlines <= lines:
backstep *= 2
if backstep > length:
backstep = length
filedes.seek(-backstep, 2)
newlines = len(filedes.readlines())
if length:
# Go back after reading to end of file
filedes.seek(-backstep, 2)
# Skip any extra lines caused by the exponential backtracking.
# We could probably speed up convergence with binary search...
while newlines > lines:
dummy = filedes.readline()
newlines -= 1
backstep = length - filedes.tell()
# Now we're at the wanted spot - print rest of file
for _ in range(newlines):
output_lines.append(filedes.readline())
filedes.close()
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {
'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']
}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
show_dest = accepted['with_dest'][0].lower() == 'true'
listing = []
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
settings_dict = load_settings(client_id, configuration)
javascript = '%s\n%s' % (select_all_javascript(),
selected_file_actions_javascript())
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s Files' % configuration.short_title
title_entry['javascript'] = javascript
output_objects.append({'object_type': 'header',
'text': '%s Files' % configuration.short_title
})
location_pre_html = \
"""
<div class='files'>
<table class='files'>
<tr class='title'><td class='centertext'>
Working directory:
</td></tr>
<tr><td class='centertext'>
"""
output_objects.append({'object_type': 'html_form', 'text'
: location_pre_html})
for pattern in pattern_list:
links = []
links.append({'object_type': 'link', 'text':
'%s HOME' % configuration.short_title,
'destination': 'ls.py?path=.'})
prefix = ''
parts = pattern.split(os.sep)
for i in parts:
prefix = os.path.join(prefix, i)
links.append({'object_type': 'link', 'text': i,
'destination': 'ls.py?path=%s' % prefix})
output_objects.append({'object_type': 'multilinkline', 'links'
: links})
location_post_html = """
</td></tr>
</table>
</div>
<br />
"""
output_objects.append({'object_type': 'html_form', 'text'
: location_post_html})
more_html = \
"""
<div class='files'>
<form method='post' name='fileform' onSubmit='return selectedFilesAction();'>
<table class='files'>
<tr class='title'><td class='centertext' colspan=2>
Advanced file actions
</td></tr>
<tr><td>
Action on paths selected below
(please hold mouse cursor over button for a description):
</td>
<td class='centertext'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='v' />
<input type='submit' title='Show concatenated contents (cat)' onClick='document.pressed=this.value' value='cat' />
<input type='submit' onClick='document.pressed=this.value' value='head' title='Show first lines (head)' />
<input type='submit' onClick='document.pressed=this.value' value='tail' title='Show last lines (tail)' />
<input type='submit' onClick='document.pressed=this.value' value='wc' title='Count lines/words/chars (wc)' />
<input type='submit' onClick='document.pressed=this.value' value='stat' title='Show details (stat)' />
<input type='submit' onClick='document.pressed=this.value' value='touch' title='Update timestamp (touch)' />
<input type='submit' onClick='document.pressed=this.value' value='truncate' title='truncate! (truncate)' />
<input type='submit' onClick='document.pressed=this.value' value='rm' title='delete! (rm)' />
<input type='submit' onClick='document.pressed=this.value' value='rmdir' title='Remove directory (rmdir)' />
<input type='submit' onClick='document.pressed=this.value' value='submit' title='Submit file (submit)' />
</td></tr>
</table>
</form>
</div>
"""
output_objects.append({'object_type': 'html_form', 'text'
: more_html})
dir_listings = []
output_objects.append({
'object_type': 'dir_listings',
'dir_listings': dir_listings,
'flags': flags,
'show_dest': show_dest,
})
first_match = None
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
real_path = os.path.abspath(server_path)
if not valid_user_path(real_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s! (%s)'
% (client_id, op_name, real_path, pattern))
continue
match.append(real_path)
if not first_match:
first_match = real_path
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for real_path in match:
if real_path + os.sep == base_dir:
relative_path = '.'
else:
relative_path = real_path.replace(base_dir, '')
entries = []
dir_listing = {
'object_type': 'dir_listing',
'relative_path': relative_path,
'entries': entries,
'flags': flags,
}
dest = ''
if show_dest:
if os.path.isfile(real_path):
dest = os.path.basename(real_path)
elif recursive(flags):
# references to '.' or similar are stripped by abspath
if real_path + os.sep == base_dir:
dest = ''
else:
# dest = os.path.dirname(real_path).replace(base_dir, "")
dest = os.path.basename(real_path) + os.sep
handle_expand(
output_objects,
entries,
base_dir,
real_path,
flags,
dest,
0,
show_dest,
)
dir_listings.append(dir_listing)
output_objects.append({'object_type': 'html_form', 'text'
: """
<div class='files'>
<table class='files'>
<tr class='title'><td class='centertext'>
Filter paths (wildcards like * and ? are allowed)
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />
<input type='text' name='path' value='' />
<input type='submit' value='Filter' />
</form>
</td></tr>
</table>
</div>
"""
% flags})
# Short/long format buttons
htmlform = \
"""<table class='files'>
<tr class='title'><td class='centertext' colspan=4>
File view options
</td></tr>
<tr><td colspan=4><br /></td></tr>
<tr class='title'><td>Parameter</td><td>Setting</td><td>Enable</td><td>Disable</td></tr>
<tr><td>Long format</td><td>
%s</td><td>"""\
% long_list(flags)\
+ """
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% (flags + 'l')
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% flags.replace('l', '')
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
# Recursive output
htmlform += \
"""
<!-- Non-/recursive list buttons -->
<tr><td>Recursion</td><td>
%s</td><td>"""\
% recursive(flags)
htmlform += \
"""
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% (flags + 'r')
for entry in pattern_list:
htmlform += " <input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% flags.replace('r', '')
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='Off' /><br />
</form>
</td></tr>
"""
htmlform += \
"""
<!-- Show dot files buttons -->
<tr><td>Show hidden files</td><td>
%s</td><td>"""\
% all(flags)
htmlform += \
"""
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% (flags + 'a')
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='On' /><br />
</form>
</td><td>
<form method='post' action='ls.py'>
<input type='hidden' name='output_format' value='html' />
<input type='hidden' name='flags' value='%s' />"""\
% flags.replace('a', '')
for entry in pattern_list:
htmlform += "<input type='hidden' name='path' value='%s' />"\
% entry
htmlform += \
"""
<input type='submit' value='Off' /><br />
</form>
</td></tr>
</table>
"""
# show flag buttons after contents to avoid
output_objects.append({'object_type': 'html_form', 'text'
: htmlform})
# create upload file form
if first_match:
# use first match for current directory
# Note that base_dir contains an ending slash
if os.path.isdir(first_match):
dir_path = first_match
else:
dir_path = os.path.dirname(first_match)
if dir_path + os.sep == base_dir:
relative_dir = '.'
else:
relative_dir = dir_path.replace(base_dir, '')
output_objects.append({'object_type': 'html_form', 'text'
: """
<br />
<table class='files'>
<tr class='title'><td class='centertext' colspan=2>
Edit file
</td><td><br /></td></tr>
<tr><td>
Fill in the path of a file to edit and press 'edit' to open that file in the<br />
online file editor. Alternatively a file can be selected for editing through<br />
the listing of personal files.
</td><td colspan=2 class='righttext'>
<form name='editor' method='post' action='editor.py'>
<input type='hidden' name='output_format' value='html' />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input type='text' name='path' size=50 value='' />
<input type='submit' value='edit' />
</form>
</td></tr>
</table>
<br />
<table class='files'>
<tr class='title'><td class='centertext' colspan=4>
Create directory
</td></tr>
<tr><td>
Name of new directory to be created in current directory (%(dest_dir)s)
</td><td class='righttext' colspan=3>
<form action='mkdir.py' method=post>
<input name='path' size=50 />
<input name='current_dir' type='hidden' value='%(dest_dir)s' />
<input type='submit' value='Create' name='mkdirbutton' />
</form>
</td></tr>
</table>
<br />
<form enctype='multipart/form-data' action='textarea.py' method='post'>
<table class='files'>
<tr class='title'><td class='centertext' colspan=4>
Upload file
</td></tr>
<tr><td colspan=4>
Upload file to current directory (%(dest_dir)s)
</td></tr>
<tr><td colspan=2>
Extract package files (.zip, .tar.gz, .tar.bz2)
</td><td colspan=2>
<input type='checkbox' name='extract_0' />
</td></tr>
<tr><td colspan=2>
Submit mRSL files (also .mRSL files included in packages)
</td><td colspan=2>
<input type='checkbox' name='submitmrsl_0' checked />
</td></tr>
<tr><td>
File to upload
</td><td class='righttext' colspan=3>
<input name='fileupload_0_0_0' type='file' />
</td></tr>
<tr><td>
Optional remote filename (extra useful in windows)
</td><td class='righttext' colspan=3>
<input name='default_remotefilename_0' type='hidden' value='%(dest_dir)s' />
<input name='remotefilename_0' type='text' size='50' value='%(dest_dir)s' />
<input type='submit' value='Upload' name='sendfile' />
</td></tr>
</table>
</form>
"""
% {'dest_dir': relative_dir + os.sep}})
return (output_objects, status)
