def __cron_log(configuration, client_id, msg, level="info"):
"""Wrapper to send a single msg to user cron log file"""
client_dir = client_id_dir(client_id)
log_dir_path = os.path.join(configuration.user_home, client_dir,
cron_output_dir)
log_path = os.path.join(log_dir_path, cron_log_name)
if not os.path.exists(log_dir_path):
try:
os.makedirs(log_dir_path)
except:
pass
cron_logger = logging.getLogger('cron')
cron_logger.setLevel(logging.INFO)
handler = logging.handlers.RotatingFileHandler(
log_path, maxBytes=cron_log_size, backupCount=cron_log_cnt - 1)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler.setFormatter(formatter)
cron_logger.addHandler(handler)
if level == 'error':
cron_logger.error(msg)
elif level == 'warning':
cron_logger.warning(msg)
else:
cron_logger.info(msg)
handler.flush()
handler.close()
cron_logger.removeHandler(handler)
def remove2fa(configuration, user_id, verbose, force=False):
"""Removes two factor secrets and settings for client_id
NOTE: Active sessions remain until logout"""
_logger = configuration.logger
if verbose:
print("Removing 2FA setup for %r" % user_id)
allow_missing = False
if force:
allow_missing = True
client_dir = client_id_dir(user_id)
settings_dir = os.path.join(configuration.user_settings, client_dir)
key_path = os.path.join(settings_dir, twofactor_key_name)
if verbose:
print("Removing key file: %s" % key_path)
status = delete_file(key_path, _logger, allow_missing=allow_missing)
if not status and not force:
return status
interval_path = os.path.join(settings_dir, twofactor_interval_name)
if verbose:
print("Removing interval file: %s" % interval_path)
status = delete_file(interval_path, _logger, allow_missing=True)
if not status and not force:
return status
twofactor_settings_path = os.path.join(settings_dir, twofactor_filename)
if verbose:
print("Removing twofactor file: %s" % twofactor_settings_path)
status = delete_file(twofactor_settings_path,
_logger,
allow_missing=allow_missing)
return status
def manage_pending_peers(configuration, client_id, action, change_list):
"""Helper to manage changes to pending peers list of client_id"""
_logger = configuration.logger
client_dir = client_id_dir(client_id)
pending_peers_path = os.path.join(configuration.user_settings, client_dir,
pending_peers_filename)
try:
pending_peers = load(pending_peers_path)
except Exception as exc:
if os.path.exists(pending_peers_path):
_logger.warning("could not load pending peers from %s: %s" %
(pending_peers_path, exc))
pending_peers = []
change_dict = dict(change_list)
# NOTE: always remove old first to replace any existing and move them last
pending_peers = [(i, j)
for (i, j) in pending_peers if not i in change_dict]
if action == "add":
pending_peers += change_list
elif action == "remove":
pass
else:
_logger.error(
"unsupported action in manage pending peers: %s" % action)
return False
try:
dump(pending_peers, pending_peers_path)
return True
except Exception as exc:
_logger.warning("could not save pending peers to %s: %s" %
(pending_peers_path, exc))
return False
def check_account_accessible(configuration,
username,
proto,
environ=None,
io_login=True,
expand_alias=True):
"""Check username account status and expire field in cache and user DB
if needed and return a boolean to tell if account is accessible or not.
The proto argument is used to detect if only users are allowed or if e.g.
sharelinks, jobs or jupyter mounts should also be checked.
The optional environ overrides the environment dict which is otherwise
taken from os.environ and the io_login is a boolean to decide if
configuration.site_io_account_expire should be honored or if expire should
be enforced only if configuration.user_openid_enforce_expire is set. The
optional expand_alias can be used to force expansion of username from an
alias to the full DN so that e.g. the user email address alias can be
provided and automatically looked up. This is particularly convenient when
called from PAM for SFTP.
"""
_logger = configuration.logger
if not environ:
environ = os.environ
# We might end up here from IO daemon logins with user, sharelink, job and
# jupyter mounts. We should let all but actual user logins pass for now.
# TODO: consider checking underlying user for other types eventually?
if detect_special_login(configuration, username, proto):
_logger.debug("found %s as special %s login" % (username, proto))
return True
# NOTE: now we know username must be an ordinary user to check
client_id = username
if configuration.site_enable_gdp:
client_id = get_base_client_id(configuration,
client_id,
expand_oid_alias=expand_alias)
elif expand_alias:
# Use client_id_dir to make it work even if already expanded
home_dir = os.path.join(configuration.user_home,
client_id_dir(client_id))
real_home = os.path.realpath(home_dir)
real_id = os.path.basename(real_home)
client_id = client_dir_id(real_id)
(account_accessible, account_status,
_) = check_account_status(configuration, client_id)
if not account_accessible:
_logger.debug("%s account %s" % (client_id, account_status))
return False
if io_login and not configuration.site_io_account_expire:
_logger.debug("%s account active and no IO expire" % client_id)
return True
if not io_login and not configuration.user_openid_enforce_expire:
_logger.debug("%s account active and no OpenID expire" % client_id)
return True
(pending_expire, account_expire,
_) = check_account_expire(configuration, client_id, environ)
return pending_expire
def get_account_expire_cache(configuration, client_id):
"""Check if account with client_id has an expire mark in the cache and
if so return the timestamp associated with it.
"""
_logger = configuration.logger
if not client_id:
_logger.error("invalid client ID: %s" % client_id)
return False
client_dir = client_id_dir(client_id)
base_dir = os.path.join(configuration.mig_system_run, expire_marks_dir)
return get_filemark(configuration, base_dir, client_dir)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
status = returnvalues.OK
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Job Manager'
user_settings = title_entry.get('user_settings', {})
title_entry['style'] = css_tmpl(configuration, user_settings)
csrf_map = {}
method = 'post'
limit = get_csrf_limit(configuration)
for target_op in csrf_backends:
csrf_map[target_op] = make_csrf_token(configuration, method, target_op,
client_id, limit)
(add_import, add_init, add_ready) = js_tmpl_parts(csrf_map)
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
output_objects.append({'object_type': 'header', 'text': 'Job Manager'})
output_objects.append({
'object_type': 'table_pager',
'entry_name': 'jobs',
'default_entries': default_pager_entries,
'form_append': pager_append()
})
output_objects.append({'object_type': 'html_form', 'text': html_post()})
return (output_objects, status)
def migrate_job(config, job, peer):
protocol = 'https'
port = ''
server = peer['fqdn']
# Remove schedule hint from job before migration
del job['SCHEDULE_HINT']
# Make sure legacy jobs don't fail
if 'MIGRATE_COUNT' not in job:
job['MIGRATE_COUNT'] = str(0)
# Add or increment migration counter
migrate_count = int(job['MIGRATE_COUNT']) + 1
job['MIGRATE_COUNT'] = str(migrate_count)
# TODO: only upload if job is not already replicated at
# remote server
# TMP!
steal_job = False
if not steal_job:
# upload pickled job to server
client_dir = client_id_dir(job['USER_CERT'])
mrsl_filename = config.mrsl_files_dir + client_dir + '/'\
+ job['JOB_ID'] + '.mRSL'
result = pickle(job, mrsl_filename, config.logger)
if not result:
config.logger.error('Aborting migration of job %s (%s)',
job['JOB_ID'], result)
return False
dest = mrsl_filename
# TMP!
# upload_reply = put_data(config, mrsl_filename, protocol, server, port, dest)
config.logger.warning(
'Actual migration disabled until fully supported')
upload_reply = (-1, 'Actual migration disabled until fully supported')
if upload_reply[0] != http_success:
return False
# migration_msg = ""
# migration_reply = put_data(config, protocol, server, port, migration_msg)
return True
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False, op_title=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
job_id_list = accepted['job_id']
external_dict = mrslkeywords.get_keywords_dict(configuration)
if not configuration.site_enable_jobs:
output_objects.append({'object_type': 'error_text', 'text':
'''Job execution is not enabled on this system'''})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
for job_id in job_id_list:
# job = Job()
filepath = os.path.join(base_dir, job_id)
filepath += '.mRSL'
(new_job_obj_status, new_job_obj) = \
create_job_object_from_pickled_mrsl(filepath, logger,
external_dict)
if not new_job_obj_status:
output_objects.append({'object_type': 'error_text', 'text'
: new_job_obj})
status = returnvalues.CLIENT_ERROR
else:
# return new_job_obj
output_objects.append({'object_type': 'jobobj', 'jobobj'
: new_job_obj})
return (output_objects, status)
def handle_proxy(proxy_string, client_id, config):
"""If ARC-enabled server: store a proxy certificate.
Arguments: proxy_string - text extracted from given upload
client_id - DN for user just being created
config - global configuration
"""
output = []
client_dir = client_id_dir(client_id)
proxy_dir = os.path.join(config.user_home, client_dir)
proxy_path = os.path.join(config.user_home, client_dir,
arcwrapper.Ui.proxy_name)
if not config.arc_clusters:
output.append({'object_type': 'error_text',
'text': 'No ARC support!'})
return output
# store the file
try:
write_file(proxy_string, proxy_path, config.logger)
os.chmod(proxy_path, 0o600)
except Exception as exc:
output.append({'object_type': 'error_text',
'text': 'Proxy file could not be written (%s)!'
% str(exc).replace(proxy_dir, '')})
return output
# provide information about the uploaded proxy
try:
session_ui = arcwrapper.Ui(proxy_dir)
proxy = session_ui.getProxy()
if proxy.IsExpired():
# can rarely happen, constructor will throw exception
output.append({'object_type': 'warning',
'text': 'Proxy certificate is expired.'})
else:
output.append({'object_type': 'text', 'text': 'Proxy for %s'
% proxy.GetIdentitySN()})
output.append({'object_type': 'text',
'text': 'Proxy certificate will expire on %s (in %s sec.)'
% (proxy.Expires(), proxy.getTimeleft())})
except arcwrapper.NoProxyError as err:
output.append({'object_type': 'warning',
'text': 'No proxy certificate to load: %s'
% err.what()})
return output
def get_accepted_peers(configuration, client_id):
"""Helper to get the list of peers accepted by client_id"""
_logger = configuration.logger
client_dir = client_id_dir(client_id)
peers_path = os.path.join(configuration.user_settings, client_dir,
peers_filename)
try:
accepted_peers = load(peers_path)
except Exception as exc:
if os.path.exists(peers_path):
_logger.warning("could not load peers from %s: %s" %
(peers_path, exc))
accepted_peers = {}
return accepted_peers
def load_crontab(client_id, configuration, allow_missing=True):
"""Load entries from plain user crontab file"""
_logger = configuration.logger
client_dir = client_id_dir(client_id)
crontab_path = os.path.join(configuration.user_settings, client_dir,
crontab_name)
try:
crontab_fd = open(crontab_path, "rb")
crontab_contents = crontab_fd.read()
crontab_fd.close()
except Exception as exc:
if not allow_missing:
_logger.error('failed reading %s crontab file: %s' % (client_id,
exc))
crontab_contents = ''
return crontab_contents
def load_atjobs(client_id, configuration, allow_missing=True):
"""Load entries from plain user atjobs file"""
_logger = configuration.logger
client_dir = client_id_dir(client_id)
atjobs_path = os.path.join(configuration.user_settings, client_dir,
atjobs_name)
try:
atjobs_fd = open(atjobs_path, "rb")
atjobs_contents = atjobs_fd.read()
atjobs_fd.close()
except Exception as exc:
if not allow_missing:
_logger.error('failed reading %s atjobs file: %s' % (client_id,
exc))
atjobs_contents = ''
return atjobs_contents
def parse_and_save_atjobs(atjobs, client_id, configuration):
"""Validate and write the atjobs for client_id"""
client_dir = client_id_dir(client_id)
atjobs_path = os.path.join(configuration.user_settings, client_dir,
atjobs_name)
status, msg = True, ''
atjobs_entries = parse_atjobs_contents(configuration, client_id,
atjobs.splitlines())
try:
atjobs_fd = open(atjobs_path, "wb")
# TODO: filter out broken lines before write?
atjobs_fd.write(atjobs)
atjobs_fd.close()
msg = "Found and saved %d valid atjobs entries" % len(atjobs_entries)
except Exception as exc:
status = False
msg = 'ERROR: writing %s atjobs file: %s' % (client_id, exc)
return (status, msg)
def parse_and_save_crontab(crontab, client_id, configuration):
"""Validate and write the crontab for client_id"""
client_dir = client_id_dir(client_id)
crontab_path = os.path.join(configuration.user_settings, client_dir,
crontab_name)
status, msg = True, ''
crontab_entries = parse_crontab_contents(configuration, client_id,
crontab.splitlines())
try:
crontab_fd = open(crontab_path, "wb")
# TODO: filter out broken lines before write?
crontab_fd.write(crontab)
crontab_fd.close()
msg = "Found and saved %d valid crontab entries" % len(crontab_entries)
except Exception as exc:
status = False
msg = 'ERROR: writing %s crontab file: %s' % (client_id, exc)
return (status, msg)
def get_account_status_cache(configuration, client_id):
"""Check if account with client_id has an status mark in the cache and
if so return the timestamp associated with it.
"""
_logger = configuration.logger
if not client_id:
_logger.error("invalid client ID: %s" % client_id)
return False
client_dir = client_id_dir(client_id)
base_dir = os.path.join(configuration.mig_system_run, status_marks_dir)
# NOTE: translate status integer encoded as timestamp back to string
status_index = get_filemark(configuration, base_dir, client_dir)
if status_index is None:
return status_index
if status_index < 0 or status_index >= len(valid_account_status):
_logger.error("invalid cached client status for %s: %s" %
(client_id, status_index))
return None
return valid_account_status[int(status_index)]
def reset_twofactor_key(client_id, configuration, seed=None, interval=None):
"""Reset 2FA secret key and write to user settings file in scrambled form.
Return the new secret key on unscrambled base32 form.
"""
_logger = configuration.logger
if configuration.site_enable_gdp:
client_id = get_base_client_id(configuration,
client_id,
expand_oid_alias=False)
client_dir = client_id_dir(client_id)
key_path = os.path.join(configuration.user_settings, client_dir,
twofactor_key_name)
try:
if pyotp is None:
raise Exception("The pyotp module is missing and required for 2FA")
if not seed:
b32_key = pyotp.random_base32(length=twofactor_key_bytes)
else:
b32_key = seed
# NOTE: pyotp.random_base32 returns unicode
# which causes trouble with WSGI
b32_key = force_utf8(b32_key)
scrambled = scramble_password(configuration.site_password_salt,
b32_key)
key_fd = open(key_path, 'w')
key_fd.write(scrambled)
key_fd.close()
# Reset interval
interval_path = os.path.join(configuration.user_settings, client_dir,
twofactor_interval_name)
delete_file(interval_path, _logger, allow_missing=True)
if interval:
i_fd = open(interval_path, 'w')
i_fd.write("%d" % interval)
i_fd.close()
except Exception as exc:
_logger.error("failed in reset 2FA key: %s" % exc)
return False
return b32_key
def read_cron_log(configuration, client_id, flags):
"""Read-in saved cron logs for crontab. We read in all rotated logs"""
client_dir = client_id_dir(client_id)
log_content = ''
for i in xrange(cron_log_cnt - 1, -1, -1):
log_name = cron_log_name
if i > 0:
log_name += '.%d' % i
log_path = os.path.join(configuration.user_home, client_dir,
cron_output_dir, log_name)
configuration.logger.debug('read from %s' % log_path)
try:
log_fd = open(log_path)
log_content += log_fd.read()
configuration.logger.debug('read in log lines:\n%s' % log_content)
log_fd.close()
except IOError:
pass
return log_content
def load_twofactor_interval(client_id, configuration):
"""Load 2FA token interval"""
_logger = configuration.logger
if configuration.site_enable_gdp:
client_id = get_base_client_id(configuration,
client_id,
expand_oid_alias=False)
client_dir = client_id_dir(client_id)
interval_path = os.path.join(configuration.user_settings, client_dir,
twofactor_interval_name)
result = None
if os.path.isfile(interval_path):
i_fd = open(interval_path)
interval = i_fd.read().strip()
i_fd.close()
try:
result = int(interval)
except Exception as exc:
result = None
_logger.error("Failed to read twofactor interval: %s" % exc)
return result
def unpack_archive(
configuration,
client_id,
src,
dst,
):
"""Inside the user home of client_id: unpack the src zip or tar
archive into the dst dir. Both src and dst are expected to be relative
paths.
Please note that src and dst should be checked for illegal directory
traversal attempts *before* getting here.
"""
client_dir = client_id_dir(client_id)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
real_src = os.path.join(base_dir, src.lstrip(os.sep))
return handle_package_upload(real_src, src, client_id,
configuration, False, dst)
def update_account_expire_cache(configuration, user_dict):
"""Create or update expire mark for account with given user_dict if it
contains the expire field.
"""
_logger = configuration.logger
if not isinstance(user_dict, dict):
_logger.error("invalid user_dict: %s" % user_dict)
return False
client_id = user_dict.get('distinguished_name', None)
if not client_id:
_logger.error("no client ID set for user: %s" % user_dict)
return False
expire = user_dict.get('expire', None)
if not expire:
_logger.info("no expire set for user: %s" % user_dict)
return True
elif isinstance(expire, basestring):
_logger.warning("found string expire value for user: %s" % user_dict)
return False
client_dir = client_id_dir(client_id)
base_dir = os.path.join(configuration.mig_system_run, expire_marks_dir)
return update_filemark(configuration, base_dir, client_dir, expire)
def load_twofactor_key(client_id, configuration, allow_missing=True):
"""Load 2FA secret key on scrambled form from user settings file and
return the unscrambled form.
"""
_logger = configuration.logger
if configuration.site_enable_gdp:
client_id = get_base_client_id(configuration,
client_id,
expand_oid_alias=False)
client_dir = client_id_dir(client_id)
key_path = os.path.join(configuration.user_settings, client_dir,
twofactor_key_name)
b32_key = None
try:
pw_fd = open(key_path)
scrambled = pw_fd.read().strip()
pw_fd.close()
b32_key = unscramble_password(configuration.site_password_salt,
scrambled)
except Exception as exc:
if not allow_missing:
_logger.error("load 2FA key failed: %s" % exc)
return b32_key
def update_account_status_cache(configuration, user_dict):
"""Create or update status mark for account with given user_dict if it
contains the status field.
"""
_logger = configuration.logger
if not isinstance(user_dict, dict):
_logger.error("invalid user_dict: %s" % user_dict)
return False
client_id = user_dict.get('distinguished_name', None)
if not client_id:
_logger.error("no client ID set for user: %s" % user_dict)
return False
# NOTE: translate status strings to integers here to encode as timestamp
status_key = user_dict.get('status', None)
if not status_key:
_logger.info("no status set for user: %s" % user_dict)
return True
if not status_key in valid_account_status:
_logger.error("invalid account status for user: %s" % user_dict)
return False
status = valid_account_status.index(status_key)
client_dir = client_id_dir(client_id)
base_dir = os.path.join(configuration.mig_system_run, status_marks_dir)
return update_filemark(configuration, base_dir, client_dir, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
patterns = accepted['path']
search = accepted['pattern'][-1]
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
for pattern in patterns:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
try:
matching = pattern_match_file(search, abs_path)
for line in matching:
output_lines.append(line)
except Exception as exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path, exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output',
'lines': output_lines,
'wrap_binary': binary(flags),
'wrap_targets': ['lines']}
if verbose(flags):
entry['path'] = relative_path
output_objects.append(entry)
return (output_objects, status)
def create_arc_job(
job,
configuration,
logger,
):
"""Analog to create_job_script for ARC jobs:
Creates symLinks for receiving result files, translates job dict to ARC
xrsl, and stores resulting job script (xrsl + sh script) for submitting.
We do _not_ create a separate job_dict with copies and SESSIONID inside,
as opposed to create_job_script, all we need is the link from
webserver_home / sessionID into the user's home directory
("job_output/job['JOB_ID']" is added to the result upload URLs in the
translation).
Returns message (ARC job ID if no error) and sessionid (None if error)
"""
if not configuration.arc_clusters:
return (None, 'No ARC support!')
if not job['JOBTYPE'] == 'arc':
return (None, 'Error. This is not an ARC job')
# Deep copy job for local changes
job_dict = deepcopy(job)
# Finally expand reserved job variables like +JOBID+ and +JOBNAME+
job_dict = expand_variables(job_dict)
# ... no more changes to job_dict from here on
client_id = str(job_dict['USER_CERT'])
# we do not want to see empty jobs here. Test as done in create_job_script.
if client_id == configuration.empty_job_name:
return (None, 'Error. empty job for ARC?')
# generate random session ID:
sessionid = hexlify(open('/dev/urandom').read(session_id_bytes))
logger.debug('session ID (for creating links): %s' % sessionid)
client_dir = client_id_dir(client_id)
# make symbolic links inside webserver_home:
#
# we need: link to owner's dir. to receive results,
# job mRSL inside sessid_to_mrsl_link_home
linklist = [(configuration.user_home + client_dir,
configuration.webserver_home + sessionid),
(configuration.mrsl_files_dir + client_dir + '/' +
str(job_dict['JOB_ID']) + '.mRSL',
configuration.sessid_to_mrsl_link_home + sessionid + '.mRSL')]
for (dest, loc) in linklist:
make_symlink(dest, loc, logger)
# the translation generates an xRSL object which specifies to execute
# a shell script with script_name. If sessionid != None, results will
# be uploaded to sid_redirect/sessionid/job_output/job_id
try:
(xrsl, script, script_name) = mrsltoxrsl.translate(job_dict, sessionid)
logger.debug('translated to xRSL: %s' % xrsl)
logger.debug('script:\n %s' % script)
except Exception as err:
# error during translation, pass a message
logger.error('Error during xRSL translation: %s' % err.__str__())
return (None, err.__str__())
# we submit directly from here (the other version above does
# copyFileToResource and gen_job_script generates all files)
# we have to put the generated script somewhere..., and submit from there.
# inputfiles are given by the user as relative paths from his home,
# so we should use that location (and clean up afterwards).
# write script (to user home)
user_home = os.path.join(configuration.user_home, client_dir)
script_path = os.path.abspath(os.path.join(user_home, script_name))
write_file(script, script_path, logger)
os.chdir(user_home)
try:
logger.debug('submitting job to ARC')
session = arcwrapper.Ui(user_home)
arc_job_ids = session.submit(xrsl)
# if no exception occurred, we are done:
job_dict['ARCID'] = arc_job_ids[0]
job_dict['SESSIONID'] = sessionid
msg = 'OK'
result = job_dict
# when errors occurred, pass a message to the caller.
except arcwrapper.ARCWrapperError as err:
msg = err.what()
result = None # unsuccessful
except arcwrapper.NoProxyError as err:
msg = 'No Proxy found: %s' % err.what()
result = None # unsuccessful
except Exception as err:
msg = err.__str__()
result = None # unsuccessful
# always remove the generated script
os.remove(script_name)
# and remove the created links immediately if failed
if not result:
for (_, link) in linklist:
os.remove(link)
logger.error('Unsuccessful ARC job submission: %s' % msg)
else:
logger.debug('submitted to ARC as job %s' % msg)
return (result, msg)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
status = returnvalues.OK
filelist = []
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to
# avoid leaking information about file system layout while
# allowing consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match^I
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# job feasibility is hard on the server, limit
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
checkcondjobs = []
for filepath in filelist:
# Extract job_id from filepath (replace doesn't modify filepath)
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
checkcondjob = {'object_type': 'checkcondjob', 'job_id': job_id}
dict = unpickle(filepath, logger)
if not dict:
checkcondjob['message'] = \
('The file containing the information ' \
'for job id %s could not be opened! ' \
'You can only check feasibility of ' \
'your own jobs!'
) % job_id
checkcondjobs.append(checkcondjob)
status = returnvalues.CLIENT_ERROR
continue
# Is the job status pending?
possible_check_states = ['QUEUED', 'RETRY', 'FROZEN']
if not dict['STATUS'] in possible_check_states:
checkcondjob['message'] = \
'You can only check feasibility of jobs with status: %s.'\
% ' or '.join(possible_check_states)
checkcondjobs.append(checkcondjob)
continue
# Actually check feasibility
feasible_res = job_feasibility(configuration, dict)
checkcondjob.update(feasible_res)
checkcondjobs.append(checkcondjob)
output_objects.append({
'object_type': 'checkcondjobs',
'checkcondjobs': checkcondjobs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards, dst and current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
# NOTE: we produce output matching an invocation of:
# du -aL --apparent-size --block-size=1 PATH [PATH ...]
filedus = []
summarize_output = summarize(flags)
for abs_path in match:
if invisible_path(abs_path):
continue
relative_path = abs_path.replace(base_dir, '')
# cache accumulated sub dir sizes - du sums into parent dir size
dir_sizes = {}
try:
# Assume a directory to walk
for (root, dirs, files) in walk(abs_path,
topdown=False,
followlinks=True):
if invisible_path(root):
continue
dir_bytes = 0
for name in files:
real_file = os.path.join(root, name)
if invisible_path(real_file):
continue
relative_file = real_file.replace(base_dir, '')
size = os.path.getsize(real_file)
dir_bytes += size
if not summarize_output:
filedus.append({
'object_type': 'filedu',
'name': relative_file,
'bytes': size
})
for name in dirs:
real_dir = os.path.join(root, name)
if invisible_path(real_dir):
continue
dir_bytes += dir_sizes[real_dir]
relative_root = root.replace(base_dir, '')
dir_bytes += os.path.getsize(root)
dir_sizes[root] = dir_bytes
if root == abs_path or not summarize_output:
filedus.append({
'object_type': 'filedu',
'name': relative_root,
'bytes': dir_bytes
})
if os.path.isfile(abs_path):
# Fall back to plain file where walk is empty
size = os.path.getsize(abs_path)
filedus.append({
'object_type': 'filedu',
'name': relative_path,
'bytes': size
})
except Exception as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
if dst:
all_lines += [
'%(bytes)d\t\t%(name)s\n' % entry for entry in filedus
]
else:
output_objects.append({
'object_type': 'filedus',
'filedus': filedus
})
if dst and not write_file(''.join(all_lines), abs_dest, logger):
output_objects.append({
'object_type':
'error_text',
'text':
"failed to write disk use to %s" % relative_dest
})
logger.error("writing disk use to %s for %s failed" %
(abs_dest, client_id))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
status = returnvalues.OK
chroot = ''
if configuration.site_enable_gdp:
chroot = get_project_from_client_id(configuration, client_id)
all_paths = accepted['path']
entry_path = all_paths[-1]
title_entry = find_entry(output_objects, 'title')
user_settings = title_entry.get('user_settings', {})
title_entry['text'] = 'File Manager'
title_entry['style'] = css_tmpl(configuration, user_settings)
legacy_buttons = False
if legacy_user_interface(configuration, user_settings):
legacy_buttons = True
logger.debug("enable legacy buttons")
if configuration.site_enable_jobs and \
'submitjob' in extract_menu(configuration, title_entry):
enable_submit = 'true'
else:
enable_submit = 'false'
csrf_map = {}
method = 'post'
limit = get_csrf_limit(configuration)
for target_op in csrf_backends:
csrf_map[target_op] = make_csrf_token(configuration, method, target_op,
client_id, limit)
(add_import, add_init,
add_ready) = js_tmpl_parts(configuration, entry_path, enable_submit,
str(configuration.site_enable_preview),
legacy_buttons, csrf_map, chroot)
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
title_entry['container_class'] = 'fillwidth',
output_objects.append({
'object_type': 'header',
'class': 'fileman-title',
'text': 'File Manager'
})
output_objects.append({
'object_type':
'html_form',
'text':
html_tmpl(configuration, client_id, title_entry, csrf_map, chroot)
})
if len(all_paths) > 1:
output_objects.append({
'object_type': 'sectionheader',
'text': 'All requested paths:'
})
for path in all_paths:
output_objects.append({
'object_type': 'link',
'text': path,
'destination': 'fileman.py?path=%s' % path
})
output_objects.append({'object_type': 'text', 'text': ''})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path cannot use wildcards here
typecheck_overrides={},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
path = accepted['path'][-1]
chosen_newline = accepted['newline'][-1]
submitjob = accepted['submitjob'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs and submitjob:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
# HTML spec dictates newlines in forms to be MS style (\r\n)
# rather than un*x style (\n): change if requested.
form_newline = '\r\n'
allowed_newline = {'unix': '\n', 'mac': '\r', 'windows': '\r\n'}
output_objects.append({
'object_type': 'header',
'text': 'Saving changes to edited file'
})
if not chosen_newline in allowed_newline.keys():
output_objects.append({
'object_type':
'error_text',
'text':
'Unsupported newline style supplied: %s (must be one of %s)' %
(chosen_newline, ', '.join(allowed_newline.keys()))
})
return (output_objects, returnvalues.CLIENT_ERROR)
saved_newline = allowed_newline[chosen_newline]
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing consistent
# error messages
abs_path = ''
unfiltered_match = glob.glob(base_dir + path)
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if abs_path == '':
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(os.path.join(base_dir, path.lstrip(os.sep)))
if not valid_user_path(configuration, abs_path, base_dir, True):
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, path))
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_path, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_path))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot edit "%s": inside a read-only location!' % path
})
status = returnvalues.CLIENT_ERROR
return (output_objects, returnvalues.CLIENT_ERROR)
(owner, time_left) = acquire_edit_lock(abs_path, client_id)
if owner != client_id:
output_objects.append({
'object_type': 'error_text',
'text': "You don't have the lock for %s!" % path
})
return (output_objects, returnvalues.CLIENT_ERROR)
try:
fh = open(abs_path, 'w+')
fh.write(user_arguments_dict['editarea'][0].replace(
form_newline, saved_newline))
fh.close()
# everything ok
output_objects.append({
'object_type': 'text',
'text': 'Saved changes to %s.' % path
})
logger.info('saved changes to %s' % path)
release_edit_lock(abs_path, client_id)
except Exception as exc:
# Don't give away information about actual fs layout
output_objects.append({
'object_type':
'error_text',
'text':
'%s could not be written! (%s)' %
(path, str(exc).replace(base_dir, ''))
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if submitjob:
output_objects.append({
'object_type': 'text',
'text': 'Submitting saved file to parser'
})
submitstatus = {'object_type': 'submitstatus', 'name': path}
(new_job_status, msg, job_id) = new_job(abs_path, client_id,
configuration, False, True)
if not new_job_status:
submitstatus['status'] = False
submitstatus['message'] = msg
else:
submitstatus['status'] = True
submitstatus['job_id'] = job_id
output_objects.append({
'object_type': 'submitstatuslist',
'submitstatuslist': [submitstatus]
})
output_objects.append({
'object_type': 'link',
'destination': 'javascript:history.back()',
'class': 'backlink iconspace',
'title': 'Go back to previous page',
'text': 'Back to previous page'
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
patterns = accepted['job_id']
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_jobs:
output_objects.append({
'object_type':
'error_text',
'text':
'''Job execution is not enabled on this system'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not patterns:
output_objects.append({
'object_type': 'error_text',
'text': 'No job_id specified!'
})
return (output_objects, returnvalues.NO_SUCH_JOB_ID)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = \
os.path.abspath(os.path.join(configuration.mrsl_files_dir,
client_dir)) + os.sep
filelist = []
keywords_dict = mrslkeywords.get_keywords_dict(configuration)
for pattern in patterns:
pattern = pattern.strip()
# Backward compatibility - all_jobs keyword should match all jobs
if pattern == all_jobs:
pattern = '*'
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern + '.mRSL')
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
# Insert valid job files in filelist for later treatment
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type':
'error_text',
'text':
'%s: You do not have any matching job IDs!' % pattern
})
status = returnvalues.CLIENT_ERROR
else:
filelist += match
# resubmit is hard on the server
if len(filelist) > 100:
output_objects.append({
'object_type':
'error_text',
'text':
'Too many matching jobs (%s)!' % len(filelist)
})
return (output_objects, returnvalues.CLIENT_ERROR)
resubmitobjs = []
status = returnvalues.OK
for filepath in filelist:
mrsl_file = filepath.replace(base_dir, '')
job_id = mrsl_file.replace('.mRSL', '')
# ("Resubmitting job with job_id: %s" % job_id)
resubmitobj = {'object_type': 'resubmitobj', 'job_id': job_id}
mrsl_dict = unpickle(filepath, logger)
if not mrsl_dict:
resubmitobj['message'] = "No such job: %s (%s)" % (job_id,
mrsl_file)
status = returnvalues.CLIENT_ERROR
resubmitobjs.append(resubmitobj)
continue
resubmit_items = keywords_dict.keys()
# loop selected keywords and create mRSL string
resubmit_job_string = ''
for dict_elem in resubmit_items:
value = ''
# Extract job value with fallback to default to support optional
# fields
job_value = mrsl_dict.get(dict_elem,
keywords_dict[dict_elem]['Value'])
if keywords_dict[dict_elem]['Type'].startswith(
'multiplekeyvalues'):
for (elem_key, elem_val) in job_value:
if elem_key:
value += '%s=%s\n' % (str(elem_key).strip(),
str(elem_val).strip())
elif keywords_dict[dict_elem]['Type'].startswith('multiple'):
for elem in job_value:
if elem:
value += '%s\n' % str(elem).rstrip()
else:
if str(job_value):
value += '%s\n' % str(job_value).rstrip()
# Only insert keywords with an associated value
if value:
if value.rstrip() != '':
resubmit_job_string += '''::%s::
%s
''' % (dict_elem, value.rstrip())
# save tempfile
(filehandle, tempfilename) = \
tempfile.mkstemp(dir=configuration.mig_system_files,
text=True)
os.write(filehandle, resubmit_job_string)
os.close(filehandle)
# submit job the usual way
(new_job_status, msg, new_job_id) = new_job(tempfilename, client_id,
configuration, False, True)
if not new_job_status:
resubmitobj['status'] = False
resubmitobj['message'] = msg
status = returnvalues.SYSTEM_ERROR
resubmitobjs.append(resubmitobj)
continue
# o.out("Resubmit failed: %s" % msg)
# o.reply_and_exit(o.ERROR)
resubmitobj['status'] = True
resubmitobj['new_job_id'] = new_job_id
resubmitobjs.append(resubmitobj)
# o.out("Resubmit successful: %s" % msg)
# o.out("%s" % msg)
output_objects.append({
'object_type': 'resubmitobjs',
'resubmitobjs': resubmitobjs
})
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
# NOTE: path can use wildcards, dst and current_dir cannot
typecheck_overrides={'path': valid_path_pattern},
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
algo_list = accepted['hash_algo']
max_chunks = int(accepted['max_chunks'][-1])
pattern_list = accepted['path']
dst = accepted['dst'][-1]
current_dir = accepted['current_dir'][-1].lstrip(os.sep)
# All paths are relative to current_dir
pattern_list = [os.path.join(current_dir, i) for i in pattern_list]
if dst:
dst = os.path.join(current_dir, dst)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({
'object_type': 'text',
'text': '%s using flag: %s' % (op_name, flag)
})
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dir = os.path.abspath(
os.path.join(base_dir, current_dir.lstrip(os.sep)))
if not valid_user_path(configuration, abs_dir, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"You're not allowed to work in %s!" % current_dir
})
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_dir, current_dir))
return (output_objects, returnvalues.CLIENT_ERROR)
if verbose(flags):
output_objects.append({
'object_type': 'text',
'text': "working in %s" % current_dir
})
if dst:
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# NOTE: dst already incorporates current_dir prefix here
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_dest = os.path.abspath(os.path.join(base_dir, dst))
logger.info('chksum in %s' % abs_dest)
# Don't use abs_path in output as it may expose underlying
# fs layout.
relative_dest = abs_dest.replace(base_dir, '')
if not valid_user_path(configuration, abs_dest, base_dir, True):
output_objects.append({
'object_type':
'error_text',
'text':
"Invalid path! (%s expands to an illegal path)" % dst
})
logger.warning('%s tried to %s restricted path %s !(%s)' %
(client_id, op_name, abs_dest, dst))
return (output_objects, returnvalues.CLIENT_ERROR)
if not check_write_access(abs_dest, parent_dir=True):
logger.warning('%s called without write access: %s' %
(op_name, abs_dest))
output_objects.append({
'object_type':
'error_text',
'text':
'cannot checksum to "%s": inside a read-only location!' %
relative_dest
})
return (output_objects, returnvalues.CLIENT_ERROR)
all_lines = []
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)' %
(client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({
'object_type': 'file_not_found',
'name': pattern
})
status = returnvalues.FILE_NOT_FOUND
for abs_path in match:
relative_path = abs_path.replace(base_dir, '')
output_lines = []
for hash_algo in algo_list:
try:
chksum_helper = _algo_map.get(hash_algo, _algo_map["md5"])
checksum = chksum_helper(abs_path, max_chunks=max_chunks)
line = "%s %s\n" % (checksum, relative_path)
logger.info("%s %s of %s: %s" %
(op_name, hash_algo, abs_path, checksum))
output_lines.append(line)
except Exception as exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output', 'lines': output_lines}
output_objects.append(entry)
all_lines += output_lines
if dst and not write_file(''.join(all_lines), abs_dest, logger):
output_objects.append({
'object_type':
'error_text',
'text':
"failed to write checksums to %s" % relative_dest
})
logger.error("writing checksums to %s for %s failed" %
(abs_dest, client_id))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)