def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
output_objects.append({'object_type': 'header', 'text'
: 'Show runtime environment details'})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
re_name = accepted['re_name'][-1]
if not valid_dir_input(configuration.re_home, re_name):
logger.warning(
"possible illegal directory traversal attempt re_name '%s'"
% re_name)
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal runtime environment name: "%s"'
% re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
if not is_runtime_environment(re_name, configuration):
output_objects.append({'object_type': 'error_text', 'text'
: "'%s' is not an existing runtime environment!"
% re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Runtime environment details'
(re_dict, msg) = get_re_dict(re_name, configuration)
if not re_dict:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not read details for "%s"' % msg})
return (output_objects, returnvalues.SYSTEM_ERROR)
output_objects.append(build_reitem_object(configuration, re_dict))
return (output_objects, returnvalues.OK)
def parse(
localfile_spaces,
job_id,
client_id,
forceddestination,
outfile='AUTOMATIC',
):
"""Parse job description and optionally write results to parsed mRSL file.
If outfile is non-empty it is used as destination file, and the keyword
AUTOMATIC is replaced by the default mrsl dir destination.
"""
configuration = get_configuration_object()
logger = configuration.logger
client_dir = client_id_dir(client_id)
# return a tuple (bool status, str msg). This is done because cgi-scripts
# are not allowed to print anything before 'the first two special lines'
# are printed
result = parser.parse(localfile_spaces)
external_dict = mrslkeywords.get_keywords_dict(configuration)
# The mRSL has the right structure check if the types are correct too
# and inline update the default external_dict entries with the ones
# from the actual job specification
(status, msg) = parser.check_types(result, external_dict, configuration)
if not status:
return (False, 'Parse failed (typecheck) %s' % msg)
logger.debug('check_types updated job dict to: %s' % external_dict)
global_dict = {}
# Insert the parts from mrslkeywords we need in the rest of the MiG system
for (key, value_dict) in external_dict.iteritems():
global_dict[key] = value_dict['Value']
# We do not expand any job variables yet in order to allow any future
# resubmits to properly expand job ID.
vgrid_list = global_dict['VGRID']
vgrid_access = user_vgrid_access(configuration, client_id)
# Replace any_vgrid keyword with all allowed vgrids (on time of submit!)
try:
any_pos = vgrid_list.index(any_vgrid)
vgrid_list[any_pos:any_pos] = vgrid_access
# Remove any additional any_vgrid keywords
while any_vgrid in vgrid_list:
vgrid_list.remove(any_vgrid)
except ValueError:
# No any_vgrid keywords in list - move along
pass
# Now validate supplied vgrids
for vgrid_name in vgrid_list:
if not vgrid_name in vgrid_access:
return (False, """Failure: You must be an owner or member of the
'%s' vgrid to submit a job to it!""" % vgrid_name)
# Fall back to default vgrid if no vgrid was supplied
if not vgrid_list:
# Please note that vgrid_list is a ref to global_dict list
# so we must modify and not replace with a new list!
vgrid_list.append(default_vgrid)
# convert specified runtime environments to upper-case and verify they
# actually exist
# do not check runtime envs if the job is for ARC (submission will
# fail later)
if global_dict.get('JOBTYPE', 'unset') != 'arc' \
and global_dict.has_key('RUNTIMEENVIRONMENT'):
re_entries_uppercase = []
for specified_re in global_dict['RUNTIMEENVIRONMENT']:
specified_re = specified_re.upper()
re_entries_uppercase.append(specified_re)
if not is_runtime_environment(specified_re, configuration):
return (False, """You have specified a non-nexisting runtime
environment '%s', therefore the job can not be run on any resources.""" % \
specified_re)
if global_dict.get('MOUNT', []) != []:
if configuration.res_default_mount_re.upper()\
not in re_entries_uppercase:
re_entries_uppercase.append(
configuration.res_default_mount_re.upper())
global_dict['RUNTIMEENVIRONMENT'] = re_entries_uppercase
if global_dict.get('JOBTYPE', 'unset').lower() == 'interactive':
# if jobtype is interactive append command to create the notification
# file .interactivejobfinished that breaks the infinite loop waiting
# for the interactive job to finish and send output files to the MiG
# server
global_dict['EXECUTE'].append('touch .interactivejobfinished')
# put job id and name of user in the dictionary
global_dict['JOB_ID'] = job_id
global_dict['USER_CERT'] = client_id
# mark job as received
global_dict['RECEIVED_TIMESTAMP'] = time.gmtime()
global_dict['STATUS'] = 'PARSE'
if forceddestination:
global_dict['FORCEDDESTINATION'] = forceddestination
if forceddestination.has_key('UNIQUE_RESOURCE_NAME'):
global_dict["RESOURCE"] = "%(UNIQUE_RESOURCE_NAME)s_*" % \
forceddestination
if forceddestination.has_key('RE_NAME'):
re_name = forceddestination['RE_NAME']
# verify the verifyfiles entries are not modified (otherwise RE creator
# can specify multiple ::VERIFYFILES:: keywords and give the entries
# other names (perhaps overwriting files in the home directories of
# resource owners executing the testprocedure)
for verifyfile in global_dict['VERIFYFILES']:
verifytypes = ['.status', '.stderr', '.stdout']
found = False
for verifytype in verifytypes:
if verifyfile == 'verify_runtime_env_%s%s' % (re_name,
verifytype):
found = True
if not found:
return (False, '''You are not allowed to specify the
::VERIFY:: keyword in a testprocedure, it is done automatically''')
# normalize any path fields to be taken relative to home
for field in ('INPUTFILES', 'OUTPUTFILES', 'EXECUTABLES', 'VERIFYFILES'):
if not global_dict.has_key(field):
continue
normalized_field = []
for line in global_dict[field]:
normalized_parts = []
line_parts = line.split(src_dst_sep)
if len(line_parts) < 1 or len(line_parts) > 2:
return (False,
'%s entries must contain 1 or 2 space-separated items'\
% field)
for part in line_parts:
# deny leading slashes i.e. force absolute to relative paths
part = part.lstrip('/')
if part.find('://') != -1:
# keep external targets as is - normpath breaks '://'
normalized_parts.append(part)
check_path = part.split('/')[-1]
else:
# normalize path to avoid e.g. './' which breaks dir
# handling on resource
check_path = os.path.normpath(part)
normalized_parts.append(check_path)
try:
valid_path(check_path)
except Exception, exc:
return (False, 'Invalid %s part in %s: %s' % \
(field, html_escape(part), exc))
normalized_field.append(' '.join(normalized_parts))
global_dict[field] = normalized_field
def run(configuration,
localfile_spaces,
unique_resource_name,
outfile='AUTOMATIC'):
"""Parse configuration in localfile_spaces and write results to outfile
if non-empty. The keyword AUTOMATIC is replaced by the expected resource
configuration path.
"""
if not configuration:
configuration = get_configuration_object()
(status, msg, conf) = get_resource_config_dict(configuration,
localfile_spaces)
if not status:
return (False, msg)
# verify runtime environments are specified correctly
if conf.has_key('RUNTIMEENVIRONMENT'):
for re in conf['RUNTIMEENVIRONMENT']:
try:
(name, value) = re
except Exception, err:
return (False, 'Runtime environment error: %s' % err)
if not refunctions.is_runtime_environment(name, configuration):
return (
False,
"Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations."
% name)
(re_dict, msg) = refunctions.get_re_dict(name, configuration)
if not re_dict:
return (False,
'Runtime environment error, could not open (%s) %s' %
(name, msg))
if not re_dict.has_key('ENVIRONMENTVARIABLE'):
if value:
# res conf has envs, but according to the template it should not
return (
False,
"%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>"
% (re, value, name))
else:
continue
re_dict_environments = re_dict['ENVIRONMENTVARIABLE']
re_dict_environment_names = []
for re_environment in re_dict_environments:
re_dict_environment_names.append(re_environment['name'])
if not len(value) == len(re_dict_environments):
return (
False,
"You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>"
% (len(value), name, len(re_dict_environments), name))
# we now know that the number of environments are
# correct, verify that there are no name duplicates
used_envnames = []
for env in value:
try:
(envname, _) = env
if envname in used_envnames:
# same envname used twice
return (
False,
"You have specified the environment '%s' more than once for the '%s' runtime environment."
% (envname, name))
used_envnames.append(envname)
except Exception, err:
return (
False,
'Runtimeenvironment error: Name and value not found in env: %s'
% err)
# verify environment names are correct according to the
# runtime environment definition do this by comparing
# list of names specified for runtime environment and
# res. conf.
# re_dict_environment_names and used_envnames should
# have the same entries!
for n in re_dict_environment_names:
# any build-in list comparison functionality?
if not n in used_envnames:
return (
False,
"You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>"
% (n, name, name))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
output_objects.append({
'object_type': 'header',
'text': 'Create runtime environment'
})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
re_template = accepted['re_template'][-1].upper().strip()
software_entries = int(accepted['software_entries'][-1])
environment_entries = int(accepted['environment_entries'][-1])
testprocedure_entry = int(accepted['testprocedure_entry'][-1])
template = {}
if re_template:
if not is_runtime_environment(re_template, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
"re_template ('%s') is not a valid existing runtime env!" %
re_template
})
return (output_objects, returnvalues.CLIENT_ERROR)
(template, msg) = get_re_dict(re_template, configuration)
if not template:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not read re_template %s. %s' % (re_template, msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Override template fields if user loaded a template and modified the
# required entries and chose update.
# Use default of 1 sw, 1 env and 0 test or template setting otherwise
if software_entries < 0:
software_entries = len(template.get('SOFTWARE', [None]))
if environment_entries < 0:
environment_entries = len(template.get('ENVIRONMENTVARIABLE', [None]))
if testprocedure_entry < 0:
testprocedure_entry = len(template.get('TESTPROCEDURE', []))
if template.has_key('SOFTWARE'):
new_sw = template['SOFTWARE'][:software_entries]
template['SOFTWARE'] = new_sw
if template.has_key('ENVIRONMENTVARIABLE'):
new_env = template['ENVIRONMENTVARIABLE'][:environment_entries]
template['ENVIRONMENTVARIABLE'] = new_env
if template.has_key('TESTPROCEDURE'):
new_test = template['TESTPROCEDURE'][:testprocedure_entry]
template['TESTPROCEDURE'] = new_test
# Avoid DoS, limit number of software_entries
if software_entries > max_software_entries:
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Maximum number of software_entries %s exceeded (%s)' % \
(max_software_entries, software_entries)})
return (output_objects, returnvalues.CLIENT_ERROR)
# Avoid DoS, limit number of environment_entries
if environment_entries > max_environment_entries:
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Maximum number of environment_entries %s exceeded (%s)' % \
(max_environment_entries, environment_entries)})
return (output_objects, returnvalues.CLIENT_ERROR)
rekeywords_dict = get_keywords_dict()
(list_status, ret) = list_runtime_environments(configuration)
if not list_status:
output_objects.append({'object_type': 'error_text', 'text': ret})
return (output_objects, returnvalues.SYSTEM_ERROR)
output_objects.append({
'object_type':
'text',
'text':
'Use existing Runtime Environment as template'
})
html_form = \
"""<form method='get' action='adminre.py'>
<select name='re_template'>
<option value=''>None</option>
"""
for existing_re in ret:
html_form += " <option value='%s'>%s</option>\n" % \
(existing_re, existing_re)
html_form += """
</select>
<input type='submit' value='Get' />
</form>"""
output_objects.append({'object_type': 'html_form', 'text': html_form})
output_objects.append({
'object_type':
'text',
'text':
'''Note that a runtime environment can not be changed after creation
and it can only be removed if not in use by any resources, so please be careful
when filling in the details'''
})
output_objects.append({
'object_type':
'text',
'text':
'''Changing the number of software and environment entries removes
all data in the form, so please enter the correct values before entering any
information.'''
})
html_form = \
"""<form method='get' action='adminre.py'>
<table>
"""
html_form += """
<tr>
<td>Number of needed software entries</td>
<td><input type='number' name='software_entries' min=0 max=99
minlength=1 maxlength=2 value='%s' required pattern='[0-9]{1,2}'
title='number of software entries needed in runtime environment' /></td>
</tr>""" % software_entries
html_form += """
<tr>
<td>Number of environment entries</td>
<td>
<input type='number' name='environment_entries' min=0 max=99
minlength=1 maxlength=2 value='%s' required pattern='[0-9]{1,2}'
title='number of environment variables provided by runtime environment' />
</td>
</tr>""" % environment_entries
output_objects.append({'object_type': 'html_form', 'text': html_form})
if testprocedure_entry == 0:
select_string = """<option value='0' selected>No</option>
<option value=1>Yes</option>"""
elif testprocedure_entry == 1:
select_string = """<option value='0'>No</option>
<option value='1' selected>Yes</option>"""
else:
output_objects.append({
'object_type':
'error_text',
'text':
'testprocedure_entry should be 0 or 1, you specified %s' %
testprocedure_entry
})
return (output_objects, returnvalues.CLIENT_ERROR)
html_form = """
<tr>
<td>Runtime environment has a testprocedure</td>
<td><select name='testprocedure_entry'>%s</select></td>
</tr>
<tr>
<td colspan=2>
<input type='hidden' name='re_template' value='%s' />
<input type='submit' value='Update fields' />
</td>
</tr>
</table>
</form><br />
""" % (select_string, re_template)
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'short_title': configuration.short_title,
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
target_op = 'createre'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
html_form += """
<form method='%(form_method)s' action='%(target_op)s.py'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<b>Runtime Environment Name</b><br />
<small>(eg. BASH-2.X-1, must be unique):</small><br />
<input class='p80width' type='text' name='re_name' required
pattern='[a-zA-Z0-9_.-]+'
title='unique name of ASCII letters and digits separated only by underscores, periods and hyphens' />
<br />
<br /><b>Description:</b><br />
<textarea class='p80width' rows='4' name='redescription'>
"""
if template:
html_form += template['DESCRIPTION'].replace('<br />', '\n')
html_form += '</textarea><br />'
soft_list = []
if software_entries > 0:
html_form += '<br /><b>Needed Software:</b><br />'
if template:
if template.has_key('SOFTWARE'):
soft_list = template['SOFTWARE']
for soft in soft_list:
html_form += """
<textarea class='p80width' rows='6' name='software'>"""
for keyname in soft.keys():
if keyname != '':
html_form += '%s=%s\n' % (keyname, soft[keyname])
html_form += '</textarea><br />'
# loop and create textareas for any missing software entries
software = rekeywords_dict['SOFTWARE']
sublevel_required = []
sublevel_optional = []
if software.has_key('Sublevel') and software['Sublevel']:
sublevel_required = software['Sublevel_required']
sublevel_optional = software['Sublevel_optional']
for _ in range(len(soft_list), software_entries):
html_form += """
<textarea class='p80width' rows='6' name='software'>"""
for sub_req in sublevel_required:
html_form += '%s= # required\n' % sub_req
for sub_opt in sublevel_optional:
html_form += '%s= # optional\n' % sub_opt
html_form += '</textarea><br />'
if template and testprocedure_entry == 1:
if template.has_key('TESTPROCEDURE'):
html_form += """
<br /><b>Testprocedure</b> (in mRSL format):<br />
<textarea class='p80width' rows='15' name='testprocedure'>"""
base64string = ''
for stringpart in template['TESTPROCEDURE']:
base64string += stringpart
decodedstring = base64.decodestring(base64string)
html_form += decodedstring
html_form += '</textarea>'
output_objects.append({
'object_type': 'html_form',
'text': html_form
})
html_form = """
<br /><b>Expected .stdout file if testprocedure is executed</b><br />
<textarea class='p80width' rows='10' name='verifystdout'>"""
if template.has_key('VERIFYSTDOUT'):
for line in template['VERIFYSTDOUT']:
html_form += line
html_form += '</textarea>'
html_form += """
<br /><b>Expected .stderr file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystderr'>"""
if template.has_key('VERIFYSTDERR'):
for line in template['VERIFYSTDERR']:
html_form += line
html_form += '</textarea>'
html_form += """
<br /><b>Expected .status file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystatus'>"""
if template.has_key('VERIFYSTATUS'):
for line in template['VERIFYSTATUS']:
html_form += line
html_form += '</textarea>'
elif testprocedure_entry == 1:
html_form += """
<br /><b>Testprocedure</b> (in mRSL format):<br />
<textarea class='p80width' rows='15' name='testprocedure'>"""
html_form += \
"""::EXECUTE::
ls
</textarea>
<br /><b>Expected .stdout file if testprocedure is executed</b><br />
<textarea class='p80width' rows='10' name='verifystdout'></textarea>
<br /><b>Expected .stderr file if testprocedure is executed</b><br />
<textarea class='p80width' rows='10' name='verifystderr'></textarea>
<br /><b>Expected .status file if testprocedure is executed</b><br />
<textarea class='p80width' rows='10' name='verifystatus'></textarea>
"""
environmentvariable = rekeywords_dict['ENVIRONMENTVARIABLE']
sublevel_required = []
sublevel_optional = []
if environmentvariable.has_key('Sublevel')\
and environmentvariable['Sublevel']:
sublevel_required = environmentvariable['Sublevel_required']
sublevel_optional = environmentvariable['Sublevel_optional']
env_list = []
if environment_entries > 0:
html_form += '<br /><b>Environments:</b><br />'
if template:
if template.has_key('ENVIRONMENTVARIABLE'):
env_list = template['ENVIRONMENTVARIABLE']
for env in env_list:
html_form += """
<textarea class='p80width' rows='4' name='environment'>"""
for keyname in env.keys():
if keyname != '':
html_form += '%s=%s\n' % (keyname, env[keyname])
html_form += '</textarea><br />'
# loop and create textareas for any missing environment entries
for _ in range(len(env_list), environment_entries):
html_form += """
<textarea class='p80width' rows='4' name='environment'>"""
for sub_req in sublevel_required:
html_form += '%s= # required\n' % sub_req
for sub_opt in sublevel_optional:
html_form += '%s= # optional\n' % sub_opt
html_form += '</textarea><br />'
html_form += """<br /><br /><input type='submit' value='Create' />
</form>
"""
output_objects.append({
'object_type': 'html_form',
'text': html_form % fill_helpers
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Delete runtime environment'
output_objects.append({'object_type': 'header', 'text'
: 'Delete runtime environment'})
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
re_name = accepted['re_name'][-1]
if not valid_dir_input(configuration.re_home, re_name):
logger.warning(
"possible illegal directory traversal attempt re_name '%s'"
% re_name)
output_objects.append({'object_type': 'error_text', 'text'
: 'Illegal runtime environment name: "%s"'
% re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
# Check whether re_name represents a runtime environment
if not is_runtime_environment(re_name, configuration):
output_objects.append({'object_type': 'error_text',
'text': "No such runtime environment: '%s'"
% re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
re_dict = get_re_dict(re_name, configuration)
if not re_dict[0]:
output_objects.append(
{'object_type': 'error_text',
'text': 'Could not read runtime environment details for %s'
% re_name})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Make sure the runtime environment belongs to the user trying to delete it
if client_id != re_dict[0]['CREATOR']:
output_objects.append({'object_type': 'error_text', 'text': \
'You are not the owner of runtime environment "%s"' % re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
# Prevent delete if the runtime environment is used by any resources
actives = resources_using_re(configuration, re_name)
# If the runtime environment is active, an error message is printed, along
# with a list of the resources using the runtime environment
if actives:
output_objects.append(
{'object_type': 'error_text', 'text':
"Can't delete runtime environment '%s' in use by resources:"
% re_name})
output_objects.append({'object_type': 'list', 'list'
: actives})
output_objects.append({'object_type': 'link', 'destination': 'redb.py',
'class': 'infolink', 'title':
'Show runtime environments',
'text': 'Show runtime environments'})
return (output_objects, returnvalues.CLIENT_ERROR)
# Delete the runtime environment
(status, msg) = delete_runtimeenv(re_name, configuration)
# If something goes wrong when trying to delete runtime environment
# re_name, an error is displayed.
if not status:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not remove %s runtime environment: %s'
% (re_name, msg)})
return (output_objects, returnvalues.SYSTEM_ERROR)
# If deletion of runtime environment re_name is successful, we just
# return OK
else:
output_objects.append(
{'object_type': 'text', 'text'
: 'Successfully deleted runtime environment: "%s"' % re_name})
output_objects.append({'object_type': 'link', 'destination': 'redb.py',
'class': 'infolink',
'title': 'Show runtime environments',
'text': 'Show runtime environments'})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
output_objects.append({"object_type": "header", "text": "Create runtime environment"})
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
re_template = accepted["re_template"][-1].upper().strip()
software_entries = int(accepted["software_entries"][-1])
environment_entries = int(accepted["environment_entries"][-1])
testprocedure_entry = int(accepted["testprocedure_entry"][-1])
template = {}
if re_template:
if not is_runtime_environment(re_template, configuration):
output_objects.append(
{
"object_type": "error_text",
"text": "re_template ('%s') is not a valid existing runtime env!" % re_template,
}
)
return (output_objects, returnvalues.CLIENT_ERROR)
(template, msg) = get_re_dict(re_template, configuration)
if not template:
output_objects.append(
{"object_type": "error_text", "text": "Could not read re_template %s. %s" % (re_template, msg)}
)
return (output_objects, returnvalues.SYSTEM_ERROR)
# Override template fields if user loaded a template and modified the
# required entries and chose update.
# Use default of 1 sw, 1 env and 0 test or template setting otherwise
if software_entries < 0:
software_entries = len(template.get("SOFTWARE", [None]))
if environment_entries < 0:
environment_entries = len(template.get("ENVIRONMENTVARIABLE", [None]))
if testprocedure_entry < 0:
testprocedure_entry = len(template.get("TESTPROCEDURE", []))
if template.has_key("SOFTWARE"):
new_sw = template["SOFTWARE"][:software_entries]
template["SOFTWARE"] = new_sw
if template.has_key("ENVIRONMENTVARIABLE"):
new_env = template["ENVIRONMENTVARIABLE"][:environment_entries]
template["ENVIRONMENTVARIABLE"] = new_env
if template.has_key("TESTPROCEDURE"):
new_test = template["TESTPROCEDURE"][:testprocedure_entry]
template["TESTPROCEDURE"] = new_test
# Avoid DoS, limit number of software_entries
if software_entries > max_software_entries:
output_objects.append(
{
"object_type": "error_text",
"text": "Maximum number of software_entries %s exceeded (%s)"
% (max_software_entries, software_entries),
}
)
return (output_objects, returnvalues.CLIENT_ERROR)
# Avoid DoS, limit number of environment_entries
if environment_entries > max_environment_entries:
output_objects.append(
{
"object_type": "error_text",
"text": "Maximum number of environment_entries %s exceeded (%s)"
% (max_environment_entries, environment_entries),
}
)
return (output_objects, returnvalues.CLIENT_ERROR)
rekeywords_dict = get_keywords_dict()
(status, ret) = list_runtime_environments(configuration)
if not status:
output_objects.append({"object_type": "error_text", "text": ret})
return (output_objects, returnvalues.SYSTEM_ERROR)
output_objects.append({"object_type": "text", "text": "Use existing RE as template"})
html_form = """<form method='get' action='adminre.py'>
<select name='re_template'>
<option value=''>None</option>
"""
for existing_re in ret:
html_form += " <option value='%s'>%s</option>\n" % (existing_re, existing_re)
html_form += """
</select>
<input type='submit' value='Get' />
</form>"""
output_objects.append({"object_type": "html_form", "text": html_form})
output_objects.append(
{
"object_type": "text",
"text": """Note that a runtime environment can not be changed after creation
and it can only be removed if not in use by any resources, so please be careful
when filling in the details""",
}
)
output_objects.append(
{
"object_type": "text",
"text": """Changing the number of software and environment entries removes
all data in the form, so please enter the correct values before entering any
information.""",
}
)
html_form = """<form method='get' action='adminre.py'>
<table>
"""
html_form += (
"""
<tr>
<td>Number of needed software entries</td>
<td><input type='text' size='2' name='software_entries' value='%s' /></td>
</tr>"""
% software_entries
)
html_form += (
"""
<tr>
<td>Number of environment entries</td>
<td>
<input type='text' size='2' name='environment_entries' value='%s' />
</td>
</tr>"""
% environment_entries
)
output_objects.append({"object_type": "html_form", "text": html_form})
if testprocedure_entry == 0:
select_string = """<option value='0' selected>No</option>
<option value=1>Yes</option>"""
elif testprocedure_entry == 1:
select_string = """<option value='0'>No</option>
<option value='1' selected>Yes</option>"""
else:
output_objects.append(
{
"object_type": "error_text",
"text": "testprocedure_entry should be 0 or 1, you specified %s" % testprocedure_entry,
}
)
return (output_objects, returnvalues.CLIENT_ERROR)
html_form = """
<tr>
<td>Runtime environment has a testprocedure</td>
<td><select name='testprocedure_entry'>%s</select></td>
</tr>
<tr>
<td colspan=2>
<input type='hidden' name='re_template' value='%s' />
<input type='submit' value='Update fields' />
</td>
</tr>
</table>
</form><br />
""" % (
select_string,
re_template,
)
html_form += """
<form method='post' action='createre.py'>
<b>RE Name</b><br />
<small>(eg. BASH-2.X-1, must be unique):</small><br />
<input type='text' size='40' name='re_name' /><br />
<br /><b>Description:</b><br />
<textarea cols='50' rows='2' name='redescription'>
"""
if template:
html_form += template["DESCRIPTION"].replace("<br />", "\n")
html_form += "</textarea><br />"
soft_list = []
if software_entries > 0:
html_form += "<br /><b>Needed Software:</b><br />"
if template:
if template.has_key("SOFTWARE"):
soft_list = template["SOFTWARE"]
for soft in soft_list:
html_form += """
<textarea cols='50' rows='5' name='software'>"""
for keyname in soft.keys():
if keyname != "":
html_form += "%s=%s\n" % (keyname, soft[keyname])
html_form += "</textarea><br />"
# loop and create textareas for any missing software entries
software = rekeywords_dict["SOFTWARE"]
sublevel_required = []
sublevel_optional = []
if software.has_key("Sublevel") and software["Sublevel"]:
sublevel_required = software["Sublevel_required"]
sublevel_optional = software["Sublevel_optional"]
for _ in range(len(soft_list), software_entries):
html_form += """
<textarea cols='50' rows='5' name='software'>"""
for sub_req in sublevel_required:
html_form += "%s= # required\n" % sub_req
for sub_opt in sublevel_optional:
html_form += "%s= # optional\n" % sub_opt
html_form += "</textarea><br />"
if template and testprocedure_entry == 1:
if template.has_key("TESTPROCEDURE"):
html_form += """
<br /><b>Testprocedure</b> (in mRSL format):<br />
<textarea cols='50' rows='15' name='testprocedure'>"""
base64string = ""
for stringpart in template["TESTPROCEDURE"]:
base64string += stringpart
decodedstring = base64.decodestring(base64string)
html_form += decodedstring
html_form += "</textarea>"
output_objects.append({"object_type": "html_form", "text": html_form})
html_form = """
<br /><b>Expected .stdout file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystdout'>"""
if template.has_key("VERIFYSTDOUT"):
for line in template["VERIFYSTDOUT"]:
html_form += line
html_form += "</textarea>"
html_form += """
<br /><b>Expected .stderr file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystderr'>"""
if template.has_key("VERIFYSTDERR"):
for line in template["VERIFYSTDERR"]:
html_form += line
html_form += "</textarea>"
html_form += """
<br /><b>Expected .status file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystatus'>"""
if template.has_key("VERIFYSTATUS"):
for line in template["VERIFYSTATUS"]:
html_form += line
html_form += "</textarea>"
elif testprocedure_entry == 1:
html_form += """
<br /><b>Testprocedure</b> (in mRSL format):<br />
<textarea cols='50' rows='15' name='testprocedure'>"""
html_form += """::EXECUTE::
ls
</textarea>
<br /><b>Expected .stdout file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystdout'></textarea>
<br /><b>Expected .stderr file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystderr'></textarea>
<br /><b>Expected .status file if testprocedure is executed</b><br />
<textarea cols='50' rows='10' name='verifystatus'></textarea>
"""
environmentvariable = rekeywords_dict["ENVIRONMENTVARIABLE"]
sublevel_required = []
sublevel_optional = []
if environmentvariable.has_key("Sublevel") and environmentvariable["Sublevel"]:
sublevel_required = environmentvariable["Sublevel_required"]
sublevel_optional = environmentvariable["Sublevel_optional"]
env_list = []
if environment_entries > 0:
html_form += "<br /><b>Environments:</b><br />"
if template:
if template.has_key("ENVIRONMENTVARIABLE"):
env_list = template["ENVIRONMENTVARIABLE"]
for env in env_list:
html_form += """
<textarea cols='50' rows='3' name='environment'>"""
for keyname in env.keys():
if keyname != "":
html_form += "%s=%s\n" % (keyname, env[keyname])
html_form += "</textarea><br />"
# loop and create textareas for any missing environment entries
for _ in range(len(env_list), environment_entries):
html_form += """
<textarea cols='50' rows='3' name='environment'>"""
for sub_req in sublevel_required:
html_form += "%s= # required\n" % sub_req
for sub_opt in sublevel_optional:
html_form += "%s= # optional\n" % sub_opt
html_form += "</textarea><br />"
html_form += """<br /><br /><input type='submit' value='Create' />
</form>
"""
output_objects.append({"object_type": "html_form", "text": html_form})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Show Runtime Environment Details'
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
re_name = accepted['re_name'][-1]
output_objects.append({
'object_type': 'header',
'text': 'Show runtime environment details'
})
if not valid_dir_input(configuration.re_home, re_name):
logger.warning(
"possible illegal directory traversal attempt re_name '%s'" %
re_name)
output_objects.append({
'object_type':
'error_text',
'text':
'Illegal runtime environment name: "%s"' % re_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not is_runtime_environment(re_name, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
"'%s' is not an existing runtime environment!" % re_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Runtime environment details'
(re_dict, msg) = get_re_dict(re_name, configuration)
if not re_dict:
output_objects.append({
'object_type': 'error_text',
'text': 'Could not read details for "%s"' % msg
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Set providers explicitly after build_reitem_object to avoid import loop
re_item = build_reitem_object(configuration, re_dict)
re_name = re_item['name']
re_item['providers'] = resources_using_re(configuration, re_name)
re_item['resource_count'] = len(re_item['providers'])
output_objects.append(re_item)
return (output_objects, returnvalues.OK)
def parse(
localfile_spaces,
job_id,
client_id,
forceddestination,
outfile='AUTOMATIC',
):
"""Parse job description and optionally write results to parsed mRSL file.
If outfile is non-empty it is used as destination file, and the keyword
AUTOMATIC is replaced by the default mrsl dir destination.
"""
configuration = get_configuration_object()
logger = configuration.logger
client_dir = client_id_dir(client_id)
# return a tuple (bool status, str msg). This is done because cgi-scripts
# are not allowed to print anything before 'the first two special lines'
# are printed
result = parser.parse(localfile_spaces)
external_dict = mrslkeywords.get_keywords_dict(configuration)
# The mRSL has the right structure check if the types are correct too
# and inline update the default external_dict entries with the ones
# from the actual job specification
(status, msg) = parser.check_types(result, external_dict,
configuration)
if not status:
return (False, 'Parse failed (typecheck) %s' % msg)
logger.debug('check_types updated job dict to: %s' % external_dict)
global_dict = {}
# Insert the parts from mrslkeywords we need in the rest of the MiG system
for (key, value_dict) in external_dict.iteritems():
global_dict[key] = value_dict['Value']
# We do not expand any job variables yet in order to allow any future
# resubmits to properly expand job ID.
vgrid_list = global_dict['VGRID']
allowed_vgrids = user_allowed_vgrids(configuration, client_id)
# Replace any_vgrid keyword with all allowed vgrids (on time of submit!)
try:
any_pos = vgrid_list.index(any_vgrid)
vgrid_list[any_pos:any_pos] = allowed_vgrids
# Remove any additional any_vgrid keywords
while any_vgrid in vgrid_list:
vgrid_list.remove(any_vgrid)
except ValueError:
# No any_vgrid keywords in list - move along
pass
# Now validate supplied vgrids
for vgrid_name in vgrid_list:
if not vgrid_name in allowed_vgrids:
return (False, """Failure: You must be an owner or member of the
'%s' vgrid to submit a job to it!""" % vgrid_name)
# Fall back to default vgrid if no vgrid was supplied
if not vgrid_list:
# Please note that vgrid_list is a ref to global_dict list
# so we must modify and not replace with a new list!
vgrid_list.append(default_vgrid)
# convert specified runtime environments to upper-case and verify they
# actually exist
# do not check runtime envs if the job is for ARC (submission will
# fail later)
if global_dict.get('JOBTYPE', 'unset') != 'arc' \
and global_dict.has_key('RUNTIMEENVIRONMENT'):
re_entries_uppercase = []
for specified_re in global_dict['RUNTIMEENVIRONMENT']:
specified_re = specified_re.upper()
re_entries_uppercase.append(specified_re)
if not is_runtime_environment(specified_re, configuration):
return (False, """You have specified a non-nexisting runtime
environment '%s', therefore the job can not be run on any resources.""" % \
specified_re)
if global_dict.get('MOUNT', []) != []:
re_entries_uppercase.append(configuration.res_default_mount_re.upper())
global_dict['RUNTIMEENVIRONMENT'] = re_entries_uppercase
if global_dict.get('JOBTYPE', 'unset').lower() == 'interactive':
# if jobtype is interactive append command to create the notification
# file .interactivejobfinished that breaks the infinite loop waiting
# for the interactive job to finish and send output files to the MiG
# server
global_dict['EXECUTE'].append('touch .interactivejobfinished')
# put job id and name of user in the dictionary
global_dict['JOB_ID'] = job_id
global_dict['USER_CERT'] = client_id
# mark job as received
global_dict['RECEIVED_TIMESTAMP'] = time.gmtime()
global_dict['STATUS'] = 'PARSE'
if forceddestination:
global_dict['FORCEDDESTINATION'] = forceddestination
if forceddestination.has_key('UNIQUE_RESOURCE_NAME'):
global_dict["RESOURCE"] = "%(UNIQUE_RESOURCE_NAME)s_*" % \
forceddestination
if forceddestination.has_key('RE_NAME'):
re_name = forceddestination['RE_NAME']
# verify the verifyfiles entries are not modified (otherwise RE creator
# can specify multiple ::VERIFYFILES:: keywords and give the entries
# other names (perhaps overwriting files in the home directories of
# resource owners executing the testprocedure)
for verifyfile in global_dict['VERIFYFILES']:
verifytypes = ['.status', '.stderr', '.stdout']
found = False
for verifytype in verifytypes:
if verifyfile == 'verify_runtime_env_%s%s' % (re_name,
verifytype):
found = True
if not found:
return (False, '''You are not allowed to specify the
::VERIFY:: keyword in a testprocedure, it is done automatically''')
# normalize any path fields to be taken relative to home
for field in ('INPUTFILES', 'OUTPUTFILES', 'EXECUTABLES',
'VERIFYFILES'):
if not global_dict.has_key(field):
continue
normalized_field = []
for line in global_dict[field]:
normalized_parts = []
line_parts = line.split()
if len(line_parts) < 1 or len(line_parts) > 2:
return (False,
'%s entries must contain 1 or 2 space-separated items'\
% field)
for part in line_parts:
# deny leading slashes i.e. force absolute to relative paths
part = part.lstrip('/')
if part.find('://') != -1:
# keep external targets as is - normpath breaks '://'
normalized_parts.append(part)
check_path = part.split('/')[-1]
else:
# normalize path to avoid e.g. './' which breaks dir
# handling on resource
check_path = os.path.normpath(part)
normalized_parts.append(check_path)
try:
valid_path(check_path)
except Exception, exc:
return (False, 'Invalid %s part in %s: %s' % \
(field, html_escape(part), exc))
normalized_field.append(' '.join(normalized_parts))
global_dict[field] = normalized_field
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'Delete runtime environment'
output_objects.append({
'object_type': 'header',
'text': 'Delete runtime environment'
})
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
re_name = accepted['re_name'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not valid_dir_input(configuration.re_home, re_name):
logger.warning(
"possible illegal directory traversal attempt re_name '%s'" %
re_name)
output_objects.append({
'object_type':
'error_text',
'text':
'Illegal runtime environment name: "%s"' % re_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Check whether re_name represents a runtime environment
if not is_runtime_environment(re_name, configuration):
output_objects.append({
'object_type':
'error_text',
'text':
"No such runtime environment: '%s'" % re_name
})
return (output_objects, returnvalues.CLIENT_ERROR)
(re_dict, load_msg) = get_re_dict(re_name, configuration)
if not re_dict:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not read runtime environment details for %s: %s' %
(re_name, load_msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Make sure the runtime environment belongs to the user trying to delete it
if client_id != re_dict['CREATOR']:
output_objects.append({'object_type': 'error_text', 'text': \
'You are not the owner of runtime environment "%s"' % re_name})
return (output_objects, returnvalues.CLIENT_ERROR)
# Prevent delete if the runtime environment is used by any resources
actives = resources_using_re(configuration, re_name)
# If the runtime environment is active, an error message is printed, along
# with a list of the resources using the runtime environment
if actives:
output_objects.append({
'object_type':
'error_text',
'text':
"Can't delete runtime environment '%s' in use by resources:" %
re_name
})
output_objects.append({'object_type': 'list', 'list': actives})
output_objects.append({
'object_type': 'link',
'destination': 'redb.py',
'class': 'infolink iconspace',
'title': 'Show runtime environments',
'text': 'Show runtime environments'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Delete the runtime environment
(del_status, msg) = delete_runtimeenv(re_name, configuration)
# If something goes wrong when trying to delete runtime environment
# re_name, an error is displayed.
if not del_status:
output_objects.append({
'object_type':
'error_text',
'text':
'Could not remove %s runtime environment: %s' % (re_name, msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# If deletion of runtime environment re_name is successful, we just
# return OK
else:
output_objects.append({
'object_type':
'text',
'text':
'Successfully deleted runtime environment: "%s"' % re_name
})
output_objects.append({
'object_type': 'link',
'destination': 'redb.py',
'class': 'infolink iconspace',
'title': 'Show runtime environments',
'text': 'Show runtime environments'
})
return (output_objects, returnvalues.OK)
def run(localfile_spaces, unique_resource_name, outfile='AUTOMATIC'
):
"""Parse configuration in localfile_spaces and write results to outfile
if non-empty. The keyword AUTOMATIC is replaced by the expected resource
configuration path.
"""
(status, msg, conf) = get_resource_config_dict(localfile_spaces)
if not status:
return (False, msg)
# verify runtime environments are specified correctly
if conf.has_key('RUNTIMEENVIRONMENT'):
for re in conf['RUNTIMEENVIRONMENT']:
try:
(name, value) = re
except Exception, err:
return (False, 'Runtime environment error: %s' % err)
if not refunctions.is_runtime_environment(name,
configuration):
return (False,
"Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations."
% name)
(re_dict, msg) = refunctions.get_re_dict(name,
configuration)
if not re_dict:
return (False,
'Runtime environment error, could not open (%s) %s'
% (name, msg))
if not re_dict.has_key('ENVIRONMENTVARIABLE'):
if value:
# res conf has envs, but according to the template it should not
return (False,
"%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>"
% (re, value, name))
else:
continue
re_dict_environments = re_dict['ENVIRONMENTVARIABLE']
re_dict_environment_names = []
for re_environment in re_dict_environments:
re_dict_environment_names.append(re_environment['name'])
if not len(value) == len(re_dict_environments):
return (False,
"You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>"
% (len(value), name,
len(re_dict_environments), name))
# we now know that the number of environments are
# correct, verify that there are no name duplicates
used_envnames = []
for env in value:
try:
(envname, _) = env
if envname in used_envnames:
# same envname used twice
return (False,
"You have specified the environment '%s' more than once for the '%s' runtime environment."
% (envname, name))
used_envnames.append(envname)
except Exception, err:
return (False,
'Runtimeenvironment error: Name and value not found in env: %s'
% err)
# verify environment names are correct according to the
# runtime environment definition do this by comparing
# list of names specified for runtime environment and
# res. conf.
# re_dict_environment_names and used_envnames should
# have the same entries!
for n in re_dict_environment_names:
# any build-in list comparison functionality?
if not n in used_envnames:
return (False,
"You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>"
% (n, name, name))