def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] output_objects.append({'object_type': 'header', 'text' : 'Show runtime environment details'}) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) re_name = accepted['re_name'][-1] if not valid_dir_input(configuration.re_home, re_name): logger.warning( "possible illegal directory traversal attempt re_name '%s'" % re_name) output_objects.append({'object_type': 'error_text', 'text' : 'Illegal runtime environment name: "%s"' % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) if not is_runtime_environment(re_name, configuration): output_objects.append({'object_type': 'error_text', 'text' : "'%s' is not an existing runtime environment!" % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Runtime environment details' (re_dict, msg) = get_re_dict(re_name, configuration) if not re_dict: output_objects.append({'object_type': 'error_text', 'text' : 'Could not read details for "%s"' % msg}) return (output_objects, returnvalues.SYSTEM_ERROR) output_objects.append(build_reitem_object(configuration, re_dict)) return (output_objects, returnvalues.OK)
def parse( localfile_spaces, job_id, client_id, forceddestination, outfile='AUTOMATIC', ): """Parse job description and optionally write results to parsed mRSL file. If outfile is non-empty it is used as destination file, and the keyword AUTOMATIC is replaced by the default mrsl dir destination. """ configuration = get_configuration_object() logger = configuration.logger client_dir = client_id_dir(client_id) # return a tuple (bool status, str msg). This is done because cgi-scripts # are not allowed to print anything before 'the first two special lines' # are printed result = parser.parse(localfile_spaces) external_dict = mrslkeywords.get_keywords_dict(configuration) # The mRSL has the right structure check if the types are correct too # and inline update the default external_dict entries with the ones # from the actual job specification (status, msg) = parser.check_types(result, external_dict, configuration) if not status: return (False, 'Parse failed (typecheck) %s' % msg) logger.debug('check_types updated job dict to: %s' % external_dict) global_dict = {} # Insert the parts from mrslkeywords we need in the rest of the MiG system for (key, value_dict) in external_dict.iteritems(): global_dict[key] = value_dict['Value'] # We do not expand any job variables yet in order to allow any future # resubmits to properly expand job ID. vgrid_list = global_dict['VGRID'] vgrid_access = user_vgrid_access(configuration, client_id) # Replace any_vgrid keyword with all allowed vgrids (on time of submit!) try: any_pos = vgrid_list.index(any_vgrid) vgrid_list[any_pos:any_pos] = vgrid_access # Remove any additional any_vgrid keywords while any_vgrid in vgrid_list: vgrid_list.remove(any_vgrid) except ValueError: # No any_vgrid keywords in list - move along pass # Now validate supplied vgrids for vgrid_name in vgrid_list: if not vgrid_name in vgrid_access: return (False, """Failure: You must be an owner or member of the '%s' vgrid to submit a job to it!""" % vgrid_name) # Fall back to default vgrid if no vgrid was supplied if not vgrid_list: # Please note that vgrid_list is a ref to global_dict list # so we must modify and not replace with a new list! vgrid_list.append(default_vgrid) # convert specified runtime environments to upper-case and verify they # actually exist # do not check runtime envs if the job is for ARC (submission will # fail later) if global_dict.get('JOBTYPE', 'unset') != 'arc' \ and global_dict.has_key('RUNTIMEENVIRONMENT'): re_entries_uppercase = [] for specified_re in global_dict['RUNTIMEENVIRONMENT']: specified_re = specified_re.upper() re_entries_uppercase.append(specified_re) if not is_runtime_environment(specified_re, configuration): return (False, """You have specified a non-nexisting runtime environment '%s', therefore the job can not be run on any resources.""" % \ specified_re) if global_dict.get('MOUNT', []) != []: if configuration.res_default_mount_re.upper()\ not in re_entries_uppercase: re_entries_uppercase.append( configuration.res_default_mount_re.upper()) global_dict['RUNTIMEENVIRONMENT'] = re_entries_uppercase if global_dict.get('JOBTYPE', 'unset').lower() == 'interactive': # if jobtype is interactive append command to create the notification # file .interactivejobfinished that breaks the infinite loop waiting # for the interactive job to finish and send output files to the MiG # server global_dict['EXECUTE'].append('touch .interactivejobfinished') # put job id and name of user in the dictionary global_dict['JOB_ID'] = job_id global_dict['USER_CERT'] = client_id # mark job as received global_dict['RECEIVED_TIMESTAMP'] = time.gmtime() global_dict['STATUS'] = 'PARSE' if forceddestination: global_dict['FORCEDDESTINATION'] = forceddestination if forceddestination.has_key('UNIQUE_RESOURCE_NAME'): global_dict["RESOURCE"] = "%(UNIQUE_RESOURCE_NAME)s_*" % \ forceddestination if forceddestination.has_key('RE_NAME'): re_name = forceddestination['RE_NAME'] # verify the verifyfiles entries are not modified (otherwise RE creator # can specify multiple ::VERIFYFILES:: keywords and give the entries # other names (perhaps overwriting files in the home directories of # resource owners executing the testprocedure) for verifyfile in global_dict['VERIFYFILES']: verifytypes = ['.status', '.stderr', '.stdout'] found = False for verifytype in verifytypes: if verifyfile == 'verify_runtime_env_%s%s' % (re_name, verifytype): found = True if not found: return (False, '''You are not allowed to specify the ::VERIFY:: keyword in a testprocedure, it is done automatically''') # normalize any path fields to be taken relative to home for field in ('INPUTFILES', 'OUTPUTFILES', 'EXECUTABLES', 'VERIFYFILES'): if not global_dict.has_key(field): continue normalized_field = [] for line in global_dict[field]: normalized_parts = [] line_parts = line.split(src_dst_sep) if len(line_parts) < 1 or len(line_parts) > 2: return (False, '%s entries must contain 1 or 2 space-separated items'\ % field) for part in line_parts: # deny leading slashes i.e. force absolute to relative paths part = part.lstrip('/') if part.find('://') != -1: # keep external targets as is - normpath breaks '://' normalized_parts.append(part) check_path = part.split('/')[-1] else: # normalize path to avoid e.g. './' which breaks dir # handling on resource check_path = os.path.normpath(part) normalized_parts.append(check_path) try: valid_path(check_path) except Exception, exc: return (False, 'Invalid %s part in %s: %s' % \ (field, html_escape(part), exc)) normalized_field.append(' '.join(normalized_parts)) global_dict[field] = normalized_field
def run(configuration, localfile_spaces, unique_resource_name, outfile='AUTOMATIC'): """Parse configuration in localfile_spaces and write results to outfile if non-empty. The keyword AUTOMATIC is replaced by the expected resource configuration path. """ if not configuration: configuration = get_configuration_object() (status, msg, conf) = get_resource_config_dict(configuration, localfile_spaces) if not status: return (False, msg) # verify runtime environments are specified correctly if conf.has_key('RUNTIMEENVIRONMENT'): for re in conf['RUNTIMEENVIRONMENT']: try: (name, value) = re except Exception, err: return (False, 'Runtime environment error: %s' % err) if not refunctions.is_runtime_environment(name, configuration): return ( False, "Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations." % name) (re_dict, msg) = refunctions.get_re_dict(name, configuration) if not re_dict: return (False, 'Runtime environment error, could not open (%s) %s' % (name, msg)) if not re_dict.has_key('ENVIRONMENTVARIABLE'): if value: # res conf has envs, but according to the template it should not return ( False, "%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>" % (re, value, name)) else: continue re_dict_environments = re_dict['ENVIRONMENTVARIABLE'] re_dict_environment_names = [] for re_environment in re_dict_environments: re_dict_environment_names.append(re_environment['name']) if not len(value) == len(re_dict_environments): return ( False, "You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>" % (len(value), name, len(re_dict_environments), name)) # we now know that the number of environments are # correct, verify that there are no name duplicates used_envnames = [] for env in value: try: (envname, _) = env if envname in used_envnames: # same envname used twice return ( False, "You have specified the environment '%s' more than once for the '%s' runtime environment." % (envname, name)) used_envnames.append(envname) except Exception, err: return ( False, 'Runtimeenvironment error: Name and value not found in env: %s' % err) # verify environment names are correct according to the # runtime environment definition do this by comparing # list of names specified for runtime environment and # res. conf. # re_dict_environment_names and used_envnames should # have the same entries! for n in re_dict_environment_names: # any build-in list comparison functionality? if not n in used_envnames: return ( False, "You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>" % (n, name, name))
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] output_objects.append({ 'object_type': 'header', 'text': 'Create runtime environment' }) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) re_template = accepted['re_template'][-1].upper().strip() software_entries = int(accepted['software_entries'][-1]) environment_entries = int(accepted['environment_entries'][-1]) testprocedure_entry = int(accepted['testprocedure_entry'][-1]) template = {} if re_template: if not is_runtime_environment(re_template, configuration): output_objects.append({ 'object_type': 'error_text', 'text': "re_template ('%s') is not a valid existing runtime env!" % re_template }) return (output_objects, returnvalues.CLIENT_ERROR) (template, msg) = get_re_dict(re_template, configuration) if not template: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not read re_template %s. %s' % (re_template, msg) }) return (output_objects, returnvalues.SYSTEM_ERROR) # Override template fields if user loaded a template and modified the # required entries and chose update. # Use default of 1 sw, 1 env and 0 test or template setting otherwise if software_entries < 0: software_entries = len(template.get('SOFTWARE', [None])) if environment_entries < 0: environment_entries = len(template.get('ENVIRONMENTVARIABLE', [None])) if testprocedure_entry < 0: testprocedure_entry = len(template.get('TESTPROCEDURE', [])) if template.has_key('SOFTWARE'): new_sw = template['SOFTWARE'][:software_entries] template['SOFTWARE'] = new_sw if template.has_key('ENVIRONMENTVARIABLE'): new_env = template['ENVIRONMENTVARIABLE'][:environment_entries] template['ENVIRONMENTVARIABLE'] = new_env if template.has_key('TESTPROCEDURE'): new_test = template['TESTPROCEDURE'][:testprocedure_entry] template['TESTPROCEDURE'] = new_test # Avoid DoS, limit number of software_entries if software_entries > max_software_entries: output_objects.append( {'object_type': 'error_text', 'text' : 'Maximum number of software_entries %s exceeded (%s)' % \ (max_software_entries, software_entries)}) return (output_objects, returnvalues.CLIENT_ERROR) # Avoid DoS, limit number of environment_entries if environment_entries > max_environment_entries: output_objects.append( {'object_type': 'error_text', 'text' : 'Maximum number of environment_entries %s exceeded (%s)' % \ (max_environment_entries, environment_entries)}) return (output_objects, returnvalues.CLIENT_ERROR) rekeywords_dict = get_keywords_dict() (list_status, ret) = list_runtime_environments(configuration) if not list_status: output_objects.append({'object_type': 'error_text', 'text': ret}) return (output_objects, returnvalues.SYSTEM_ERROR) output_objects.append({ 'object_type': 'text', 'text': 'Use existing Runtime Environment as template' }) html_form = \ """<form method='get' action='adminre.py'> <select name='re_template'> <option value=''>None</option> """ for existing_re in ret: html_form += " <option value='%s'>%s</option>\n" % \ (existing_re, existing_re) html_form += """ </select> <input type='submit' value='Get' /> </form>""" output_objects.append({'object_type': 'html_form', 'text': html_form}) output_objects.append({ 'object_type': 'text', 'text': '''Note that a runtime environment can not be changed after creation and it can only be removed if not in use by any resources, so please be careful when filling in the details''' }) output_objects.append({ 'object_type': 'text', 'text': '''Changing the number of software and environment entries removes all data in the form, so please enter the correct values before entering any information.''' }) html_form = \ """<form method='get' action='adminre.py'> <table> """ html_form += """ <tr> <td>Number of needed software entries</td> <td><input type='number' name='software_entries' min=0 max=99 minlength=1 maxlength=2 value='%s' required pattern='[0-9]{1,2}' title='number of software entries needed in runtime environment' /></td> </tr>""" % software_entries html_form += """ <tr> <td>Number of environment entries</td> <td> <input type='number' name='environment_entries' min=0 max=99 minlength=1 maxlength=2 value='%s' required pattern='[0-9]{1,2}' title='number of environment variables provided by runtime environment' /> </td> </tr>""" % environment_entries output_objects.append({'object_type': 'html_form', 'text': html_form}) if testprocedure_entry == 0: select_string = """<option value='0' selected>No</option> <option value=1>Yes</option>""" elif testprocedure_entry == 1: select_string = """<option value='0'>No</option> <option value='1' selected>Yes</option>""" else: output_objects.append({ 'object_type': 'error_text', 'text': 'testprocedure_entry should be 0 or 1, you specified %s' % testprocedure_entry }) return (output_objects, returnvalues.CLIENT_ERROR) html_form = """ <tr> <td>Runtime environment has a testprocedure</td> <td><select name='testprocedure_entry'>%s</select></td> </tr> <tr> <td colspan=2> <input type='hidden' name='re_template' value='%s' /> <input type='submit' value='Update fields' /> </td> </tr> </table> </form><br /> """ % (select_string, re_template) form_method = 'post' csrf_limit = get_csrf_limit(configuration) fill_helpers = { 'short_title': configuration.short_title, 'form_method': form_method, 'csrf_field': csrf_field, 'csrf_limit': csrf_limit } target_op = 'createre' csrf_token = make_csrf_token(configuration, form_method, target_op, client_id, csrf_limit) fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token}) html_form += """ <form method='%(form_method)s' action='%(target_op)s.py'> <input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' /> <b>Runtime Environment Name</b><br /> <small>(eg. BASH-2.X-1, must be unique):</small><br /> <input class='p80width' type='text' name='re_name' required pattern='[a-zA-Z0-9_.-]+' title='unique name of ASCII letters and digits separated only by underscores, periods and hyphens' /> <br /> <br /><b>Description:</b><br /> <textarea class='p80width' rows='4' name='redescription'> """ if template: html_form += template['DESCRIPTION'].replace('<br />', '\n') html_form += '</textarea><br />' soft_list = [] if software_entries > 0: html_form += '<br /><b>Needed Software:</b><br />' if template: if template.has_key('SOFTWARE'): soft_list = template['SOFTWARE'] for soft in soft_list: html_form += """ <textarea class='p80width' rows='6' name='software'>""" for keyname in soft.keys(): if keyname != '': html_form += '%s=%s\n' % (keyname, soft[keyname]) html_form += '</textarea><br />' # loop and create textareas for any missing software entries software = rekeywords_dict['SOFTWARE'] sublevel_required = [] sublevel_optional = [] if software.has_key('Sublevel') and software['Sublevel']: sublevel_required = software['Sublevel_required'] sublevel_optional = software['Sublevel_optional'] for _ in range(len(soft_list), software_entries): html_form += """ <textarea class='p80width' rows='6' name='software'>""" for sub_req in sublevel_required: html_form += '%s= # required\n' % sub_req for sub_opt in sublevel_optional: html_form += '%s= # optional\n' % sub_opt html_form += '</textarea><br />' if template and testprocedure_entry == 1: if template.has_key('TESTPROCEDURE'): html_form += """ <br /><b>Testprocedure</b> (in mRSL format):<br /> <textarea class='p80width' rows='15' name='testprocedure'>""" base64string = '' for stringpart in template['TESTPROCEDURE']: base64string += stringpart decodedstring = base64.decodestring(base64string) html_form += decodedstring html_form += '</textarea>' output_objects.append({ 'object_type': 'html_form', 'text': html_form }) html_form = """ <br /><b>Expected .stdout file if testprocedure is executed</b><br /> <textarea class='p80width' rows='10' name='verifystdout'>""" if template.has_key('VERIFYSTDOUT'): for line in template['VERIFYSTDOUT']: html_form += line html_form += '</textarea>' html_form += """ <br /><b>Expected .stderr file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystderr'>""" if template.has_key('VERIFYSTDERR'): for line in template['VERIFYSTDERR']: html_form += line html_form += '</textarea>' html_form += """ <br /><b>Expected .status file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystatus'>""" if template.has_key('VERIFYSTATUS'): for line in template['VERIFYSTATUS']: html_form += line html_form += '</textarea>' elif testprocedure_entry == 1: html_form += """ <br /><b>Testprocedure</b> (in mRSL format):<br /> <textarea class='p80width' rows='15' name='testprocedure'>""" html_form += \ """::EXECUTE:: ls </textarea> <br /><b>Expected .stdout file if testprocedure is executed</b><br /> <textarea class='p80width' rows='10' name='verifystdout'></textarea> <br /><b>Expected .stderr file if testprocedure is executed</b><br /> <textarea class='p80width' rows='10' name='verifystderr'></textarea> <br /><b>Expected .status file if testprocedure is executed</b><br /> <textarea class='p80width' rows='10' name='verifystatus'></textarea> """ environmentvariable = rekeywords_dict['ENVIRONMENTVARIABLE'] sublevel_required = [] sublevel_optional = [] if environmentvariable.has_key('Sublevel')\ and environmentvariable['Sublevel']: sublevel_required = environmentvariable['Sublevel_required'] sublevel_optional = environmentvariable['Sublevel_optional'] env_list = [] if environment_entries > 0: html_form += '<br /><b>Environments:</b><br />' if template: if template.has_key('ENVIRONMENTVARIABLE'): env_list = template['ENVIRONMENTVARIABLE'] for env in env_list: html_form += """ <textarea class='p80width' rows='4' name='environment'>""" for keyname in env.keys(): if keyname != '': html_form += '%s=%s\n' % (keyname, env[keyname]) html_form += '</textarea><br />' # loop and create textareas for any missing environment entries for _ in range(len(env_list), environment_entries): html_form += """ <textarea class='p80width' rows='4' name='environment'>""" for sub_req in sublevel_required: html_form += '%s= # required\n' % sub_req for sub_opt in sublevel_optional: html_form += '%s= # optional\n' % sub_opt html_form += '</textarea><br />' html_form += """<br /><br /><input type='submit' value='Create' /> </form> """ output_objects.append({ 'object_type': 'html_form', 'text': html_form % fill_helpers }) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Delete runtime environment' output_objects.append({'object_type': 'header', 'text' : 'Delete runtime environment'}) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) re_name = accepted['re_name'][-1] if not valid_dir_input(configuration.re_home, re_name): logger.warning( "possible illegal directory traversal attempt re_name '%s'" % re_name) output_objects.append({'object_type': 'error_text', 'text' : 'Illegal runtime environment name: "%s"' % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) # Check whether re_name represents a runtime environment if not is_runtime_environment(re_name, configuration): output_objects.append({'object_type': 'error_text', 'text': "No such runtime environment: '%s'" % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) re_dict = get_re_dict(re_name, configuration) if not re_dict[0]: output_objects.append( {'object_type': 'error_text', 'text': 'Could not read runtime environment details for %s' % re_name}) return (output_objects, returnvalues.SYSTEM_ERROR) # Make sure the runtime environment belongs to the user trying to delete it if client_id != re_dict[0]['CREATOR']: output_objects.append({'object_type': 'error_text', 'text': \ 'You are not the owner of runtime environment "%s"' % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) # Prevent delete if the runtime environment is used by any resources actives = resources_using_re(configuration, re_name) # If the runtime environment is active, an error message is printed, along # with a list of the resources using the runtime environment if actives: output_objects.append( {'object_type': 'error_text', 'text': "Can't delete runtime environment '%s' in use by resources:" % re_name}) output_objects.append({'object_type': 'list', 'list' : actives}) output_objects.append({'object_type': 'link', 'destination': 'redb.py', 'class': 'infolink', 'title': 'Show runtime environments', 'text': 'Show runtime environments'}) return (output_objects, returnvalues.CLIENT_ERROR) # Delete the runtime environment (status, msg) = delete_runtimeenv(re_name, configuration) # If something goes wrong when trying to delete runtime environment # re_name, an error is displayed. if not status: output_objects.append({'object_type': 'error_text', 'text' : 'Could not remove %s runtime environment: %s' % (re_name, msg)}) return (output_objects, returnvalues.SYSTEM_ERROR) # If deletion of runtime environment re_name is successful, we just # return OK else: output_objects.append( {'object_type': 'text', 'text' : 'Successfully deleted runtime environment: "%s"' % re_name}) output_objects.append({'object_type': 'link', 'destination': 'redb.py', 'class': 'infolink', 'title': 'Show runtime environments', 'text': 'Show runtime environments'}) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False) defaults = signature()[1] output_objects.append({"object_type": "header", "text": "Create runtime environment"}) (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) re_template = accepted["re_template"][-1].upper().strip() software_entries = int(accepted["software_entries"][-1]) environment_entries = int(accepted["environment_entries"][-1]) testprocedure_entry = int(accepted["testprocedure_entry"][-1]) template = {} if re_template: if not is_runtime_environment(re_template, configuration): output_objects.append( { "object_type": "error_text", "text": "re_template ('%s') is not a valid existing runtime env!" % re_template, } ) return (output_objects, returnvalues.CLIENT_ERROR) (template, msg) = get_re_dict(re_template, configuration) if not template: output_objects.append( {"object_type": "error_text", "text": "Could not read re_template %s. %s" % (re_template, msg)} ) return (output_objects, returnvalues.SYSTEM_ERROR) # Override template fields if user loaded a template and modified the # required entries and chose update. # Use default of 1 sw, 1 env and 0 test or template setting otherwise if software_entries < 0: software_entries = len(template.get("SOFTWARE", [None])) if environment_entries < 0: environment_entries = len(template.get("ENVIRONMENTVARIABLE", [None])) if testprocedure_entry < 0: testprocedure_entry = len(template.get("TESTPROCEDURE", [])) if template.has_key("SOFTWARE"): new_sw = template["SOFTWARE"][:software_entries] template["SOFTWARE"] = new_sw if template.has_key("ENVIRONMENTVARIABLE"): new_env = template["ENVIRONMENTVARIABLE"][:environment_entries] template["ENVIRONMENTVARIABLE"] = new_env if template.has_key("TESTPROCEDURE"): new_test = template["TESTPROCEDURE"][:testprocedure_entry] template["TESTPROCEDURE"] = new_test # Avoid DoS, limit number of software_entries if software_entries > max_software_entries: output_objects.append( { "object_type": "error_text", "text": "Maximum number of software_entries %s exceeded (%s)" % (max_software_entries, software_entries), } ) return (output_objects, returnvalues.CLIENT_ERROR) # Avoid DoS, limit number of environment_entries if environment_entries > max_environment_entries: output_objects.append( { "object_type": "error_text", "text": "Maximum number of environment_entries %s exceeded (%s)" % (max_environment_entries, environment_entries), } ) return (output_objects, returnvalues.CLIENT_ERROR) rekeywords_dict = get_keywords_dict() (status, ret) = list_runtime_environments(configuration) if not status: output_objects.append({"object_type": "error_text", "text": ret}) return (output_objects, returnvalues.SYSTEM_ERROR) output_objects.append({"object_type": "text", "text": "Use existing RE as template"}) html_form = """<form method='get' action='adminre.py'> <select name='re_template'> <option value=''>None</option> """ for existing_re in ret: html_form += " <option value='%s'>%s</option>\n" % (existing_re, existing_re) html_form += """ </select> <input type='submit' value='Get' /> </form>""" output_objects.append({"object_type": "html_form", "text": html_form}) output_objects.append( { "object_type": "text", "text": """Note that a runtime environment can not be changed after creation and it can only be removed if not in use by any resources, so please be careful when filling in the details""", } ) output_objects.append( { "object_type": "text", "text": """Changing the number of software and environment entries removes all data in the form, so please enter the correct values before entering any information.""", } ) html_form = """<form method='get' action='adminre.py'> <table> """ html_form += ( """ <tr> <td>Number of needed software entries</td> <td><input type='text' size='2' name='software_entries' value='%s' /></td> </tr>""" % software_entries ) html_form += ( """ <tr> <td>Number of environment entries</td> <td> <input type='text' size='2' name='environment_entries' value='%s' /> </td> </tr>""" % environment_entries ) output_objects.append({"object_type": "html_form", "text": html_form}) if testprocedure_entry == 0: select_string = """<option value='0' selected>No</option> <option value=1>Yes</option>""" elif testprocedure_entry == 1: select_string = """<option value='0'>No</option> <option value='1' selected>Yes</option>""" else: output_objects.append( { "object_type": "error_text", "text": "testprocedure_entry should be 0 or 1, you specified %s" % testprocedure_entry, } ) return (output_objects, returnvalues.CLIENT_ERROR) html_form = """ <tr> <td>Runtime environment has a testprocedure</td> <td><select name='testprocedure_entry'>%s</select></td> </tr> <tr> <td colspan=2> <input type='hidden' name='re_template' value='%s' /> <input type='submit' value='Update fields' /> </td> </tr> </table> </form><br /> """ % ( select_string, re_template, ) html_form += """ <form method='post' action='createre.py'> <b>RE Name</b><br /> <small>(eg. BASH-2.X-1, must be unique):</small><br /> <input type='text' size='40' name='re_name' /><br /> <br /><b>Description:</b><br /> <textarea cols='50' rows='2' name='redescription'> """ if template: html_form += template["DESCRIPTION"].replace("<br />", "\n") html_form += "</textarea><br />" soft_list = [] if software_entries > 0: html_form += "<br /><b>Needed Software:</b><br />" if template: if template.has_key("SOFTWARE"): soft_list = template["SOFTWARE"] for soft in soft_list: html_form += """ <textarea cols='50' rows='5' name='software'>""" for keyname in soft.keys(): if keyname != "": html_form += "%s=%s\n" % (keyname, soft[keyname]) html_form += "</textarea><br />" # loop and create textareas for any missing software entries software = rekeywords_dict["SOFTWARE"] sublevel_required = [] sublevel_optional = [] if software.has_key("Sublevel") and software["Sublevel"]: sublevel_required = software["Sublevel_required"] sublevel_optional = software["Sublevel_optional"] for _ in range(len(soft_list), software_entries): html_form += """ <textarea cols='50' rows='5' name='software'>""" for sub_req in sublevel_required: html_form += "%s= # required\n" % sub_req for sub_opt in sublevel_optional: html_form += "%s= # optional\n" % sub_opt html_form += "</textarea><br />" if template and testprocedure_entry == 1: if template.has_key("TESTPROCEDURE"): html_form += """ <br /><b>Testprocedure</b> (in mRSL format):<br /> <textarea cols='50' rows='15' name='testprocedure'>""" base64string = "" for stringpart in template["TESTPROCEDURE"]: base64string += stringpart decodedstring = base64.decodestring(base64string) html_form += decodedstring html_form += "</textarea>" output_objects.append({"object_type": "html_form", "text": html_form}) html_form = """ <br /><b>Expected .stdout file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystdout'>""" if template.has_key("VERIFYSTDOUT"): for line in template["VERIFYSTDOUT"]: html_form += line html_form += "</textarea>" html_form += """ <br /><b>Expected .stderr file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystderr'>""" if template.has_key("VERIFYSTDERR"): for line in template["VERIFYSTDERR"]: html_form += line html_form += "</textarea>" html_form += """ <br /><b>Expected .status file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystatus'>""" if template.has_key("VERIFYSTATUS"): for line in template["VERIFYSTATUS"]: html_form += line html_form += "</textarea>" elif testprocedure_entry == 1: html_form += """ <br /><b>Testprocedure</b> (in mRSL format):<br /> <textarea cols='50' rows='15' name='testprocedure'>""" html_form += """::EXECUTE:: ls </textarea> <br /><b>Expected .stdout file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystdout'></textarea> <br /><b>Expected .stderr file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystderr'></textarea> <br /><b>Expected .status file if testprocedure is executed</b><br /> <textarea cols='50' rows='10' name='verifystatus'></textarea> """ environmentvariable = rekeywords_dict["ENVIRONMENTVARIABLE"] sublevel_required = [] sublevel_optional = [] if environmentvariable.has_key("Sublevel") and environmentvariable["Sublevel"]: sublevel_required = environmentvariable["Sublevel_required"] sublevel_optional = environmentvariable["Sublevel_optional"] env_list = [] if environment_entries > 0: html_form += "<br /><b>Environments:</b><br />" if template: if template.has_key("ENVIRONMENTVARIABLE"): env_list = template["ENVIRONMENTVARIABLE"] for env in env_list: html_form += """ <textarea cols='50' rows='3' name='environment'>""" for keyname in env.keys(): if keyname != "": html_form += "%s=%s\n" % (keyname, env[keyname]) html_form += "</textarea><br />" # loop and create textareas for any missing environment entries for _ in range(len(env_list), environment_entries): html_form += """ <textarea cols='50' rows='3' name='environment'>""" for sub_req in sublevel_required: html_form += "%s= # required\n" % sub_req for sub_opt in sublevel_optional: html_form += "%s= # optional\n" % sub_opt html_form += "</textarea><br />" html_form += """<br /><br /><input type='submit' value='Create' /> </form> """ output_objects.append({"object_type": "html_form", "text": html_form}) return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) defaults = signature()[1] title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Show Runtime Environment Details' (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) re_name = accepted['re_name'][-1] output_objects.append({ 'object_type': 'header', 'text': 'Show runtime environment details' }) if not valid_dir_input(configuration.re_home, re_name): logger.warning( "possible illegal directory traversal attempt re_name '%s'" % re_name) output_objects.append({ 'object_type': 'error_text', 'text': 'Illegal runtime environment name: "%s"' % re_name }) return (output_objects, returnvalues.CLIENT_ERROR) if not is_runtime_environment(re_name, configuration): output_objects.append({ 'object_type': 'error_text', 'text': "'%s' is not an existing runtime environment!" % re_name }) return (output_objects, returnvalues.CLIENT_ERROR) title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Runtime environment details' (re_dict, msg) = get_re_dict(re_name, configuration) if not re_dict: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not read details for "%s"' % msg }) return (output_objects, returnvalues.SYSTEM_ERROR) # Set providers explicitly after build_reitem_object to avoid import loop re_item = build_reitem_object(configuration, re_dict) re_name = re_item['name'] re_item['providers'] = resources_using_re(configuration, re_name) re_item['resource_count'] = len(re_item['providers']) output_objects.append(re_item) return (output_objects, returnvalues.OK)
def parse( localfile_spaces, job_id, client_id, forceddestination, outfile='AUTOMATIC', ): """Parse job description and optionally write results to parsed mRSL file. If outfile is non-empty it is used as destination file, and the keyword AUTOMATIC is replaced by the default mrsl dir destination. """ configuration = get_configuration_object() logger = configuration.logger client_dir = client_id_dir(client_id) # return a tuple (bool status, str msg). This is done because cgi-scripts # are not allowed to print anything before 'the first two special lines' # are printed result = parser.parse(localfile_spaces) external_dict = mrslkeywords.get_keywords_dict(configuration) # The mRSL has the right structure check if the types are correct too # and inline update the default external_dict entries with the ones # from the actual job specification (status, msg) = parser.check_types(result, external_dict, configuration) if not status: return (False, 'Parse failed (typecheck) %s' % msg) logger.debug('check_types updated job dict to: %s' % external_dict) global_dict = {} # Insert the parts from mrslkeywords we need in the rest of the MiG system for (key, value_dict) in external_dict.iteritems(): global_dict[key] = value_dict['Value'] # We do not expand any job variables yet in order to allow any future # resubmits to properly expand job ID. vgrid_list = global_dict['VGRID'] allowed_vgrids = user_allowed_vgrids(configuration, client_id) # Replace any_vgrid keyword with all allowed vgrids (on time of submit!) try: any_pos = vgrid_list.index(any_vgrid) vgrid_list[any_pos:any_pos] = allowed_vgrids # Remove any additional any_vgrid keywords while any_vgrid in vgrid_list: vgrid_list.remove(any_vgrid) except ValueError: # No any_vgrid keywords in list - move along pass # Now validate supplied vgrids for vgrid_name in vgrid_list: if not vgrid_name in allowed_vgrids: return (False, """Failure: You must be an owner or member of the '%s' vgrid to submit a job to it!""" % vgrid_name) # Fall back to default vgrid if no vgrid was supplied if not vgrid_list: # Please note that vgrid_list is a ref to global_dict list # so we must modify and not replace with a new list! vgrid_list.append(default_vgrid) # convert specified runtime environments to upper-case and verify they # actually exist # do not check runtime envs if the job is for ARC (submission will # fail later) if global_dict.get('JOBTYPE', 'unset') != 'arc' \ and global_dict.has_key('RUNTIMEENVIRONMENT'): re_entries_uppercase = [] for specified_re in global_dict['RUNTIMEENVIRONMENT']: specified_re = specified_re.upper() re_entries_uppercase.append(specified_re) if not is_runtime_environment(specified_re, configuration): return (False, """You have specified a non-nexisting runtime environment '%s', therefore the job can not be run on any resources.""" % \ specified_re) if global_dict.get('MOUNT', []) != []: re_entries_uppercase.append(configuration.res_default_mount_re.upper()) global_dict['RUNTIMEENVIRONMENT'] = re_entries_uppercase if global_dict.get('JOBTYPE', 'unset').lower() == 'interactive': # if jobtype is interactive append command to create the notification # file .interactivejobfinished that breaks the infinite loop waiting # for the interactive job to finish and send output files to the MiG # server global_dict['EXECUTE'].append('touch .interactivejobfinished') # put job id and name of user in the dictionary global_dict['JOB_ID'] = job_id global_dict['USER_CERT'] = client_id # mark job as received global_dict['RECEIVED_TIMESTAMP'] = time.gmtime() global_dict['STATUS'] = 'PARSE' if forceddestination: global_dict['FORCEDDESTINATION'] = forceddestination if forceddestination.has_key('UNIQUE_RESOURCE_NAME'): global_dict["RESOURCE"] = "%(UNIQUE_RESOURCE_NAME)s_*" % \ forceddestination if forceddestination.has_key('RE_NAME'): re_name = forceddestination['RE_NAME'] # verify the verifyfiles entries are not modified (otherwise RE creator # can specify multiple ::VERIFYFILES:: keywords and give the entries # other names (perhaps overwriting files in the home directories of # resource owners executing the testprocedure) for verifyfile in global_dict['VERIFYFILES']: verifytypes = ['.status', '.stderr', '.stdout'] found = False for verifytype in verifytypes: if verifyfile == 'verify_runtime_env_%s%s' % (re_name, verifytype): found = True if not found: return (False, '''You are not allowed to specify the ::VERIFY:: keyword in a testprocedure, it is done automatically''') # normalize any path fields to be taken relative to home for field in ('INPUTFILES', 'OUTPUTFILES', 'EXECUTABLES', 'VERIFYFILES'): if not global_dict.has_key(field): continue normalized_field = [] for line in global_dict[field]: normalized_parts = [] line_parts = line.split() if len(line_parts) < 1 or len(line_parts) > 2: return (False, '%s entries must contain 1 or 2 space-separated items'\ % field) for part in line_parts: # deny leading slashes i.e. force absolute to relative paths part = part.lstrip('/') if part.find('://') != -1: # keep external targets as is - normpath breaks '://' normalized_parts.append(part) check_path = part.split('/')[-1] else: # normalize path to avoid e.g. './' which breaks dir # handling on resource check_path = os.path.normpath(part) normalized_parts.append(check_path) try: valid_path(check_path) except Exception, exc: return (False, 'Invalid %s part in %s: %s' % \ (field, html_escape(part), exc)) normalized_field.append(' '.join(normalized_parts)) global_dict[field] = normalized_field
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_header=False) title_entry = find_entry(output_objects, 'title') title_entry['text'] = 'Delete runtime environment' output_objects.append({ 'object_type': 'header', 'text': 'Delete runtime environment' }) defaults = signature()[1] (validate_status, accepted) = validate_input_and_cert( user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) re_name = accepted['re_name'][-1] if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) if not valid_dir_input(configuration.re_home, re_name): logger.warning( "possible illegal directory traversal attempt re_name '%s'" % re_name) output_objects.append({ 'object_type': 'error_text', 'text': 'Illegal runtime environment name: "%s"' % re_name }) return (output_objects, returnvalues.CLIENT_ERROR) # Check whether re_name represents a runtime environment if not is_runtime_environment(re_name, configuration): output_objects.append({ 'object_type': 'error_text', 'text': "No such runtime environment: '%s'" % re_name }) return (output_objects, returnvalues.CLIENT_ERROR) (re_dict, load_msg) = get_re_dict(re_name, configuration) if not re_dict: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not read runtime environment details for %s: %s' % (re_name, load_msg) }) return (output_objects, returnvalues.SYSTEM_ERROR) # Make sure the runtime environment belongs to the user trying to delete it if client_id != re_dict['CREATOR']: output_objects.append({'object_type': 'error_text', 'text': \ 'You are not the owner of runtime environment "%s"' % re_name}) return (output_objects, returnvalues.CLIENT_ERROR) # Prevent delete if the runtime environment is used by any resources actives = resources_using_re(configuration, re_name) # If the runtime environment is active, an error message is printed, along # with a list of the resources using the runtime environment if actives: output_objects.append({ 'object_type': 'error_text', 'text': "Can't delete runtime environment '%s' in use by resources:" % re_name }) output_objects.append({'object_type': 'list', 'list': actives}) output_objects.append({ 'object_type': 'link', 'destination': 'redb.py', 'class': 'infolink iconspace', 'title': 'Show runtime environments', 'text': 'Show runtime environments' }) return (output_objects, returnvalues.CLIENT_ERROR) # Delete the runtime environment (del_status, msg) = delete_runtimeenv(re_name, configuration) # If something goes wrong when trying to delete runtime environment # re_name, an error is displayed. if not del_status: output_objects.append({ 'object_type': 'error_text', 'text': 'Could not remove %s runtime environment: %s' % (re_name, msg) }) return (output_objects, returnvalues.SYSTEM_ERROR) # If deletion of runtime environment re_name is successful, we just # return OK else: output_objects.append({ 'object_type': 'text', 'text': 'Successfully deleted runtime environment: "%s"' % re_name }) output_objects.append({ 'object_type': 'link', 'destination': 'redb.py', 'class': 'infolink iconspace', 'title': 'Show runtime environments', 'text': 'Show runtime environments' }) return (output_objects, returnvalues.OK)
def run(localfile_spaces, unique_resource_name, outfile='AUTOMATIC' ): """Parse configuration in localfile_spaces and write results to outfile if non-empty. The keyword AUTOMATIC is replaced by the expected resource configuration path. """ (status, msg, conf) = get_resource_config_dict(localfile_spaces) if not status: return (False, msg) # verify runtime environments are specified correctly if conf.has_key('RUNTIMEENVIRONMENT'): for re in conf['RUNTIMEENVIRONMENT']: try: (name, value) = re except Exception, err: return (False, 'Runtime environment error: %s' % err) if not refunctions.is_runtime_environment(name, configuration): return (False, "Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations." % name) (re_dict, msg) = refunctions.get_re_dict(name, configuration) if not re_dict: return (False, 'Runtime environment error, could not open (%s) %s' % (name, msg)) if not re_dict.has_key('ENVIRONMENTVARIABLE'): if value: # res conf has envs, but according to the template it should not return (False, "%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>" % (re, value, name)) else: continue re_dict_environments = re_dict['ENVIRONMENTVARIABLE'] re_dict_environment_names = [] for re_environment in re_dict_environments: re_dict_environment_names.append(re_environment['name']) if not len(value) == len(re_dict_environments): return (False, "You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>" % (len(value), name, len(re_dict_environments), name)) # we now know that the number of environments are # correct, verify that there are no name duplicates used_envnames = [] for env in value: try: (envname, _) = env if envname in used_envnames: # same envname used twice return (False, "You have specified the environment '%s' more than once for the '%s' runtime environment." % (envname, name)) used_envnames.append(envname) except Exception, err: return (False, 'Runtimeenvironment error: Name and value not found in env: %s' % err) # verify environment names are correct according to the # runtime environment definition do this by comparing # list of names specified for runtime environment and # res. conf. # re_dict_environment_names and used_envnames should # have the same entries! for n in re_dict_environment_names: # any build-in list comparison functionality? if not n in used_envnames: return (False, "You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>" % (n, name, name))