コード例 #1
0
ファイル: vgridaccess.py プロジェクト: heromod/migrid
def user_allowed_user_confs(configuration, client_id):
    """Extract a map of users that client_id can really view and maybe
    contact.

    Returns a map from user IDs to lists of user confs.

    User IDs are anonymized unless explicitly configured otherwise.
    """
    allowed = {}
    allowed_vgrids = user_allowed_vgrids(configuration, client_id)

    # Find all potential users from vgrid member and ownership

    user_map = get_user_map(configuration)

    # Map only contains the raw user names - anonymize as requested

    anon_map = {}
    for user in user_map.keys():
        anon_map[user] = user_map[user][USERID]

    # Now select only the ones that actually still are allowed for that vgrid

    for (user, conf) in user_map.items():
        allowed[anon_map[user]] = conf
    return allowed
コード例 #2
0
ファイル: jobfeasibility.py プロジェクト: heromod/migrid
def validate_vgrid(configuration, job, errors):
    """Returns a list of user specified vgrids that are allowed."""

    allowed_vgrids = set(user_allowed_vgrids(configuration, job['USER_CERT']))

    if skip_validation(configuration, job, 'VGRID'):

        # all allowed, possibly empty

        return list(allowed_vgrids)

    specified_vgrids = set(job['VGRID'])

    not_allowed = specified_vgrids.difference(allowed_vgrids)
    if not_allowed:
        errors['VGRID'] = \
        'The following VGrids are not allowed for the current user:, %s' % \
        (not_allowed)

    if not allowed_vgrids.intersection(specified_vgrids):

        # all allowed, possibly []

        return list(allowed_vgrids)
    else:

        #validated specifed and allowed

        return list(allowed_vgrids.intersection(specified_vgrids))
コード例 #3
0
ファイル: jobfeasibility.py プロジェクト: heromod/migrid
def complement_vgrids(configuration, job, vgrids):
    """Returns a list of allowed vgrids where those specified-and-allowed
    have been subtracted.
    """

    allowed_vgrids = set(user_allowed_vgrids(configuration, job['USER_CERT']))

    return list(allowed_vgrids.difference(vgrids))
コード例 #4
0
ファイル: vgridaccess.py プロジェクト: heromod/migrid
def user_allowed_res_exes(configuration, client_id):
    """Extract a map of resources that client_id can really submit to.
    There is no guarantee that they will ever accept any further jobs.

    Returns a map from resource IDs to lists of exe node names.

    Resource IDs are anonymized unless explicitly configured otherwise.
    
    Please note that vgrid participation is a mutual agreement between vgrid
    owners and resource owners, so that a resource only truly participates
    in a vgrid if the vgrid *and* resource owners configured it so.
    """
    allowed = {}

    # Extend allowed_vgrids with any parent vgrids here to fit inheritance

    allowed_vgrids = user_allowed_vgrids(configuration, client_id,
                                         inherited=True)

    # Find all potential resources from vgrid sign up

    vgrid_map = get_vgrid_map(configuration)
    vgrid_map_res = vgrid_map[RESOURCES]

    # Map only contains the raw resource names - anonymize as requested

    anon_map = {}
    for res in vgrid_map_res.keys():
        anon_map[res] = vgrid_map_res[res][RESID]

    # Now select only the ones that actually still are allowed for that vgrid

    for (res, all_exes) in vgrid_map_res.items():
        shared = [i for i in all_exes[ALLOW] if i in allowed_vgrids]
        if not shared:
            continue
        match = []
        for exe in [i for i in all_exes.keys() if i not in RES_SPECIALS]:
            if [i for i in shared if i in all_exes[exe]]:
                match.append(exe)
        allowed[anon_map[res]] = match
    return allowed
コード例 #5
0
ファイル: vgridaccess.py プロジェクト: heromod/migrid
def user_allowed_res_confs(configuration, client_id):
    """Extract a map of resources that client_id can really submit to.
    There is no guarantee that they will ever accept any further jobs.

    Returns a map from resource IDs to resource conf dictionaries.

    Resources are anonymized unless explicitly configured otherwise, but
    the resource confs are always raw.
    
    Please note that vgrid participation is a mutual agreement between vgrid
    owners and resource owners, so that a resource only truly participates
    in a vgrid if the vgrid *and* resource owners configured it so.
    """
    allowed = {}

    # Extend allowed_vgrids with any parent vgrids here to fit inheritance

    allowed_vgrids = user_allowed_vgrids(configuration, client_id,
                                         inherited=True)

    # Find all potential resources from vgrid sign up

    vgrid_map = get_vgrid_map(configuration)
    vgrid_map_res = vgrid_map[RESOURCES]
    resource_map = get_resource_map(configuration)

    # Map only contains the raw resource names - anonymize as requested

    anon_map = {}
    for res in vgrid_map_res.keys():
        anon_map[res] = vgrid_map_res[res][RESID]

    # Now select only the ones that actually still are allowed for that vgrid

    for (res, all_exes) in vgrid_map_res.items():
        shared = [i for i in all_exes[ALLOW] if i in allowed_vgrids]
        if not shared:
            continue
        allowed[anon_map[res]] = resource_map.get(res, {CONF: {}})[CONF]
    return allowed
コード例 #6
0
ファイル: showstats.py プロジェクト: ucphhpc/migrid
        output_objects.append({
            'object_type': 'sectionheader',
            'text': 'No data available'
        })
        output_objects.append({
            'object_type':
            'text',
            'text':
            '''
The query you have requested did not return any data.
                               '''
        })
        return (output_objects, returnvalues.OK)

    # select the vgrids we are going to consider
    my_vgrids = vgrid.user_allowed_vgrids(configuration, client_id)

    # process the received data:

    # view field names, could be configurable as well,
    # but this is anyway highly proprietary code
    table_names = {
        'count': 'Job Count',
        'wall_duration': 'Accumulated Wall Clock Time',
        'charge': 'Accumulated Charge'
    }

    lookupdict = dict([(tuple(d['key']), d['value']) for d in data])

    if group_level == 1:
        # if view not grouped in time (currently unused)
コード例 #7
0
ファイル: mrslparser.py プロジェクト: heromod/migrid
def parse(
    localfile_spaces,
    job_id,
    client_id,
    forceddestination,
    outfile='AUTOMATIC',
    ):
    """Parse job description and optionally write results to parsed mRSL file.
    If outfile is non-empty it is used as destination file, and the keyword
    AUTOMATIC is replaced by the default mrsl dir destination.
    """

    configuration = get_configuration_object()
    logger = configuration.logger
    client_dir = client_id_dir(client_id)

    # return a tuple (bool status, str msg). This is done because cgi-scripts
    # are not allowed to print anything before 'the first two special lines'
    # are printed

    result = parser.parse(localfile_spaces)

    external_dict = mrslkeywords.get_keywords_dict(configuration)

    # The mRSL has the right structure check if the types are correct too
    # and inline update the default external_dict entries with the ones
    # from the actual job specification

    (status, msg) = parser.check_types(result, external_dict,
            configuration)
    if not status:
        return (False, 'Parse failed (typecheck) %s' % msg)

    logger.debug('check_types updated job dict to: %s' % external_dict)

    global_dict = {}

    # Insert the parts from mrslkeywords we need in the rest of the MiG system

    for (key, value_dict) in external_dict.iteritems():
        global_dict[key] = value_dict['Value']

    # We do not expand any job variables yet in order to allow any future
    # resubmits to properly expand job ID.

    vgrid_list = global_dict['VGRID']
    allowed_vgrids = user_allowed_vgrids(configuration, client_id)

    # Replace any_vgrid keyword with all allowed vgrids (on time of submit!)

    try:
        any_pos = vgrid_list.index(any_vgrid)
        vgrid_list[any_pos:any_pos] = allowed_vgrids

        # Remove any additional any_vgrid keywords

        while any_vgrid in vgrid_list:
            vgrid_list.remove(any_vgrid)
    except ValueError:

        # No any_vgrid keywords in list - move along

        pass

    # Now validate supplied vgrids

    for vgrid_name in vgrid_list:
        if not vgrid_name in allowed_vgrids:
            return (False, """Failure: You must be an owner or member of the
'%s' vgrid to submit a job to it!""" % vgrid_name)

    # Fall back to default vgrid if no vgrid was supplied

    if not vgrid_list:

        # Please note that vgrid_list is a ref to global_dict list
        # so we must modify and not replace with a new list!

        vgrid_list.append(default_vgrid)

    # convert specified runtime environments to upper-case and verify they
    # actually exist

    # do not check runtime envs if the job is for ARC (submission will
    # fail later)
    if global_dict.get('JOBTYPE', 'unset') != 'arc' \
        and global_dict.has_key('RUNTIMEENVIRONMENT'):
        re_entries_uppercase = []
        for specified_re in global_dict['RUNTIMEENVIRONMENT']:
            specified_re = specified_re.upper()
            re_entries_uppercase.append(specified_re)
            if not is_runtime_environment(specified_re, configuration):
                return (False, """You have specified a non-nexisting runtime
environment '%s', therefore the job can not be run on any resources.""" % \
                        specified_re)
        if global_dict.get('MOUNT', []) != []:
            re_entries_uppercase.append(configuration.res_default_mount_re.upper())
            
        global_dict['RUNTIMEENVIRONMENT'] = re_entries_uppercase

    if global_dict.get('JOBTYPE', 'unset').lower() == 'interactive':

        # if jobtype is interactive append command to create the notification
        # file .interactivejobfinished that breaks the infinite loop waiting
        # for the interactive job to finish and send output files to the MiG
        # server

        global_dict['EXECUTE'].append('touch .interactivejobfinished')

    # put job id and name of user in the dictionary

    global_dict['JOB_ID'] = job_id
    global_dict['USER_CERT'] = client_id

    # mark job as received

    global_dict['RECEIVED_TIMESTAMP'] = time.gmtime()
    global_dict['STATUS'] = 'PARSE'
    if forceddestination:
        global_dict['FORCEDDESTINATION'] = forceddestination
        if forceddestination.has_key('UNIQUE_RESOURCE_NAME'):
            global_dict["RESOURCE"] = "%(UNIQUE_RESOURCE_NAME)s_*" % \
                                      forceddestination
        if forceddestination.has_key('RE_NAME'):
            re_name = forceddestination['RE_NAME']

            # verify the verifyfiles entries are not modified (otherwise RE creator
            # can specify multiple ::VERIFYFILES:: keywords and give the entries
            # other names (perhaps overwriting files in the home directories of
            # resource owners executing the testprocedure)

            for verifyfile in global_dict['VERIFYFILES']:
                verifytypes = ['.status', '.stderr', '.stdout']
                found = False
                for verifytype in verifytypes:
                    if verifyfile == 'verify_runtime_env_%s%s' % (re_name,
                            verifytype):
                        found = True
                if not found:
                    return (False, '''You are not allowed to specify the
::VERIFY:: keyword in a testprocedure, it is done automatically''')

    # normalize any path fields to be taken relative to home

    for field in ('INPUTFILES', 'OUTPUTFILES', 'EXECUTABLES',
                  'VERIFYFILES'):
        if not global_dict.has_key(field):
            continue
        normalized_field = []
        for line in global_dict[field]:
            normalized_parts = []
            line_parts = line.split()
            if len(line_parts) < 1 or len(line_parts) > 2:
                return (False,
                        '%s entries must contain 1 or 2 space-separated items'\
                        % field)
            for part in line_parts:

                # deny leading slashes i.e. force absolute to relative paths

                part = part.lstrip('/')
                if part.find('://') != -1:

                    # keep external targets as is - normpath breaks '://'

                    normalized_parts.append(part)
                    check_path = part.split('/')[-1]
                else:

                    # normalize path to avoid e.g. './' which breaks dir
                    # handling on resource

                    check_path = os.path.normpath(part)
                    normalized_parts.append(check_path)
                try:
                    valid_path(check_path)
                except Exception, exc:
                    return (False, 'Invalid %s part in %s: %s' % \
                            (field, html_escape(part), exc))
            normalized_field.append(' '.join(normalized_parts))
        global_dict[field] = normalized_field
コード例 #8
0
ファイル: showvgridmonitor.py プロジェクト: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = initialize_main_variables(client_id, op_header=False)
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict, defaults, output_objects, client_id, configuration, allow_rejects=False
    )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    meta = (
        """<meta http-equiv="refresh" content="%s" />
"""
        % configuration.sleep_secs
    )
    style = themed_styles(configuration)
    script = """
<script type="text/javascript" src="/images/js/jquery.js"></script>
<script type="text/javascript" src="/images/js/jquery.tablesorter.js"></script>

<script type="text/javascript" >

$(document).ready(function() {

          // table initially sorted by col. 1 (name)
          var sortOrder = [[1,0]];

          // use image path for sorting if there is any inside
          var imgTitle = function(contents) {
              var key = $(contents).find("a").attr("class");
              if (key == null) {
                  key = $(contents).html();
              }
              return key;
          }
          $("table.monitor").tablesorter({widgets: ["zebra"],
                                          textExtraction: imgTitle,
                                         });
          $("table.monitor").each(function () {
              try {
                  $(this).trigger("sorton", [sortOrder]);
              } catch(err) {
                  /* tablesorter chokes on empty tables - just continue */
              }
          });
     }
);
</script>
"""

    title_entry = find_entry(output_objects, "title")
    title_entry["text"] = "%s Monitor" % configuration.short_title
    title_entry["meta"] = meta
    title_entry["style"] = style
    title_entry["javascript"] = script

    allowed_vgrids = user_allowed_vgrids(configuration, client_id)
    vgrid_list = accepted["vgrid_name"]
    if all_vgrids in accepted["vgrid_name"]:
        vgrid_list = [i for i in vgrid_list if all_vgrids != i] + allowed_vgrids

    # Force list to sequence of unique entries

    for vgrid_name in set(vgrid_list):
        html = ""
        if not vgrid_is_owner_or_member(vgrid_name, client_id, configuration):
            output_objects.append(
                {
                    "object_type": "error_text",
                    "text": """You must be an owner or member of %s %s
to access the monitor."""
                    % (vgrid_name, configuration.site_vgrid_label),
                }
            )
            return (output_objects, returnvalues.CLIENT_ERROR)

        monitor_file = os.path.join(configuration.vgrid_home, vgrid_name, "%s.html" % configuration.vgrid_monitor)
        try:
            monitor_fd = open(monitor_file, "r")
            past_header = False
            for line in monitor_fd:
                if -1 != line.find("end of raw header"):
                    past_header = True
                    continue
                if not past_header:
                    continue
                if -1 != line.find("begin raw footer:"):
                    break
                html += str(line)
            monitor_fd.close()
        except Exception, exc:
            output_objects.append(
                {
                    "object_type": "error_text",
                    "text": "Error reading %s monitor page (%s)" % (configuration.site_vgrid_label, exc),
                }
            )
            return (output_objects, returnvalues.SYSTEM_ERROR)

        output_objects.append({"object_type": "html_form", "text": html})
コード例 #9
0
ファイル: sendrequestaction.py プロジェクト: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    defaults = signature()[1]

    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    if not correct_handler('POST'):
        output_objects.append(
            {'object_type': 'error_text', 'text'
             : 'Only accepting POST requests to prevent unintended updates'})
        return (output_objects, returnvalues.CLIENT_ERROR)

    title_entry = find_entry(output_objects, 'title')
    title_entry['text'] = '%s send request' % \
                            configuration.short_title
    output_objects.append({'object_type': 'header', 'text'
                          : '%s send request' % \
                            configuration.short_title})

    target_id = client_id
    vgrid_name = accepted['vgrid_name'][-1].strip()
    visible_user_name = accepted['cert_id'][-1].strip()
    visible_res_name = accepted['unique_resource_name'][-1].strip()
    request_type = accepted['request_type'][-1].strip().lower()
    request_text = accepted['request_text'][-1].strip()
    protocols = [proto.strip() for proto in accepted['protocol']]
    use_any = False
    if any_protocol in protocols:
        use_any = True
        protocols = configuration.notify_protocols
    protocols = [proto.lower() for proto in protocols]

    valid_request_types = ['resourceowner', 'resourceaccept', 'vgridowner',
                           'vgridmember','vgridresource', 'vgridaccept',
                           'plain']
    if not request_type in valid_request_types:
        output_objects.append({
            'object_type': 'error_text', 'text'
            : '%s is not a valid request_type (valid types: %s)!'
            % (request_type.lower(),
               valid_request_types)})
        return (output_objects, returnvalues.CLIENT_ERROR)

    if not protocols:
        output_objects.append({
            'object_type': 'error_text', 'text':
            'No protocol specified!'})
        return (output_objects, returnvalues.CLIENT_ERROR)

    user_map = get_user_map(configuration)
    reply_to = user_map[client_id][USERID]

    if request_type == "plain":
        if not visible_user_name:
            output_objects.append({
                'object_type': 'error_text', 'text':
                'No user ID specified!'})
            return (output_objects, returnvalues.CLIENT_ERROR)

        user_id = visible_user_name
        anon_map = anon_to_real_user_map(configuration.user_home)
        if anon_map.has_key(visible_user_name):
            user_id = anon_map[visible_user_name]
        if not user_map.has_key(user_id):
            output_objects.append({'object_type': 'error_text',
                                   'text': 'No such user: %s' % \
                                   visible_user_name
                                   })
            return (output_objects, returnvalues.CLIENT_ERROR)
        target_name = user_id
        user_dict = user_map[user_id]
        allow_vgrids = user_allowed_vgrids(configuration, client_id)
        vgrids_allow_email = user_dict[CONF].get('VGRIDS_ALLOW_EMAIL', [])
        vgrids_allow_im = user_dict[CONF].get('VGRIDS_ALLOW_IM', [])
        if any_vgrid in vgrids_allow_email:
            email_vgrids = allow_vgrids
        else:
            email_vgrids = set(vgrids_allow_email).intersection(allow_vgrids)
        if any_vgrid in vgrids_allow_im:
            im_vgrids = allow_vgrids
        else:
            im_vgrids = set(vgrids_allow_im).intersection(allow_vgrids)
        if use_any:
            # Do not try disabled protocols if ANY was requested
            if not email_vgrids:
                protocols = [proto for proto in protocols \
                             if proto not in email_keyword_list]
            if not im_vgrids:
                protocols = [proto for proto in protocols \
                             if proto in email_keyword_list]
        if not email_vgrids and [proto for proto in protocols \
                                 if proto in email_keyword_list]:
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'You are not allowed to send emails to %s!' % \
                visible_user_name
                })
            return (output_objects, returnvalues.CLIENT_ERROR)
        if not im_vgrids and [proto for proto in protocols \
                              if proto not in email_keyword_list]:
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'You are not allowed to send instant messages to %s!' % \
                visible_user_name
                })
            return (output_objects, returnvalues.CLIENT_ERROR)
        for proto in protocols:
            if not user_dict[CONF].get(proto.upper(), False):
                if use_any:
                    # Remove missing protocols if ANY protocol was requested
                    protocols = [i for i in protocols if i != proto]
                else:
                    output_objects.append({
                        'object_type': 'error_text', 'text'
                        : 'User %s does not accept %s messages!' % \
                        (visible_user_name, proto)
                        })
                    return (output_objects, returnvalues.CLIENT_ERROR)
        if not protocols:
            output_objects.append({
                'object_type': 'error_text', 'text':
                'User %s does not accept requested protocol(s) messages!' % \
                visible_user_name})
            return (output_objects, returnvalues.CLIENT_ERROR)
        target_list = [user_id]
    elif request_type == "vgridaccept":
        # Always allow accept messages but only between vgrid members/owners
        user_id = visible_user_name
        if not vgrid_name:
            output_objects.append({
                'object_type': 'error_text', 'text': 'No vgrid_name specified!'})
            return (output_objects, returnvalues.CLIENT_ERROR)
        if vgrid_name.upper() == default_vgrid.upper():
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'No requests for %s are not allowed!' % \
                default_vgrid
                })
            return (output_objects, returnvalues.CLIENT_ERROR)
        if not vgrid_is_owner(vgrid_name, client_id, configuration):
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'You are not an owner of %s or a parent %s!' % \
                (vgrid_name, configuration.site_vgrid_label)})
            return (output_objects, returnvalues.CLIENT_ERROR)
        allow_vgrids = user_allowed_vgrids(configuration, client_id)
        if not vgrid_name in allow_vgrids:
            output_objects.append({
                'object_type': 'error_text', 'text':
                'Invalid %s message! (%s sv %s)' % (request_type, user_id,
                                                    allow_vgrids)})
            return (output_objects, returnvalues.CLIENT_ERROR)
        target_id = '%s %s owners' % (vgrid_name, configuration.site_vgrid_label)
        target_name = vgrid_name
        target_list = [user_id]
    elif request_type == "resourceaccept":
        # Always allow accept messages between actual resource owners
        user_id = visible_user_name
        if not visible_res_name:
            output_objects.append({
                'object_type': 'error_text', 'text':
                'No resource ID specified!'})
            return (output_objects, returnvalues.CLIENT_ERROR)
        unique_resource_name = visible_res_name
        target_name = unique_resource_name
        res_map = get_resource_map(configuration)
        if not res_map.has_key(unique_resource_name):
            output_objects.append({'object_type': 'error_text',
                                   'text': 'No such resource: %s' % \
                                   unique_resource_name
                                   })
            return (output_objects, returnvalues.CLIENT_ERROR)
        owners_list = res_map[unique_resource_name][OWNERS]
        if not client_id in owners_list or not user_id in owners_list:
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'Invalid resource owner accept message!'})
            return (output_objects, returnvalues.CLIENT_ERROR)
        target_id = '%s resource owners' % unique_resource_name
        target_name = unique_resource_name
        target_list = [user_id]
    elif request_type == "resourceowner":
        if not visible_res_name:
            output_objects.append({
                'object_type': 'error_text', 'text':
                'No resource ID specified!'})
            return (output_objects, returnvalues.CLIENT_ERROR)
        
        unique_resource_name = visible_res_name
        anon_map = anon_to_real_res_map(configuration.resource_home)
        if anon_map.has_key(visible_res_name):
            unique_resource_name = anon_map[visible_res_name]
        target_name = unique_resource_name
        res_map = get_resource_map(configuration)
        if not res_map.has_key(unique_resource_name):
            output_objects.append({'object_type': 'error_text',
                                   'text': 'No such resource: %s' % \
                                   visible_res_name
                                   })
            return (output_objects, returnvalues.CLIENT_ERROR)
        target_list = res_map[unique_resource_name][OWNERS]
        if client_id in target_list:
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'You are already an owner of %s!' % unique_resource_name
                })
            return (output_objects, returnvalues.CLIENT_ERROR)
    elif request_type in ["vgridmember", "vgridowner", "vgridresource"]:
        unique_resource_name = visible_res_name
        if not vgrid_name:
            output_objects.append({
                'object_type': 'error_text', 'text': 'No vgrid_name specified!'})
            return (output_objects, returnvalues.CLIENT_ERROR)

        # default vgrid is read-only
        
        if vgrid_name.upper() == default_vgrid.upper():
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'No requests for %s are not allowed!' % \
                default_vgrid
                })
            return (output_objects, returnvalues.CLIENT_ERROR)

        # stop owner or member request if already an owner

        if request_type != 'vgridresource':
            if vgrid_is_owner(vgrid_name, client_id, configuration):
                output_objects.append({
                    'object_type': 'error_text', 'text'
                    : 'You are already an owner of %s or a parent %s!' % \
                    (vgrid_name, configuration.site_vgrid_label)})
                return (output_objects, returnvalues.CLIENT_ERROR)

        # only ownership requests are allowed for existing members

        if request_type == 'vgridmember':
            if vgrid_is_member(vgrid_name, client_id, configuration):
                output_objects.append({
                    'object_type': 'error_text', 'text'
                    : 'You are already a member of %s or a parent %s.' % \
                    (vgrid_name, configuration.site_vgrid_label)})
                return (output_objects, returnvalues.CLIENT_ERROR)

        # set target to resource and prevent repeated resource access requests

        if request_type == 'vgridresource':
            target_id = unique_resource_name
            if vgrid_is_resource(vgrid_name, unique_resource_name,
                                 configuration):
                output_objects.append({
                    'object_type': 'error_text', 'text'
                    : 'You already have access to %s or a parent %s.' % \
                    (vgrid_name, configuration.site_vgrid_label)})
                return (output_objects, returnvalues.CLIENT_ERROR)

        # Find all VGrid owners

        target_name = vgrid_name
        (status, target_list) = vgrid_list(vgrid_name, 'owners', configuration)
        if not status:
            output_objects.append({
                'object_type': 'error_text', 'text'
                : 'Could not load list of current owners for %s %s!'
                % (vgrid_name, configuration.site_vgrid_label)})
            return (output_objects, returnvalues.CLIENT_ERROR)

    else:
        output_objects.append({
            'object_type': 'error_text', 'text': 'Invalid request type: %s' % \
            request_type})
        return (output_objects, returnvalues.CLIENT_ERROR)

    # Now send request to all targets in turn
    # TODO: inform requestor if no owners have mail/IM set in their settings
    
    for target in target_list:
        # USER_CERT entry is destination

        notify = []
        for proto in protocols:
            notify.append('%s: SETTINGS' % proto)
        job_dict = {'NOTIFY': notify, 'JOB_ID': 'NOJOBID', 'USER_CERT': target}

        notifier = notify_user_thread(
            job_dict,
            [target_id, target_name, request_type, request_text, reply_to],
            'SENDREQUEST',
            logger,
            '',
            configuration,
            )

        # Try finishing delivery but do not block forever on one message
        notifier.join(30)
    output_objects.append({'object_type': 'text', 'text':
                           'Sent %s message to %d people' % \
                           (request_type, len(target_list))})
    output_objects.append({'object_type': 'text', 'text':
                           """Please make sure you have notifications
configured on your Setings page if you expect a reply to this message"""})
    
    return (output_objects, returnvalues.OK)
コード例 #10
0
ファイル: showstats.py プロジェクト: heromod/migrid
    #  2. convert from json to dictionary, extract values we need
    # ...we do not really need json here...
    reply = jsonreply.replace('\r','')
    data = json.loads(reply)['rows'] # :: list of dict with "key","value"

    if not data:
        output_objects.append({'object_type': 'sectionheader', 'text' :
                               'No data available'})
        output_objects.append({'object_type': 'text', 'text' :
                               '''
The query you have requested did not return any data.
                               '''})
        return (output_objects, returnvalues.OK)

    # select the vgrids we are going to consider
    my_vgrids = vgrid.user_allowed_vgrids(configuration, client_id)

    # process the received data:

    # view field names, could be configurable as well, 
    # but this is anyway highly proprietary code
    table_names = {'count':'Job Count',
                   'wall_duration': 'Accumulated Wall Clock Time',
                   'charge': 'Accumulated Charge'}

    
    lookupdict = dict( [ (tuple(d['key']),d['value']) for d in data ] )

    if group_level == 1: 
        # if view not grouped in time (currently unused)
        # => only one column, but rows with other keys
コード例 #11
0
ファイル: vmachines.py プロジェクト: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    output_objects.append({'object_type': 'header', 'text':
                           'Virtual Machines'})
    status = returnvalues.OK
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    machine_name = accepted['machine_name'][-1].strip()
    memory = int(accepted['memory'][-1])
    disk = int(accepted['disk'][-1])
    vgrid = [name.strip() for name in accepted['vgrid']]
    architecture = accepted['architecture'][-1].strip()
    cpu_count = int(accepted['cpu_count'][-1])
    cpu_time = int(accepted['cpu_time'][-1])
    op_sys = accepted['os'][-1].strip()
    flavor = accepted['flavor'][-1].strip()
    hypervisor_re = accepted['hypervisor_re'][-1].strip()
    sys_re = accepted['sys_re'][-1].strip()
    action = accepted['action'][-1].strip()

    title_entry = find_entry(output_objects, 'title')
    title_entry['text'] = 'Virtual Machines'

    # jquery support for tablesorter and confirmation on "leave":

    css_helpers = {'css_base': os.path.join(configuration.site_images, 'css'),
                   'skin_base': configuration.site_skin_base}
    title_entry['style'] = themed_styles(configuration)
    title_entry['javascript'] = '''
<script type="text/javascript" src="/images/js/jquery.js"></script>
<script type="text/javascript" src="/images/js/jquery-ui.js"></script>
<script type="text/javascript" src="/images/js/jquery.confirm.js"></script>

<script type="text/javascript" >

$(document).ready(function() {

          // init confirmation dialog
          $( "#confirm_dialog" ).dialog(
              // see http://jqueryui.com/docs/dialog/ for options
              { autoOpen: false,
                modal: true, closeOnEscape: true,
                width: 500,
                buttons: {
                   "Cancel": function() { $( "#" + name ).dialog("close"); }
                }
              });

          $(".vm-tabs").tabs();
     }
);
</script>
'''

    if not configuration.site_enable_vmachines:
        output_objects.append({'object_type': 'text', 'text':
                               '''Virtual machines are disabled on this site.
Please contact the Grid admins %s if you think they should be enabled.
''' % configuration.admin_email})
        return (output_objects, returnvalues.OK)

    machine_req = {'memory': memory, 'disk': disk, 'cpu_count': cpu_count,
                   'cpu_time': cpu_time, 'architecture': architecture,
                   'vgrid': vgrid, 'os': op_sys, 'flavor': flavor,
                   'hypervisor_re': hypervisor_re, 'sys_re': sys_re}
    
    menu_items = ['vmrequest']

    # Html fragments

    submenu = render_menu(configuration, menu_class='navsubmenu',
                          base_menu=[], user_menu=menu_items)

    welcome_text = 'Welcome to your %s virtual machine management!' % \
                   configuration.short_title
    desc_text = '''On this page you can:
<ul>
    <li>Request Virtual Machines, by clicking on the button above</li>
    <li>See your virtual machines in the list below.</li>
    <li>Start, and connect to your Virtual Machine by clicking on it.</li>
    <li>Edit or delete your Virtual Machine from the Advanced tab.</li>
</ul>
'''

    output_objects.append({'object_type': 'html_form',
                           'text':'''
 <div id="confirm_dialog" title="Confirm" style="background:#fff;">
  <div id="confirm_text"><!-- filled by js --></div>
   <textarea cols="40" rows="4" id="confirm_input" style="display:none;"></textarea>
 </div>
'''})
    
    output_objects.append({'object_type': 'html_form', 'text': submenu})
    output_objects.append({'object_type': 'html_form', 'text'
                          : '<p>&nbsp;</p>'})
    output_objects.append({'object_type': 'sectionheader', 'text'
                          : welcome_text})
    output_objects.append({'object_type': 'html_form', 'text'
                          : desc_text})


    user_vms = vms.vms_list(client_id, configuration)
    if action == 'create':
        if not configuration.site_enable_vmachines:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "Virtual machines are disabled on this server"})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
        if not machine_name:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "requested build without machine name"})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)            
        elif machine_name in [vm["name"] for vm in user_vms]:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "requested machine name '%s' already exists!" % machine_name})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
        elif not flavor in vms.available_flavor_list(configuration):
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "requested pre-built flavor not available: %s" % flavor})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
        elif not hypervisor_re in \
                 vms.available_hypervisor_re_list(configuration):
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "requested hypervisor runtime env not available: %s" % \
                 hypervisor_re})
        elif not sys_re in vms.available_sys_re_list(configuration):
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "requested system pack runtime env not available: %s" % \
                 sys_re})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)

        # TODO: support custom build of machine using shared/vmbuilder.py

        # request for existing pre-built machine

        vms.create_vm(client_id, configuration, machine_name, machine_req)

    (action_status, action_msg, job_id) = (True, '', None)
    if action in ['start', 'stop', 'edit', 'delete']:
        if not configuration.site_enable_vmachines:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "Virtual machines are disabled on this server"})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
    if action == 'start':
        machine = {}
        for entry in user_vms:
            if machine_name == entry['name']:
                for name in machine_req.keys():
                    if isinstance(entry[name], basestring) and \
                                  entry[name].isdigit():
                        machine[name] = int(entry[name])
                    else:
                        machine[name] = entry[name]
                break
        (action_status, action_msg, job_id) = \
                        vms.enqueue_vm(client_id, configuration, machine_name,
                                       machine)
    elif action == 'edit':
        if not machine_name in [vm['name'] for vm in user_vms]:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "No such virtual machine: %s" % machine_name})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
        (action_status, action_msg) = \
                        vms.edit_vm(client_id, configuration, machine_name,
                                    machine_req)
    elif action == 'delete':
        if not machine_name in [vm['name'] for vm in user_vms]:
            output_objects.append(
                {'object_type': 'error_text', 'text':
                 "No such virtual machine: %s" % machine_name})
            status = returnvalues.CLIENT_ERROR
            return (output_objects, status)
        (action_status, action_msg) = \
                        vms.delete_vm(client_id, configuration, machine_name)
    elif action == 'stop':
        
        # TODO: manage stop - use live I/O to create vmname.stop in job dir

        pass

    if not action_status:
        output_objects.append({'object_type': 'error_text', 'text':
                               action_msg})
    
    # List the machines here

    output_objects.append({'object_type': 'sectionheader', 'text'
                          : 'Your machines:'})

    # Grab the vms available for the user

    machines = vms.vms_list(client_id, configuration)

    # Visual representation mapping of the machine state

    machine_states = {
        'EXECUTING': 'vm_running.jpg',
        'CANCELED': 'vm_off.jpg',
        'FAILED': 'vm_off.jpg',
        'FINISHED': 'vm_off.jpg',
        'UNKNOWN': 'vm_off.jpg',
        'QUEUED': 'vm_booting.jpg',
        'PARSE': 'vm_booting.jpg',
        }

    # Empirical upper bound on boot time in seconds used to decide between
    # desktop init and ready states

    boot_secs = 130

    # CANCELED/FAILED/FINISHED -> Powered Off
    # QUEUED -> Booting

    if len(machines) > 0:

        # Create a pretty list with start/edit/stop/connect links

        pretty_machines = \
            '<table style="border: 0; background: none;"><tr>'
        side_by_side = 3  # How many machines should be shown in a row?

        col = 0
        for machine in machines:

            # Machines on a row

            if col % side_by_side == 0:
                pretty_machines += '</tr><tr>'
            col += 1

            # Html format machine specifications in a fieldset

            password = '******'
            exec_time = 0
            if machine['job_id'] != 'UNKNOWN' and \
                   machine['status'] == 'EXECUTING':

                # TODO: improve on this time selection...
                # ... in distributed there is no global clock!

                exec_time = time.time() - 3600 \
                            - time.mktime(machine['execution_time'])
                password = vms.vnc_jobid(machine['job_id'])

            machine_specs = {}
            machine_specs.update(machine)
            machine_specs['password'] = password
            show_specs = """<fieldset>
<legend>VM Specs:</legend><ul class="no-bullets">
<li><input type="text" readonly value="%(os)s"> base system</li>
<li><input type="text" readonly value="%(flavor)s"> software flavor</li>
<li><input type="text" readonly value="%(memory)s"> MB memory</li>
<li><input type="text" readonly value="%(disk)s"> GB disk</li>
<li><input type="text" readonly value="%(cpu_count)s"> CPU's</li>
<li><input type="text" readonly value="%(vm_arch)s"> architecture</li>
"""
            if password != 'UNKNOWN':
                show_specs += """            
<li><input type="text" readonly value="%(password)s"> as VNC password</li>
"""
            show_specs += """            
</form></ul></fieldset>"""
            edit_specs = """<fieldset>
<legend>Edit VM Specs:</legend><ul class="no-bullets">
<form method="post" action="vmachines.py">
<input type="hidden" name="action" value="edit">
<input type="hidden" name="machine_name" value="%(name)s">
<input type="hidden" name="output_format" value="html">

<li><input type="text" readonly name="os" value="%(os)s"> base system</li>
<li><input type="text" readonly name="flavor" value="%(flavor)s"> software flavor</li>
<li><input type="text" readonly name="hypervisor_re" value="%(hypervisor_re)s"> hypervisor runtime env</li>
<li><input type="text" readonly name="sys_re" value="%(sys_re)s"> image pack runtime env</li>
<li><input type="text" name="memory" value="%(memory)s"> MB memory</li>
<li><input type="text" name="disk" value="%(disk)s"> GB disk</li>
<li><input type="text" name="cpu_count" value="%(cpu_count)s"> CPU's</li>
<li><select name="architecture">
"""
            for arch in [''] + configuration.architectures:
                select = ''
                if arch == machine_specs['architecture']:
                    select = 'selected'
                edit_specs += "<option %s value='%s'>%s</option>" % (select, arch,
                                                                arch)
            edit_specs += """</select> resource architecture
<li><input type="text" name="cpu_time" value="%(cpu_time)s"> s time slot</li>
<li><select name="vgrid" multiple>"""
            for vgrid_name in [any_vgrid] + \
                    user_allowed_vgrids(configuration, client_id):
                select = ''
                if vgrid_name in machine_specs['vgrid']:
                    select = 'selected'
                edit_specs += "<option %s>%s</option>" % (select, vgrid_name)
            edit_specs += """</select> %s(s)</li>""" % \
                          configuration.site_vgrid_label
            if password != 'UNKNOWN':
                edit_specs += """
<li><input type="text" readonly value="%(password)s"> as VNC password</li>
"""
            edit_specs += """
<input class="styled_button" type="submit" value="Save Changes">
</form>"""
            js_name = 'deletevm%s' % hexlify("%(name)s" % machine_specs)
            helper = html_post_helper(js_name, 'vmachines.py',
                                      {'machine_name': machine_specs['name'],
                                       'action': 'delete'})
            edit_specs += helper
            edit_specs += """<input class="styled_button" type="submit"
value="Delete Machine" onClick="javascript: confirmDialog(%s, '%s');" >
""" % (js_name, "Really permanently delete %(name)s VM?" % machine_specs)
            edit_specs += """</ul></fieldset>"""
            if machine['status'] == 'EXECUTING' and exec_time > boot_secs:
                machine_image = '<img src="/images/vms/' \
                    + machine_states[machine['status']] + '">'
            elif machine['status'] == 'EXECUTING' and exec_time < boot_secs:
                machine_image = \
                    '<img src="/images/vms/vm_desktop_loading.jpg' \
                    + '">'
            else:
                machine_image = '<img src="/images/vms/' \
                    + machine_states[machine['status']] + '">'
            machine_link = vms.machine_link(machine_image,
                    machine['job_id'], machine['name'], machine['uuid'
                    ], machine['status'], machine_req)

            # Smack all the html together

            fill_dict = {}
            fill_dict.update(machine)
            fill_dict['link'] = machine_link
            fill_dict['show_specs'] = show_specs % machine_specs
            fill_dict['edit_specs'] = edit_specs % machine_specs
            pretty_machines += '''
<td style="vertical-align: top;">
<fieldset><legend>%(name)s</legend>
<div id="%(name)s-tabs" class="vm-tabs">
<ul>
<li><a href="#%(name)s-overview">Overview</a></li>
<li><a href="#%(name)s-edit">Advanced</a></li>
</ul>
<div id="%(name)s-overview">
<p>%(link)s</p>
%(show_specs)s
</div>
<div id="%(name)s-edit">
%(edit_specs)s
</div>
</div>
</fieldset>
</td>''' % fill_dict

        pretty_machines += '</tr></table>'

        output_objects.append({'object_type': 'html_form', 'text'
                              : pretty_machines})
    else:
        output_objects.append(
            {'object_type': 'text', 'text'
             : "You don't have any virtual machines! "
             "Click 'Request Virtual Machine' to become a proud owner :)"
             })

    return (output_objects, status)
コード例 #12
0
ファイル: submitjob.py プロジェクト: heromod/migrid
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    client_dir = client_id_dir(client_id)
    status = returnvalues.OK
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    show_description = accepted['description'][-1].lower() == 'true'

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(os.path.join(configuration.user_home,
                               client_dir)) + os.sep

    template_path = os.path.join(base_dir, default_mrsl_filename)

    title_entry = find_entry(output_objects, 'title')
    title_entry['text'] = 'Submit Job'
    output_objects.append({'object_type': 'header', 'text'
                          : 'Submit Job'})
    default_mrsl = get_default_mrsl(template_path)
    settings_dict = load_settings(client_id, configuration)
    if not settings_dict or not settings_dict.has_key('SUBMITUI'):
        logger.info('Settings dict does not have SUBMITUI key - using default'
                    )
        submit_style = configuration.submitui[0]
    else:
        submit_style = settings_dict['SUBMITUI']

    # We generate all 3 variants of job submission (fields, textarea, files),
    # initially hide them and allow to switch between them using js.

    # could instead extract valid prefixes as in settings.py
    # (means: by "eval" from configuration). We stick to hard-coding.
    submit_options = ['fields_form', 'textarea_form', 'files_form']

    css_helpers = {'css_base': os.path.join(configuration.site_images, 'css'),
                   'skin_base': configuration.site_skin_base}
    title_entry['style'] = themed_styles(configuration,
                                         base=['jquery.fileupload.css',
                                               'jquery.fileupload-ui.css'],
                                         skin=['fileupload-ui.custom.css'])
    title_entry['javascript'] = '''
<script type="text/javascript" src="/images/js/jquery.js"></script>
<script type="text/javascript" src="/images/js/jquery-ui.js"></script>
<!--  Filemanager is only needed for fancy upload init wrapper -->
<script type="text/javascript" src="/images/js/jquery.form.js"></script>
<script type="text/javascript" src="/images/js/jquery.filemanager.js"></script>

<!-- Fancy file uploader and dependencies -->
<!-- The Templates plugin is included to render the upload/download listings -->
<script type="text/javascript" src="/images/js/tmpl.min.js"></script>
<!-- The Load Image plugin is included for the preview images and image resizing functionality -->
<script type="text/javascript" src="/images/js/load-image.min.js"></script>
<!-- The Iframe Transport is required for browsers without support for XHR file uploads -->
<script type="text/javascript" src="/images/js/jquery.iframe-transport.js"></script>
<!-- The basic File Upload plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload.js"></script>
<!-- The File Upload processing plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload-process.js"></script>
<!-- The File Upload image preview & resize plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload-image.js"></script>
<!-- The File Upload validation plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload-validate.js"></script>
<!-- The File Upload user interface plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload-ui.js"></script>
<!-- The File Upload jQuery UI plugin -->
<script type="text/javascript" src="/images/js/jquery.fileupload-jquery-ui.js"></script>

<script type="text/javascript" >

    options = %s;

    function setDisplay(this_id, new_d) {
        //console.log("setDisplay with: "+this_id);
        el = document.getElementById(this_id)
        if ( el == undefined || el.style == undefined ) {
            console.log("failed to locate display element: "+this_id);
            return; // avoid js null ref errors
        }
        el.style.display=new_d;
    }

    function switchTo(name) {
        //console.log("switchTo: "+name);
        for (o=0; o < options.length; o++) {
            if (name == options[o]) {
                setDisplay(options[o],"block");
            } else {
                setDisplay(options[o],"none");
            }
        }
    }

    function openFancyUpload() {
        var open_dialog = mig_fancyuploadchunked_init("fancyuploadchunked_dialog");
        var remote_path = ".";
        open_dialog("Upload Files", function() { return false; },
                    remote_path, false);
    }

    $(document).ready( function() {
         //console.log("document ready handler");
         switchTo("%s");
         $("#fancydialog").click(openFancyUpload);
    });

</script>
''' % (submit_options, submit_style + "_form")

    output_objects.append({'object_type': 'text', 'text':
                           'This page is used to submit jobs to the grid.'})

    output_objects.append({'object_type': 'verbatim',
                           'text': '''
There are %s interface styles available that you can choose among:''' % \
                           len(submit_options)})

    links = []
    for opt in submit_options:
        name = opt.split('_', 2)[0] 
        links.append({'object_type': 'link', 
                      'destination': "javascript:switchTo('%s')" % opt,
                      'class': 'submit%slink' % name,
                      'title': 'Switch to %s submit interface' % name,
                      'text' : '%s style' % name,
                      })
    output_objects.append({'object_type': 'multilinkline', 'links': links})

    output_objects.append({'object_type': 'text', 'text': '''
Please note that changes to the job description are *not* automatically
transferred if you switch style.'''}) 

    output_objects.append({'object_type': 'html_form', 'text':
                           '<div id="fields_form" style="display:none;">\n'})
    
    # Fields
    output_objects.append({'object_type': 'sectionheader', 'text'
                          : 'Please fill in your job description in the fields'
                           ' below:'
                          })
    output_objects.append({'object_type': 'text', 'text'
                          : """
Please fill in one or more fields below to define your job before hitting
Submit Job at the bottom of the page.
Empty fields will simply result in the default value being used and each field
is accompanied by a help link providing further details about the field."""})
    output_objects.append({'object_type': 'html_form', 'text'
                          : """
<div class="submitjob">
<form method="post" action="submitfields.py">
"""
                          })
    show_fields = get_job_specs(configuration)
    try:
        parsed_mrsl = dict(parse_lines(default_mrsl))
    except:
        parsed_mrsl = {}

    # Find allowed VGrids and Runtimeenvironments and add them to
    # configuration object for automated choice handling
    
    allowed_vgrids = user_allowed_vgrids(configuration, client_id) + \
                     [any_vgrid]
    allowed_vgrids.sort()
    configuration.vgrids = allowed_vgrids
    (re_status, allowed_run_envs) = list_runtime_environments(configuration)
    if not re_status:
        logger.error('Failed to extract allowed runtime envs: %s' % \
                     allowed_run_envs)
        allowed_run_envs = []
    allowed_run_envs.sort()
    configuration.runtimeenvironments = allowed_run_envs
    user_res = user_allowed_res_exes(configuration, client_id)

    # Allow any exe unit on all allowed resources
        
    allowed_resources = ['%s_*' % res for res in user_res.keys()]
    allowed_resources.sort()
    configuration.resources = allowed_resources
    field_size = 30
    area_cols = 80
    area_rows = 5

    for (field, spec) in show_fields:
        title = spec['Title']
        if show_description:
            description = '%s<br />' % spec['Description']
        else:
            description = ''
        field_type = spec['Type']
        # Use saved value and fall back to default if it is missing
        saved = parsed_mrsl.get('::%s::' % field, None)
        if saved:
            if not spec['Type'].startswith('multiple'):
                default = saved[0]
            else:
                default = saved
        else:
            default = spec['Value']
        # Hide sandbox field if sandboxes are disabled
        if field == 'SANDBOX' and not configuration.site_enable_sandboxes:
            continue
        if 'invisible' == spec['Editor']:
            continue
        if 'custom' == spec['Editor']:
            continue
        output_objects.append({'object_type': 'html_form', 'text'
                                   : """
<b>%s:</b>&nbsp;<a class='infolink' href='docs.py?show=job#%s'>help</a><br />
%s""" % (title, field, description)
                               })
        
        if 'input' == spec['Editor']:
            if field_type.startswith('multiple'):
                output_objects.append({'object_type': 'html_form', 'text'
                                       : """
<textarea name='%s' cols='%d' rows='%d'>%s</textarea><br />
""" % (field, area_cols, area_rows, '\n'.join(default))
                               })
            else:
                output_objects.append({'object_type': 'html_form', 'text'
                                       : """
<input type='text' name='%s' size='%d' value='%s' /><br />
""" % (field, field_size, default)
                               })
        elif 'select' == spec['Editor']:
            choices = available_choices(configuration, client_id,
                                        field, spec)
            res_value = default
            value_select = ''
            if field_type.startswith('multiple'):
                value_select += '<div class="scrollselect">'
                for name in choices:
                    # Blank default value does not make sense here
                    if not str(name):
                        continue
                    selected = ''
                    if str(name) in res_value:
                        selected = 'checked'
                    value_select += '''
                        <input type="checkbox" name="%s" %s value=%s>%s<br />
                        ''' % (field, selected, name, name)
                value_select += '</div>\n'
            else:
                value_select += "<select name='%s'>\n" % field
                for name in choices:
                    selected = ''
                    if str(res_value) == str(name):
                        selected = 'selected'
                    display = "%s" % name
                    if display == '':
                        display = ' '
                    value_select += """<option %s value='%s'>%s</option>\n""" \
                                    % (selected, name, display)
                value_select += """</select><br />\n"""    
            output_objects.append({'object_type': 'html_form', 'text'
                                   : value_select
                                   })
        output_objects.append({'object_type': 'html_form', 'text': "<br />"})

    output_objects.append({'object_type': 'html_form', 'text'
                          : """
<br />
<table class='centertext'>
<tr><td><input type='submit' value='Submit Job' />
<input type='checkbox' name='save_as_default'> Save as default job template
</td></tr>
</table>
<br />
</form>
</div>
"""
                           })
    output_objects.append({'object_type': 'html_form', 
                           'text': '''
</div><!-- fields_form-->
<div id="textarea_form" style="display:none;">
'''})
    
    # Textarea
    output_objects.append({'object_type': 'sectionheader', 'text'
                          : 'Please enter your mRSL job description below:'
                          })
    output_objects.append({'object_type': 'html_form', 'text'
                          : """
<div class='smallcontent'>
Job descriptions can use a wide range of keywords to specify job requirements
and actions.<br />
Each keyword accepts one or more values of a particular type.<br />
The full list of keywords with their default values and format is available in
the on-demand <a href='docs.py?show=job'>mRSL Documentation</a>.
<p>
Actual examples for inspiration:
<a href=/public/cpuinfo.mRSL>CPU Info</a>,
<a href=/public/basic-io.mRSL>Basic I/O</a>,
<a href=/public/notification.mRSL>Job Notification</a>,
<a href=/public/povray.mRSL>Povray</a> and
<a href=/public/vcr.mRSL>VCR</a>
</div>
    """})

    output_objects.append({'object_type': 'html_form', 'text'
                          : """
<!-- 
Please note that textarea.py chokes if no nonempty KEYWORD_X_Y_Z fields 
are supplied: thus we simply send a bogus jobname which does nothing
-->
<form method="post" action="textarea.py">
<table class="submitjob">
<tr><td class="centertext">
<input type=hidden name=jobname_0_0_0 value=" " />
<textarea cols="82" rows="25" name="mrsltextarea_0">
%(default_mrsl)s
</textarea>
</td></tr>
<tr><td class='centertext'>
<input type="submit" value="Submit Job" />
<input type="checkbox" name="save_as_default" >Save as default job template
</td></tr>
</table>
</form>
"""
                           % {'default_mrsl': default_mrsl}})

    output_objects.append({'object_type': 'html_form', 
                           'text': '''
</div><!-- textarea_form-->
<div id="files_form" style="display:none;">
'''})
    # Upload form

    output_objects.append({'object_type': 'sectionheader', 'text'
                          : 'Please upload your job file or packaged job files'
                           ' below:'
                          })
    output_objects.append({'object_type': 'html_form', 'text'
                          : """
<form enctype='multipart/form-data' action='textarea.py' method='post'>
<table class='files'>
<tr class='title'><td class='centertext' colspan=4>
Upload job files
</td></tr>
<tr><td colspan=3>
Upload file to current directory (%(dest_dir)s)
</td><td><br /></td></tr>
<tr><td colspan=2>
Extract package files (.zip, .tar.gz, .tar.bz2)
</td><td colspan=2>
<input type=checkbox name='extract_0' />
</td></tr>
<tr><td colspan=2>
Submit mRSL files (also .mRSL files included in packages)
</td><td colspan=2>
<input type=checkbox name='submitmrsl_0' checked />
</td></tr>
<tr><td>    
File to upload
</td><td class='righttext' colspan=3>
<input name='fileupload_0_0_0' type='file'/>
</td></tr>
<tr><td>
Optional remote filename (extra useful in windows)
</td><td class='righttext' colspan=3>
<input name='default_remotefilename_0' type='hidden' value='%(dest_dir)s'/>
<input name='remotefilename_0' type='text' size='50' value='%(dest_dir)s'/>
<input type='submit' value='Upload' name='sendfile'/>
</td></tr>
</table>
</form>
<table class='files'>
<tr class='title'><td class='centertext'>
Upload other files efficiently (using chunking).
</td></tr>
<tr><td class='centertext'>
<button id='fancydialog'>Open Upload dialog</button>
</td></tr>
</table>
</div><!-- files_form-->

<div id='fancyuploadchunked_dialog' title='Upload File' style='display: none;'>

    <!-- The file upload form used as target for the file upload widget -->
    <form id='fancyfileupload' action='uploadchunked.py?output_format=json;action=put'
        method='POST' enctype='multipart/form-data'>
        <fieldset id='fancyfileuploaddestbox'>
            <label id='fancyfileuploaddestlabel' for='fancyfileuploaddest'>
                Optional final destination dir:
            </label>
            <input id='fancyfileuploaddest' type='text' size=60 value=''>
        </fieldset>

        <!-- The fileupload-buttonbar contains buttons to add/delete files and start/cancel the upload -->
        <div class='fileupload-buttonbar'>
            <div class='fileupload-buttons'>
                <!-- The fileinput-button span is used to style the file input field as button -->
                <span class='fileinput-button'>
                    <span>Add files...</span>
                    <input type='file' name='files[]' multiple>
                </span>
                <button type='submit' class='start'>Start upload</button>
                <button type='reset' class='cancel'>Cancel upload</button>
                <button type='button' class='delete'>Delete</button>
                <input type='checkbox' class='toggle'>
                <!-- The global file processing state -->
                <span class='fileupload-process'></span>
            </div>
            <!-- The global progress state -->
            <div class='fileupload-progress fade' style='display:none'>
                <!-- The global progress bar -->
                <div class='progress' role='progressbar' aria-valuemin='0' aria-valuemax='100'></div>
                <!-- The extended global progress state -->
                <div class='progress-extended'>&nbsp;</div>
            </div>
        </div>
        <!-- The table listing the files available for upload/download -->
        <table role='presentation' class='table table-striped'><tbody class='uploadfileslist'></tbody></table>
    </form>
    <!-- For status and error output messages -->
    <div id='fancyuploadchunked_output'></div>
</div>
    """ % {'dest_dir': '.' + os.sep}})
    
    output_objects.append({'object_type': 'html_form', 
                           'text': '''
<!-- The template to display files available for upload -->
<script id="template-upload" type="text/x-tmpl">
{% console.log("using upload template"); %}
{% console.log("... with upload files: "+$.fn.dump(o)); %}
{% var dest_dir = "./" + $("#fancyfileuploaddest").val(); %}
{% console.log("using upload dest: "+dest_dir); %}
{% for (var i=0, file; file=o.files[i]; i++) { %}
    {% var rel_path = $.fn.normalizePath(dest_dir+"/"+file.name); %}
    {% console.log("using upload rel_path: "+rel_path); %}
    <tr class="template-upload fade">
        <td>
            <span class="preview"></span>
        </td>
        <td>
            <p class="name">{%=rel_path%}</p>
            <strong class="error"></strong>
        </td>
        <td>
            <div class="size pending">Processing...</div>
            <div class="progress"></div>
        </td>
        <td>
            {% if (!i && !o.options.autoUpload) { %}
                <button class="start" disabled>Start</button>
            {% } %}
            {% if (!i) { %}
                <button class="cancel">Cancel</button>
            {% } %}
        </td>
    </tr>
{% } %}
</script>
<!-- The template to display files available for download -->
<script id="template-download" type="text/x-tmpl">
{% console.log("using download template"); %}
{% console.log("... with download files: "+$.fn.dump(o)); %}
{% for (var i=0, file; file=o.files[i]; i++) { %}
    {% var rel_path = $.fn.normalizePath("./"+file.name); %}
    {% console.log("using download rel_path: "+rel_path); %}
    <tr class="template-download fade">
        <td>
            <span class="preview">
                {% if (file.thumbnailUrl) { %}
                <a href="{%=file.url%}" title="{%=file.name%}" download="{%=file.name%}" data-gallery><img src="{%=file.thumbnailUrl%}"></a>
                {% } %}
            </span>
        </td>
        <td>
            <p class="name">
                <a href="{%=file.url%}" title="{%=file.name%}" download="{%=file.name%}" {%=file.thumbnailUrl?\'data-gallery\':\'\'%}>{%=rel_path%}</a>
            </p>
            {% if (file.error) { %}
                <div><span class="error">Error</span> {%=file.error%}</div>
            {% } %}
        </td>
        <td>
            <div class="size">{%=o.formatFileSize(file.size)%}</div>
        </td>
        <td>
            <button class="delete" data-type="{%=file.deleteType%}" data-url="{%=file.deleteUrl%}"{% if (file.deleteWithCredentials) { %} data-xhr-fields=\'{"withCredentials":true}\'{% } %}>{% if (file.deleteUrl) { %}Delete{% } else { %}Dismiss{% } %}</button>
            <input type="checkbox" name="delete" value="1" class="toggle">
        </td>
    </tr>
{% } %}
</script>
    '''})
    return (output_objects, status)