def parse_peers_userid(configuration, raw_entries):
"""Parse list of user IDs into a list of peers"""
_logger = configuration.logger
peers = []
err = []
for entry in raw_entries:
raw_user = distinguished_name_to_user(entry.strip())
missing = [i for i in peers_fields if i not in raw_user]
if missing:
err.append("Parsed peers did NOT contain required field(s): %s"
% ', '.join(missing))
continue
# IMPORTANT: extract ONLY peers fields and validate to avoid abuse
peer_user = dict([(i, raw_user.get(i, '').strip())
for i in peers_fields])
defaults = dict([(i, REJECT_UNSET) for i in peer_user])
(accepted, rejected) = validated_input(peer_user, defaults,
list_wrap=True)
if rejected:
_logger.warning('skip peer with invalid value(s): %s : %s'
% (entry, rejected))
unsafe_err = ' , '.join(
['%s=%r' % pair for pair in peer_user.items()])
unsafe_err += '. Rejected values: ' + ', '.join(rejected)
err.append("Skip peer user with invalid value(s): %s" %
html_escape(unsafe_err))
continue
peers.append(canonical_user(configuration, peer_user, peers_fields))
_logger.debug('parsed user id into peers: %s' % peers)
return (peers, err)
def parse_peers_form(configuration, raw_lines, csv_sep):
"""Parse CSV form of peers into a list of peers"""
_logger = configuration.logger
header = None
peers = []
err = []
for line in raw_lines.split('\n'):
line = line.split('#', 1)[0].strip()
if not line:
continue
parts = [i.strip() for i in line.split(csv_sep)]
if not header:
missing = [i for i in peers_fields if i not in parts]
if missing:
err.append("Parsed peers did NOT contain required field(s): %s"
% ', '.join(missing))
header = parts
continue
if len(header) != len(parts):
_logger.warning('skip peers line with mismatch in field count: %s'
% line)
err.append("Skip peers line not matching header format: %s" %
html_escape(line + ' vs ' + csv_sep.join(header)))
continue
raw_user = dict(zip(header, parts))
# IMPORTANT: extract ONLY peers fields and validate to avoid abuse
peer_user = dict([(i, raw_user.get(i, '')) for i in peers_fields])
defaults = dict([(i, REJECT_UNSET) for i in peer_user])
(accepted, rejected) = validated_input(peer_user, defaults,
list_wrap=True)
if rejected:
_logger.warning('skip peer with invalid value(s): %s (%s)'
% (line, rejected))
unsafe_err = ' , '.join(
['%s=%r' % pair for pair in peer_user.items()])
unsafe_err += '. Rejected values: ' + ', '.join(rejected)
err.append("Skip peer user with invalid value(s): %s" %
html_escape(unsafe_err))
continue
peers.append(canonical_user(configuration, peer_user, peers_fields))
_logger.debug('parsed form into peers: %s' % peers)
return (peers, err)
def operation_value_checker(operation_value):
"""
Validate that the provided job operation is allowed. A ValueError
Exception will be raised if operation_value is invalid.
:param operation_value: The operation to be checked. Valid operations are:
'create', 'read', 'update' and 'delete'.
:return: No return.
"""
if operation_value not in ALL_OPERATIONS:
raise ValueError("Workflow operation '%s' is not valid. " %
html_escape(operation_value))
def type_value_checker(type_value):
"""
Validate that the provided job type is allowed. A ValueError
Exception will be raised if type_value is invalid.
:param type_value: The type to be checked. Valid types are 'job',
'queue', 'workflowpattern', 'workflowrecipe' and 'workflowany'
:return: No return
"""
valid_types = WORKFLOW_TYPES + WORKFLOW_SEARCH_TYPES + JOB_TYPES
if type_value not in valid_types:
raise ValueError("Request type '%s' is not valid. " %
html_escape(type_value))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False, op_menu=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input(user_arguments_dict,
defaults,
output_objects,
allow_rejects=False)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not configuration.site_enable_openid or \
not 'migoid' in configuration.site_signup_methods:
output_objects.append({
'object_type':
'error_text',
'text':
'''Local OpenID login is not enabled on this site'''
})
return (output_objects, returnvalues.SYSTEM_ERROR)
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = '%s OpenID account request' % \
configuration.short_title
title_entry['skipmenu'] = True
form_fields = [
'full_name', 'organization', 'email', 'country', 'state', 'password',
'verifypassword', 'comment'
]
title_entry['style']['advanced'] += account_css_helpers(configuration)
add_import, add_init, add_ready = account_js_helpers(
configuration, form_fields)
title_entry['script']['advanced'] += add_import
title_entry['script']['init'] += add_init
title_entry['script']['ready'] += add_ready
title_entry['script']['body'] = "class='staticpage'"
header_entry = {
'object_type':
'header',
'text':
'%s account request - with OpenID login' % configuration.short_title
}
output_objects.append(header_entry)
output_objects.append({
'object_type':
'html_form',
'text':
'''
<div id="contextual_help">
</div>
'''
})
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
user_fields = {
'full_name': '',
'organization': '',
'email': '',
'state': '',
'country': '',
'password': '',
'verifypassword': '',
'comment': ''
}
if not os.path.isdir(base_dir) and client_id:
# Redirect to extcert page with certificate requirement but without
# changing access method (CGI vs. WSGI).
extcert_url = os.environ['REQUEST_URI'].replace('-sid', '-bin')
extcert_url = os.path.join(os.path.dirname(extcert_url), 'extcert.py')
extcert_link = {
'object_type': 'link',
'destination': extcert_url,
'text': 'Sign up with existing certificate (%s)' % client_id
}
output_objects.append({
'object_type':
'warning',
'text':
'''Apparently
you already have a suitable %s certificate that you may sign up with:''' %
configuration.short_title
})
output_objects.append(extcert_link)
output_objects.append({
'object_type':
'warning',
'text':
'''However,
if you want a dedicated %s %s User OpenID you can still request one below:''' %
(configuration.short_title, configuration.user_mig_oid_title)
})
elif client_id:
for entry in (title_entry, header_entry):
entry['text'] = entry['text'].replace('request', 'request / renew')
output_objects.append({
'object_type':
'html_form',
'text':
'''<p>
Apparently you already have valid %s credentials, but if you want to add %s
User OpenID access to the same account you can do so by posting the form below.
Changing fields is <span class="warningtext"> not </span> supported, so all fields
must remain unchanged for it to work.
Otherwise it results in a request for a new account and OpenID without access
to your old files, jobs and privileges. </p>''' %
(configuration.short_title, configuration.user_mig_oid_title)
})
user_fields.update(distinguished_name_to_user(client_id))
# Override with arg values if set
for field in user_fields:
if not field in accepted:
continue
override_val = accepted[field][-1].strip()
if override_val:
user_fields[field] = override_val
user_fields = canonical_user(configuration, user_fields,
user_fields.keys())
# Site policy dictates min length greater or equal than password_min_len
policy_min_len, policy_min_classes = parse_password_policy(configuration)
user_fields.update({
'valid_name_chars':
'%s (and common accents)' % html_escape(valid_name_chars),
'valid_password_chars':
html_escape(valid_password_chars),
'password_min_len':
max(policy_min_len, password_min_len),
'password_max_len':
password_max_len,
'password_min_classes':
max(policy_min_classes, 1),
'site':
configuration.short_title
})
form_method = 'post'
csrf_limit = get_csrf_limit(configuration)
fill_helpers = {
'form_method': form_method,
'csrf_field': csrf_field,
'csrf_limit': csrf_limit
}
target_op = 'reqoidaction'
csrf_token = make_csrf_token(configuration, form_method, target_op,
client_id, csrf_limit)
fill_helpers.update({'target_op': target_op, 'csrf_token': csrf_token})
fill_helpers.update({'site_signup_hint': configuration.site_signup_hint})
# Write-protect ID fields if requested
for field in cert_field_map:
fill_helpers['readonly_%s' % field] = ''
ro_fields = [i for i in accepted['ro_fields'] if i in cert_field_map]
if keyword_auto in accepted['ro_fields']:
ro_fields += [i for i in cert_field_map if not i in ro_fields]
for field in ro_fields:
fill_helpers['readonly_%s' % field] = 'readonly'
fill_helpers.update(user_fields)
html = """
<p class='sub-title'>Please enter your information in at least the
<span class='highlight_required'>mandatory</span> fields below and press the
Send button to submit the account request to the %(site)s administrators.</p>
<p class='personal leftpad highlight_message'>
IMPORTANT: we need to identify and notify you about login info, so please use a
working Email address clearly affiliated with your Organization!
</p>
%(site_signup_hint)s
<hr />
"""
html += account_request_template(configuration,
default_values=fill_helpers)
# TODO: remove this legacy version?
html += """
<div style="height: 0; visibility: hidden; display: none;">
<!--OLD FORM-->
<form method='%(form_method)s' action='%(target_op)s.py' onSubmit='return validate_form();'>
<input type='hidden' name='%(csrf_field)s' value='%(csrf_token)s' />
<table>
<!-- NOTE: javascript support for unicode pattern matching is lacking so we
only restrict e.g. Full Name to words separated by space here. The
full check takes place in the backend, but users are better of with
sane early warnings than the cryptic backend errors.
-->
<tr><td class='mandatory label'>Full name</td><td><input id='full_name_field' type=text name=cert_name value='%(full_name)s' required pattern='[^ ]+([ ][^ ]+)+' title='Your full name, i.e. two or more names separated by space' /></td><td class=fill_space><br /></td></tr>
<tr><td class='mandatory label'>Email address</td><td><input id='email_field' type=email name=email value='%(email)s' required title='A valid email address that you read' /> </td><td class=fill_space><br /></td></tr>
<tr><td class='mandatory label'>Organization</td><td><input id='organization_field' type=text name=org value='%(organization)s' required pattern='[^ ]+([ ][^ ]+)*' title='Name of your organisation: one or more abbreviations or words separated by space' /></td><td class=fill_space><br /></td></tr>
<tr><td class='mandatory label'>Two letter country-code</td><td><input id='country_field' type=text name=country minlength=2 maxlength=2 value='%(country)s' required pattern='[A-Z]{2}' title='The two capital letters used to abbreviate your country' /></td><td class=fill_space><br /></td></tr>
<tr><td class='optional label'>State</td><td><input id='state_field' type=text name=state value='%(state)s' pattern='([A-Z]{2})?' maxlength=2 title='Leave empty or enter the capital 2-letter abbreviation of your state if you are a US resident' /> </td><td class=fill_space><br /></td></tr>
<tr><td class='mandatory label'>Password</td><td><input id='password_field' type=password name=password minlength=%(password_min_len)d maxlength=%(password_max_len)d value='%(password)s' required pattern='.{%(password_min_len)d,%(password_max_len)d}' title='Password of your choice, see help box for limitations' /> </td><td class=fill_space><br /></td></tr>
<tr><td class='mandatory label'>Verify password</td><td><input id='verifypassword_field' type=password name=verifypassword minlength=%(password_min_len)d maxlength=%(password_max_len)d value='%(verifypassword)s' required pattern='.{%(password_min_len)d,%(password_max_len)d}' title='Repeat your chosen password' /></td><td class=fill_space><br /></td></tr>
<!-- NOTE: we technically allow saving the password on scrambled form hide it by default -->
<tr class='hidden'><td class='optional label'>Password recovery</td><td class=''><input id='passwordrecovery_checkbox' type=checkbox name=passwordrecovery></td><td class=fill_space><br/></td></tr>
<tr><td class='optional label'>Optional comment or reason why you should<br />be granted a %(site)s account:</td><td><textarea id='comment_field' rows=4 name=comment title='A free-form comment where you can explain what you need the account for' ></textarea></td><td class=fill_space><br /></td></tr>
<tr><td class='label'><!-- empty area --></td><td>
<input id='submit_button' type=submit value=Send /></td><td class=fill_space><br/></td></tr>
</table>
</form>
<hr />
<div class='warn_message'>Please note that if you enable password recovery your password will be saved on encoded format but recoverable by the %(site)s administrators</div>
</div>
<br />
<br />
<!-- Hidden help text -->
<div id='help_text'>
<div id='1full_name_help'>Your full name, restricted to the characters in '%(valid_name_chars)s'</div>
<div id='1organization_help'>Organization name or acronym matching email</div>
<div id='1email_help'>Email address associated with your organization if at all possible</div>
<div id='1country_help'>Country code of your organization and on the form DE/DK/GB/US/.. , <a href='https://en.wikipedia.org/wiki/ISO_3166-1'>help</a></div>
<div id='1state_help'>Optional 2-letter ANSI state code of your organization, please just leave empty unless it is in the US or similar, <a href='https://en.wikipedia.org/wiki/List_of_U.S._state_abbreviations'>help</a></div>
<div id='1password_help'>Password is restricted to the characters:<br/><tt>%(valid_password_chars)s</tt><br/>Certain other complexity requirements apply for adequate strength. For example it must be %(password_min_len)s to %(password_max_len)s characters long and contain at least %(password_min_classes)d different character classes.</div>
<div id='1verifypassword_help'>Please repeat password</div>
<!--<div id='1comment_help'>Optional, but a short informative comment may help us verify your account needs and thus speed up our response. Typically the name of a local collaboration partner or project may be helpful.</div>-->
</div>
"""
output_objects.append({
'object_type': 'html_form',
'text': html % fill_helpers
})
return (output_objects, returnvalues.OK)
def main(client_id, user_arguments_dict):
"""
Main function used by front end.
:param client_id: A MiG user.
:param user_arguments_dict: A JSON message sent to the MiG. This will be
parsed and if valid, the relevant API handler functions are called to
generate meaningful output.
:return: (Tuple (list, Tuple(integer,string))) Returns a tuple with the
first value being a list of output objects generated by the call. The
second value is also a tuple used for error code reporting, with the first
value being an error code and the second being a brief explanation.
"""
# Ensure that the output format is in JSON
user_arguments_dict['output_format'] = ['json']
user_arguments_dict.pop('__DELAYED_INPUT__', None)
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_title=False, op_header=False,
op_menu=False)
# Add allow Access-Control-Allow-Origin to headers
# Required to allow Jupyter Widget from localhost to request against the
# API
# TODO, possibly restrict allowed origins
output_objects[0]['headers'].append(('Access-Control-Allow-Origin', '*'))
output_objects[0]['headers'].append(
('Access-Control-Allow-Headers', 'Content-Type'))
output_objects[0]['headers'].append(('Access-Control-Max-Age', 600))
output_objects[0]['headers'].append(
('Access-Control-Allow-Methods', 'POST, OPTIONS'))
output_objects[0]['headers'].append(('Content-Type', 'application/json'))
if not correct_handler('POST'):
msg = "Interaction from %s not POST request" % client_id
logger.error(msg)
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not configuration.site_enable_workflows:
output_objects.append({
'object_type':
'error_text',
'text':
'Workflows are not enabled on this system'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Input data
data = sys.stdin.read()
try:
json_data = json.loads(data, object_hook=force_utf8_rec)
except ValueError:
msg = "An invalid format was supplied to: '%s', requires a JSON " \
"compatible format" % op_name
logger.error(msg)
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.CLIENT_ERROR)
# IMPORTANT!! Do not access the json_data input before it has been
# validated by validated_input. Note attributes entry has not yet been
# validated, this is done once the type and operation is determined
accepted, rejected = validated_input(json_data,
REQUEST_SIGNATURE,
type_override=REQUEST_TYPE_MAP,
value_override=REQUEST_VALUE_MAP,
list_wrap=True)
if not accepted or rejected:
logger.error("A validation error occurred: '%s'" % rejected)
msg = "Invalid input was supplied to the request API: %s" % rejected
# TODO, Transform error messages to something more readable
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.CLIENT_ERROR)
request_type = accepted.pop('type', [None])[0]
operation = accepted.pop('operation', None)[0]
workflow_session_id = accepted.pop('workflowsessionid', None)[0]
vgrid = accepted.pop('vgrid', None)
# Note these have not been sufficiently checked, and should not be accessed
# until the valid_attributes_for_type function has been called on them.
# This is done later once we have checked the operation and request_type.
attributes = {}
for key, value in accepted.items():
if key in json_data['attributes']:
attributes[key] = value
attributes['vgrid'] = vgrid
if not valid_session_id(configuration, workflow_session_id):
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid workflowsessionid'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# workflow_session_id symlink points to the vGrid it gives access to
workflow_sessions_db = []
try:
workflow_sessions_db = load_workflow_sessions_db(configuration)
except IOError:
logger.info("Workflow sessions db didn't load, creating new db")
if not touch_workflow_sessions_db(configuration, force=True):
output_objects.append({
'object_type':
'error_text',
'text':
"Internal sessions db failure, please contact "
"an admin at '%s' to resolve this issue." %
configuration.admin_email
})
return (output_objects, returnvalues.SYSTEM_ERROR)
else:
# Try reload
workflow_sessions_db = load_workflow_sessions_db(configuration)
if workflow_session_id not in workflow_sessions_db:
logger.error("Workflow session '%s' from user '%s' not found in "
"database" % (workflow_session_id, client_id))
configuration.auth_logger.error(
"Workflow session '%s' provided by user '%s' but not present in "
"database" % (workflow_session_id, client_id))
# TODO Also track multiple attempts from the same IP
output_objects.append({
'object_type': 'error_text',
'text': 'Invalid workflowsessionid'
})
return (output_objects, returnvalues.CLIENT_ERROR)
workflow_session = workflow_sessions_db.get(workflow_session_id)
logger.info('jsoninterface found %s' % workflow_session)
owner = workflow_session['owner']
# User is vgrid owner or member
success, msg, _ = init_vgrid_script_list(vgrid, owner, configuration)
if not success:
logger.error("Illegal access attempt by user '%s' to vgrid '%s'. %s" %
(owner, vgrid, msg))
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.CLIENT_ERROR)
status, msg = valid_attributes_for_type(configuration, attributes,
request_type)
if not status:
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
status, msg = valid_operation_for_type(configuration, operation,
request_type)
if not status:
output_objects.append({
'object_type': 'error_text',
'text': html_escape(msg)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
request_func = VALID_REQUEST_OPERATIONS[request_type][operation]
status, response = request_func(configuration, owner, attributes)
output_objects.append(response)
if not status:
return (output_objects, returnvalues.SYSTEM_ERROR)
return (output_objects, returnvalues.OK)
def handle_update(configuration,
client_id,
resource_id,
user_vars,
output_objects,
new_resource=False):
"""Update existing resource configuration from request"""
logger = configuration.logger
client_dir = client_id_dir(client_id)
tmp_id = "%s.%s" % (user_vars['HOSTURL'], time.time())
pending_file = os.path.join(configuration.resource_pending, client_dir,
tmp_id)
conf_file = os.path.join(configuration.resource_home, resource_id,
'config.MiG')
output = ''
try:
logger.info('write to file: %s' % pending_file)
write_resource_config(configuration, user_vars, pending_file)
except Exception as err:
logger.error('Resource conf %s could not be written: %s' %
(pending_file, err))
output_objects.append({
'object_type': 'error_text',
'text': 'Could not write configuration!'
})
return False
if not new_resource:
(update_status,
msg) = update_resource(configuration, client_id, user_vars["HOSTURL"],
user_vars["HOSTIDENTIFIER"], pending_file)
if not update_status:
output_objects.append({
'object_type': 'error_text',
'text': 'Resource update failed:'
})
output_objects.append({'object_type': 'html_form', 'text': msg})
return False
unique_resource_name = '%(HOSTURL)s.%(HOSTIDENTIFIER)s' % user_vars
output_objects.append({
'object_type':
'text',
'text':
'Updated %s resource configuration!' % unique_resource_name
})
output_objects.append({
'object_type':
'link',
'destination':
'resadmin.py?unique_resource_name=%s' % unique_resource_name,
'class':
'adminlink iconspace',
'title':
'Administrate resource',
'text':
'Manage resource',
})
return True
elif configuration.auto_add_resource:
resource_name = user_vars['HOSTURL']
logger.info('Auto creating resource %s from %s' %
(resource_id, pending_file))
(create_status, msg) = create_resource(configuration, client_id,
resource_name, pending_file)
if not create_status:
output_objects.append({
'object_type': 'error_text',
'text': 'Resource creation failed:'
})
output_objects.append({'object_type': 'html_form', 'text': msg})
return False
output += '''Your resource was added as %s.%s
<hr />''' % (resource_name, msg)
else:
logger.info('Parsing conf %s for %s' % (pending_file, resource_id))
(run_status, msg) = confparser.run(configuration, pending_file,
resource_id, '')
if not run_status:
logger.error(msg)
output_objects.append({
'object_type': 'error_text',
'text': 'Failed to parse new configuration:'
})
output_objects.append({'object_type': 'html_form', 'text': msg})
try:
os.remove(pending_file)
except:
pass
return False
logger.info('Sending create request for %s to admins' % resource_id)
(send_status, msg) = send_resource_create_request_mail(
client_id, user_vars['HOSTURL'], pending_file, logger,
configuration)
logger.info(msg)
if not send_status:
output_objects.append({
'object_type':
'error_text',
'text':
'''Failed to send request with ID "%s" to
the %s administrator(s):
%s
Please manually contact the %s site administrator(s) (%s)
and provide this information''' %
(tmp_id, msg, configuration.short_title,
configuration.short_title, configuration.admin_email)
})
return False
output += """Your creation request of the resource: <b>%s</b>
has been sent to the %s server administration and will be processed as
soon as possible.
<hr />""" % (user_vars['HOSTURL'], configuration.short_title) \
public_key_file_content = ''
try:
key_fh = open(configuration.public_key_file, 'r')
public_key_file_content = key_fh.read()
key_fh.close()
except:
public_key_file_content = None
# Avoid line breaks in displayed key
if public_key_file_content:
public_key_info = '''
The public key you must add:<br />
***BEGIN KEY***<br />
%s<br />
***END KEY***<br />
<br />''' % public_key_file_content.replace(' ', ' ')
else:
public_key_info = '''<br />
Please request an SSH public key from the %s administrator(s) (%s)<br />
<br />''' % (configuration.short_title, html_escape(configuration.admin_email))
output += """
Please make sure the %s server can SSH to your resource without a passphrase.
The %s server's public key should be in ~/.ssh/authorized_keys for the MiG
user on the resource frontend. %s
<br />
Also, please note that %s resources require the curl command line tool from
<a href='http://www.curl.haxx.se'>cURL</a>.
<br />
<a href='resadmin.py'>View existing resources</a> where your new resource
will also eventually show up.
""" % (configuration.short_title, configuration.short_title, public_key_info,
configuration.short_title)
output_objects.append({'object_type': 'html_form', 'text': output})
return True