def handle_proxy(proxy_string, client_id, config):
"""If ARC-enabled server: store a proxy certificate.
Arguments: proxy_string - text extracted from given upload
client_id - DN for user just being created
config - global configuration
"""
output = []
client_dir = client_id_dir(client_id)
proxy_dir = os.path.join(config.user_home, client_dir)
proxy_path = os.path.join(config.user_home, client_dir, arc.Ui.proxy_name)
if not config.arc_clusters:
output.append({'object_type': 'error_text', 'text':
'No ARC support!'})
return output
# store the file
try:
write_file(proxy_string, proxy_path, config.logger)
os.chmod(proxy_path, 0600)
except Exception, exc:
output.append({'object_type': 'error_text', 'text'
: 'Proxy file could not be written (%s)!'
% str(exc).replace(proxy_dir, '')})
return output
def handle_proxy(proxy_string, client_id, config):
"""If ARC-enabled server: store a proxy certificate.
Arguments: proxy_string - text extracted from given upload
client_id - DN for user just being created
config - global configuration
"""
output = []
client_dir = client_id_dir(client_id)
proxy_dir = os.path.join(config.user_home, client_dir)
proxy_path = os.path.join(config.user_home, client_dir, arc.Ui.proxy_name)
if not config.arc_clusters:
output.append({'object_type': 'error_text', 'text': 'No ARC support!'})
return output
# store the file
try:
write_file(proxy_string, proxy_path, config.logger)
os.chmod(proxy_path, 0600)
except Exception, exc:
output.append({
'object_type':
'error_text',
'text':
'Proxy file could not be written (%s)!' %
str(exc).replace(proxy_dir, '')
})
return output
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
unique_resource_name = accepted['unique_resource_name'][-1]
resconfig = accepted['resconfig'][-1]
output_objects.append({'object_type': 'header', 'text'
: 'Trying to Update resource configuration'})
if not is_owner(client_id, unique_resource_name,
configuration.resource_home, logger):
logger.error(client_id + ' is not an owner of '
+ unique_resource_name + ': update rejected!')
output_objects.append({'object_type': 'error_text', 'text'
: 'You must be an owner of '
+ unique_resource_name
+ ' to update the configuration!'})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: race if two confs are uploaded concurrently!
host_url, host_identifier = unique_resource_name.rsplit('.', 1)
pending_file = os.path.join(configuration.resource_home,
unique_resource_name, 'config.tmp')
# write new proposed config file to disk
try:
logger.info('write to file: %s' % pending_file)
if not write_file(resconfig, pending_file, logger):
output_objects.append({'object_type': 'error_text',
'text': 'Could not write: %s' % pending_file})
return (output_objects, returnvalues.SYSTEM_ERROR)
except Exception, err:
logger.error('Resource conf %s could not be written: %s' % \
(pending_file, err))
output_objects.append({'object_type': 'error_text', 'text':
'Could not write configuration!'})
return (output_objects, returnvalues.SYSTEM_ERROR)
def write_xml(self, filename):
""" Writes the Usage Record to a file as XML """
try:
xml = self.generate_tree()
result = write_file(xml, filename, self.__logger)
except Exception, err:
self.__logger.error('Unable to write XML file: %s' % err)
def write_xml(self, filename):
""" Writes the Usage Record to a file as XML """
try:
xml = self.generate_tree()
result = write_file(xml, filename, self.__logger)
except Exception, err:
self.__logger.error('Unable to write XML file: %s' % err)
def create_verify_files(types, re_name, re_dict, base_dir, logger):
"""Create runtime env test files"""
for ver_type in types:
if re_dict.has_key('VERIFY%s' % ver_type.upper()):
if re_dict['VERIFY%s' % ver_type.upper()] != []:
file_content = ''
for line in re_dict['VERIFY%s' % ver_type.upper()]:
file_content += line + '\n'
if not write_file(
file_content.strip(), '%sverify_runtime_env_%s.%s' %
(base_dir, re_name, ver_type.lower()), logger):
raise Exception('could not write test job %s' % \
ver_type.upper())
def create_verify_files(types, re_name, re_dict, base_dir, logger):
"""Create runtime env test files"""
for ver_type in types:
if re_dict.has_key('VERIFY%s' % ver_type.upper()):
if re_dict['VERIFY%s' % ver_type.upper()] != []:
file_content = ''
for line in re_dict['VERIFY%s' % ver_type.upper()]:
file_content += line + '\n'
if not write_file(file_content.strip(),
'%sverify_runtime_env_%s.%s'
% (base_dir, re_name,
ver_type.lower()), logger):
raise Exception('could not write test job %s' % \
ver_type.upper())
except Exception, err:
# error during translation, pass a message
logger.error("Error during xRSL translation: %s" % err.__str__())
return (None, err.__str__())
# we submit directly from here (the other version above does
# copyFileToResource and gen_job_script generates all files)
# we have to put the generated script somewhere..., and submit from there.
# inputfiles are given by the user as relative paths from his home,
# so we should use that location (and clean up afterwards).
# write script (to user home)
user_home = os.path.join(configuration.user_home, client_dir)
script_path = os.path.abspath(os.path.join(user_home, script_name))
write_file(script, script_path, logger)
os.chdir(user_home)
try:
logger.debug("submitting job to ARC")
session = arc.Ui(user_home)
arc_job_ids = session.submit(xrsl)
# if no exception occurred, we are done:
job_dict["ARCID"] = arc_job_ids[0]
job_dict["SESSIONID"] = sessionid
msg = "OK"
result = job_dict
def handle_package_upload(
real_src,
relative_src,
client_id,
configuration,
submit_mrslfiles,
dst,
):
"""A file package was uploaded (eg. .zip file). Extract the content and
submit mrsl files if submit_mrsl_files is True.
"""
logger = configuration.logger
msg = ''
status = True
logger.info("handle_package_upload %s %s %s" % \
(real_src, relative_src, dst))
client_dir = client_id_dir(client_id)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# Unpack in same directory unless real_dst is given
if not dst:
real_dst = os.path.abspath(os.path.dirname(real_src))
elif os.path.isabs(dst):
real_dst = os.path.abspath(dst)
else:
real_dst = os.path.join(base_dir, dst)
real_dst += os.sep
mrslfiles_to_parse = []
real_src_lower = real_src.lower()
if real_src_lower.endswith('.zip'):
# Handle .zip file
msg += "Received '%s' for unpacking. " % relative_src
try:
zip_object = zipfile.ZipFile(real_src, 'r')
except Exception, exc:
logger.error("open zip failed: %s" % exc)
msg += 'Could not open zipfile: %s! ' % exc
return (False, msg)
logger.info("unpack entries of %s to %s" % \
(real_src, real_dst))
for zip_entry in zip_object.infolist():
msg += 'Extracting: %s . ' % zip_entry.filename
# write zip_entry to disk
local_zip_entry_name = os.path.join(real_dst, zip_entry.filename)
valid_status, valid_err = valid_user_path_name(
zip_entry.filename, local_zip_entry_name, base_dir)
if not valid_status:
status = False
msg += "Filename validation error: %s! " % valid_err
continue
# create sub dir(s) if missing
zip_entry_dir = os.path.dirname(local_zip_entry_name)
if not os.path.isdir(zip_entry_dir):
msg += 'Creating dir %s . ' % zip_entry.filename
try:
os.makedirs(zip_entry_dir, 0775)
except Exception, exc:
logger.error("create directory failed: %s" % exc)
msg += 'Error creating directory: %s! ' % exc
status = False
continue
if os.path.isdir(local_zip_entry_name):
logger.info("nothing more to do for dir entry: %s" % \
local_zip_entry_name)
continue
# TODO: can we detect and ignore symlinks?
# Zip format is horribly designed/documented:
# http://www.pkware.com/documents/casestudies/APPNOTE.TXT
# I haven't managed to find a way to detect symlinks. Thus
# they are simply created as files containing the name they
# were supposed to link to: This is inconsistent but safe :-S
# write file - symbolic links are written as files! (good for
# security)
if not write_file(zip_object.read(zip_entry.filename),
local_zip_entry_name,
logger) and \
not os.path.exists(local_zip_entry_name):
msg += 'Error unpacking %s to disk! ' % zip_entry.filename
status = False
continue
# get the size as the OS sees it
try:
__ = os.path.getsize(local_zip_entry_name)
except Exception, exc:
logger.warning("unpack may have failed: %s" % exc)
msg += \
'File %s unpacked, but could not get file size %s! '\
% (zip_entry.filename, exc)
status = False
continue
continue
elif not tar_entry.isfile():
# not a regular file - symlinks are ignored to avoid illegal
# access
msg += 'Skipping %s: not a regular file or directory! ' % \
tar_entry.name
status = False
continue
# write file!
if not write_file(tar_file_content.extractfile(tar_entry).read(),
local_tar_entry_name,
logger):
msg += 'Error unpacking file %s to disk! ' % tar_entry.name
status = False
continue
# get the size as the OS sees it
try:
__ = os.path.getsize(local_tar_entry_name)
except Exception, exc:
logger.warning("file save may have failed: %s" % exc)
msg += \
'File %s unpacked, but could not get file size %s! ' % \
(tar_entry.name, exc)
status = False
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_title=True, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# TODO: all non-file fields should be validated!!
# Input fields are mostly file stuff so do not validate it
validate_args = dict([(key, user_arguments_dict.get(key, val)) for \
(key, val) in defaults.items()])
(validate_status, accepted) = validate_input_and_cert(
validate_args,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({'object_type': 'header', 'text'
: '%s submit job/file' % configuration.short_title})
submitstatuslist = []
fileuploadobjs = []
filenumber = 0
file_fields = int(accepted.get('file_fields', -1)[-1])
save_as_default = (accepted['save_as_default'][-1] != 'False')
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
mrsl = ''
while True:
(content, file_type) = handle_form_input(filenumber,
user_arguments_dict, configuration)
if not content:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
# no field count and no data for filenumber found
break
# always append mrsltextarea if available!
try:
mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0]
content += mrsl
except:
pass
content += '\n'
mrslfiles_to_parse = []
submit_mrslfiles = False
submitmrsl_key = 'submitmrsl_%s' % filenumber
if user_arguments_dict.has_key(submitmrsl_key):
val = str(user_arguments_dict[submitmrsl_key][0]).upper()
if val == 'ON' or val == 'TRUE':
submit_mrslfiles = True
fileuploadobj = {'object_type': 'fileuploadobj',
'submitmrsl': submit_mrslfiles}
if file_type == 'plain':
# get filename
filename_key = 'FILENAME_%s' % filenumber
if not user_arguments_dict.has_key(filename_key):
output_objects.append(
{'object_type': 'error_text','text'
: ("The specified file_type is 'plain', but a filename" \
"value was not found. The missing control should be " \
"named %s") % filename_key})
return (output_objects, returnvalues.CLIENT_ERROR)
filename_val = convert_control_value_to_line(filename_key,
user_arguments_dict)
if not filename_val:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No filename found - please make sure you provide a " \
"file to upload'})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = base_dir + filename_val
valid_status, valid_err = valid_user_path_name(filename_val,
local_filename,
base_dir)
if not valid_status:
output_objects.append(
{'object_type': 'error_text', 'text': valid_err})
return (output_objects, returnvalues.CLIENT_ERROR)
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append({'object_type': 'error_text',
'text': 'Could not write: %s' % local_filename})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['saved'] = True
# msg += "%s created!" % local_filename
fileuploadobj['name'] = os.sep\
+ convert_control_value_to_line(filename_key,
user_arguments_dict)
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
mrslfiles_to_parse.append[local_filename]
elif file_type == 'fileupload':
# An input type=file was found
fileupload_key = 'fileupload_%s_0_0' % filenumber
# if not fileitem.filename:
if not user_arguments_dict.has_key(fileupload_key
+ 'filename'):
output_objects.append({'object_type': 'error_text',
'text': 'NO FILENAME error'})
return (output_objects, returnvalues.CLIENT_ERROR)
base_name = strip_dir(user_arguments_dict[fileupload_key
+ 'filename'])
if not base_name:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
# output_objects.append({'object_type': 'text', 'text':
# 'skip item %d' % filenumber})
filenumber += 1
continue
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No filename found - please make sure you provide a " \
"file to upload'})
return (output_objects, returnvalues.CLIENT_ERROR)
extract_packages = False
extract_key = 'extract_%s' % filenumber
if user_arguments_dict.has_key(extract_key):
val = str(user_arguments_dict[extract_key][0]).upper()
if val == 'ON' or val == 'TRUE':
extract_packages = True
remote_filename = ''
default_remotefilename_key = 'default_remotefilename_%s'\
% filenumber
if user_arguments_dict.has_key(default_remotefilename_key):
remote_filename = \
user_arguments_dict[default_remotefilename_key][0]
# remotefilename overwrites default_remotefilename if it exists
remotefilename_key = 'remotefilename_%s' % filenumber
if user_arguments_dict.has_key(remotefilename_key):
remote_filename = \
user_arguments_dict[remotefilename_key][0]
if not remote_filename:
remote_filename = base_name
# if remote_filename is a directory, use client's local filename
# for the last part of the filename
if remote_filename.strip().endswith(os.sep):
remote_filename += base_name
if not user_arguments_dict.has_key(fileupload_key):
output_objects.append({'object_type': 'error_text',
'text': 'File content not found!'})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = os.path.abspath(base_dir + remote_filename)
valid_status, valid_err = valid_user_path_name(remote_filename,
local_filename,
base_dir)
if not valid_status:
output_objects.append(
{'object_type': 'error_text', 'text': valid_err})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(local_filename)):
try:
os.makedirs(os.path.dirname(local_filename), 0777)
except Exception:
fileuploadobj['message'] = \
{'object_type': 'error_text',
'text': 'Exception creating dirs %s'\
% os.path.dirname(local_filename)}
fileuploadobj['name'] = remote_filename
# reads uploaded file into memory
binary = user_arguments_dict.has_key('%s_is_encoded'
% fileupload_key)
if binary:
data = user_arguments_dict[fileupload_key][-1]
data = str(base64.decodestring(data))
else:
data = user_arguments_dict[fileupload_key][-1]
# write file in memory to disk
if not write_file(data, local_filename,
configuration.logger):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error writing file in memory to disk'})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['saved'] = True
# Tell the client about the current settings (extract and submit)
# extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): "
# if extract_packages:
# extract_str += "ON"
# else:
# extract_str += "OFF"
# output_objects.append({"object_type":"text", "text":extract_str})
fileuploadobj['extract_packages'] = extract_packages
# submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): "
# if submit_mrslfiles:
# submit_str += "ON"
# else:
# submit_str += "OFF"
# output_objects.append({"object_type":"text", "text":submit_str})
# handle file package
if extract_packages\
and (local_filename.upper().endswith('.ZIP')
or local_filename.upper().endswith('.TAR.GZ')
or local_filename.upper().endswith('.TGZ')
or local_filename.upper().endswith('.TAR.BZ2')):
(status, msg) = handle_package_upload(local_filename,
remote_filename, client_id, configuration,
submit_mrslfiles, os.path.dirname(local_filename))
if status:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error in submit: %s' % msg})
else:
submitstatuslist = msg
else:
output_objects.append({'object_type': 'text',
'text': msg})
else:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error in unpack: %s' % msg})
else:
submitstatuslist = msg
else:
output_objects.append({'object_type': 'error_text',
'text': 'Problems unpacking: %s' % msg})
else:
# output_objects.append({"object_type":"text", "text":msg})
# a "normal" (non-package) file was uploaded
try:
output_objects.append({'object_type': 'text', 'text'
: 'File saved: %s' % remote_filename})
except Exception, err:
output_objects.append({'object_type': 'error_text',
'text'
: 'File seems to be saved, but could not get file size %s'
% err})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['size'] = os.path.getsize(local_filename)
fileuploadobj['name'] = remote_filename
# Check if the extension is .mRSL
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
# A .mrsl file was uploaded!
# output_objects.append({"object_type":"text", "text":
# "File name on MiG server: %s"
# % (remote_filename)})
mrslfiles_to_parse.append(local_filename)
else:
# mrsl file created by html controls. create filename. Loop until
# a filename that do not exits is created
html_generated_mrsl_dir = base_dir + 'html_generated_mrsl'
if os.path.exists(html_generated_mrsl_dir)\
and not os.path.isdir(html_generated_mrsl_dir):
# oops, user might have created a file with the same name
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Please make sure %s does not exist or is a directory!'
% 'html_generated_mrsl/'})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(html_generated_mrsl_dir):
os.mkdir(html_generated_mrsl_dir)
while True:
time_c = time.gmtime()
timestamp = '%s_%s_%s__%s_%s_%s' % (
time_c[1],
time_c[2],
time_c[0],
time_c[3],
time_c[4],
time_c[5],
)
local_filename = html_generated_mrsl_dir\
+ '/TextAreaAt_' + timestamp + '.mRSL'
if not os.path.isfile(local_filename):
break
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append(
{'object_type': 'error_text',
'text': 'Could not write: %s' % local_filename})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['name'] = os.sep\
+ 'html_generated_mrsl/TextAreaAt_' + timestamp\
+ '.mRSL'
fileuploadobj['size'] = os.path.getsize(local_filename)
mrslfiles_to_parse.append(local_filename)
fileuploadobjs.append(fileuploadobj)
# Submit selected file(s)
for mrslfile in mrslfiles_to_parse:
# do not reveal full path of mrsl file to client
relative_filename = os.sep + mrslfile.replace(base_dir, '')
submitstatus = {'object_type': 'submitstatus',
'name': relative_filename}
(status, newmsg, job_id) = new_job(mrslfile, client_id,
configuration, False, True)
if not status:
# output_objects.append({"object_type":"error_text", "text":"%s"
# % newmsg})
submitstatus['status'] = False
submitstatus['message'] = newmsg
else:
# return (output_objects, returnvalues.CLIENT_ERROR)
submitstatus['status'] = True
submitstatus['job_id'] = job_id
# output_objects.append({"object_type":"text", "text":"%s"
# % newmsg})
submitstatuslist.append(submitstatus)
# prepare next loop
filenumber += 1
if __name__ == '__main__':
print 'starting translation test. Args: ' , len(sys.argv)
logger.debug('translation for file ' + sys.argv[1] + ' starts')
if len(sys.argv) > 1:
fname = sys.argv[1]
parsed = '.'.join([fname,'parsed'])
translated = '.'.join([parsed,'xrsl'])
try:
import shared.mrslparser as p
import shared.fileio as fileio
(presult,errors) = p.parse(fname, 'test-id',
'+No+Client+Id',None,parsed)
if not presult:
print 'Errors:\n%s' % errors
else:
print 'Parsing OK, now translating'
mrsl_dict = fileio.unpickle(parsed,logger)
(xrsl,script,name) = translate(mrsl_dict,'test-name')
print '\n'.join(['Job name',name,'script',script,'XRSL'])
fileio.write_file(script, "test-id.sh", logger)
print (format_xrsl(xrsl))
fileio.write_file("%s" % xrsl, translated, logger)
print 'done'
except Exception, err:
print 'Error.'
print err.__str__()
logger.info("%s %s of %s: %s" %
(op_name, hash_algo, abs_path, checksum))
output_lines.append(line)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
"%s: '%s': %s" % (op_name, relative_path, exc)
})
logger.error("%s: failed on '%s': %s" %
(op_name, relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
entry = {'object_type': 'file_output', 'lines': output_lines}
output_objects.append(entry)
all_lines += output_lines
if dst and not write_file(''.join(all_lines), abs_dest, logger):
output_objects.append({
'object_type':
'error_text',
'text':
"failed to write checksums to %s" % relative_dest
})
logger.error("writing checksums to %s for %s failed" %
(abs_dest, client_id))
status = returnvalues.SYSTEM_ERROR
return (output_objects, status)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_title=True, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# TODO: do we need to cover more non-file fields?
# All non-file fields must be validated
validate_args = dict([(key, user_arguments_dict.get(key, val)) for \
(key, val) in defaults.items()])
# IMPORTANT: we must explicitly inlude CSRF token
validate_args[csrf_field] = user_arguments_dict.get(
csrf_field, ['AllowMe'])
(validate_status, accepted) = validate_input_and_cert(
validate_args,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'header',
'text':
'%s file handling' % configuration.short_title
})
submitstatuslist = []
fileuploadobjs = []
filenumber = 0
file_fields = int(accepted.get('file_fields', -1)[-1])
save_as_default = (accepted['save_as_default'][-1] != 'False')
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
mrsl = ''
while True:
(content, file_type) = handle_form_input(filenumber,
user_arguments_dict,
configuration)
if not content:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
# no field count and no data for filenumber found
break
# always append mrsltextarea if available!
try:
mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0]
content += mrsl
except:
pass
content += '\n'
mrslfiles_to_parse = []
submit_mrslfiles = False
submitmrsl_key = 'submitmrsl_%s' % filenumber
if configuration.site_enable_jobs and \
user_arguments_dict.has_key(submitmrsl_key):
val = str(user_arguments_dict[submitmrsl_key][0]).upper()
if val == 'ON' or val == 'TRUE':
submit_mrslfiles = True
fileuploadobj = {
'object_type': 'fileuploadobj',
'submitmrsl': submit_mrslfiles
}
if file_type == 'plain':
# get filename
filename_key = 'FILENAME_%s' % filenumber
if not user_arguments_dict.has_key(filename_key):
output_objects.append(
{'object_type': 'error_text','text'
: ("The specified file_type is 'plain', but a filename" \
"value was not found. The missing control should be " \
"named %s") % filename_key})
return (output_objects, returnvalues.CLIENT_ERROR)
filename_val = convert_control_value_to_line(
filename_key, user_arguments_dict)
if not filename_val:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
output_objects.append({
'object_type':
'error_text',
'text':
'No filename found - please make sure you provide a " \
"file to upload'
})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = base_dir + filename_val
valid_status, valid_err = valid_user_path_name(
filename_val, local_filename, base_dir)
if not valid_status:
output_objects.append({
'object_type': 'error_text',
'text': valid_err
})
return (output_objects, returnvalues.CLIENT_ERROR)
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
logger.error("%s failed to write plain file %s" % \
(op_name, local_filename))
output_objects.append({
'object_type':
'error_text',
'text':
'Could not write: %s' % local_filename
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.info("%s wrote plain file %s" % (op_name, local_filename))
fileuploadobj['saved'] = True
# msg += "%s created!" % local_filename
fileuploadobj['name'] = os.sep\
+ convert_control_value_to_line(filename_key,
user_arguments_dict)
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
mrslfiles_to_parse.append(local_filename)
elif file_type == 'fileupload':
# An input type=file was found
fileupload_key = 'fileupload_%s_0_0' % filenumber
# if not fileitem.filename:
if not user_arguments_dict.has_key(fileupload_key + 'filename'):
output_objects.append({
'object_type': 'error_text',
'text': 'NO FILENAME error'
})
return (output_objects, returnvalues.CLIENT_ERROR)
base_name = strip_dir(user_arguments_dict[fileupload_key +
'filename'])
if not base_name:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
# output_objects.append({'object_type': 'text', 'text':
# 'skip item %d' % filenumber})
filenumber += 1
continue
output_objects.append({
'object_type':
'error_text',
'text':
'No filename found - please make sure you provide a " \
"file to upload'
})
return (output_objects, returnvalues.CLIENT_ERROR)
extract_packages = False
extract_key = 'extract_%s' % filenumber
if user_arguments_dict.has_key(extract_key):
val = str(user_arguments_dict[extract_key][0]).upper()
if val == 'ON' or val == 'TRUE':
extract_packages = True
remote_filename = ''
default_remotefilename_key = 'default_remotefilename_%s'\
% filenumber
if user_arguments_dict.has_key(default_remotefilename_key):
remote_filename = \
user_arguments_dict[default_remotefilename_key][0]
# remotefilename overwrites default_remotefilename if it exists
remotefilename_key = 'remotefilename_%s' % filenumber
if user_arguments_dict.has_key(remotefilename_key):
remote_filename = \
user_arguments_dict[remotefilename_key][0]
if not remote_filename:
remote_filename = base_name
# if remote_filename is a directory, use client's local filename
# for the last part of the filename
if remote_filename.strip().endswith(os.sep):
remote_filename += base_name
if not user_arguments_dict.has_key(fileupload_key):
output_objects.append({
'object_type': 'error_text',
'text': 'File content not found!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = os.path.abspath(base_dir + remote_filename)
valid_status, valid_err = valid_user_path_name(
remote_filename, local_filename, base_dir)
if not valid_status:
output_objects.append({
'object_type': 'error_text',
'text': valid_err
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(local_filename)):
try:
os.makedirs(os.path.dirname(local_filename), 0775)
except Exception:
fileuploadobj['message'] = \
{'object_type': 'error_text',
'text': 'Exception creating dirs %s'\
% os.path.dirname(local_filename)}
fileuploadobj['name'] = remote_filename
# reads uploaded file into memory
binary = user_arguments_dict.has_key('%s_is_encoded' %
fileupload_key)
if binary:
data = user_arguments_dict[fileupload_key][-1]
data = str(base64.decodestring(data))
else:
data = user_arguments_dict[fileupload_key][-1]
# write file in memory to disk
if not write_file(data, local_filename, configuration.logger):
logger.error("%s failed to write upload file %s" % \
(op_name, local_filename))
output_objects.append({
'object_type':
'error_text',
'text':
'Error writing file in memory to disk'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.info("%s wrote upload file %s" % (op_name, local_filename))
fileuploadobj['saved'] = True
# Tell the client about the current settings (extract and submit)
# extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): "
# if extract_packages:
# extract_str += "ON"
# else:
# extract_str += "OFF"
# output_objects.append({"object_type":"text", "text":extract_str})
fileuploadobj['extract_packages'] = extract_packages
# submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): "
# if submit_mrslfiles:
# submit_str += "ON"
# else:
# submit_str += "OFF"
# output_objects.append({"object_type":"text", "text":submit_str})
# handle file package
if extract_packages\
and (local_filename.upper().endswith('.ZIP')
or local_filename.upper().endswith('.TAR.GZ')
or local_filename.upper().endswith('.TGZ')
or local_filename.upper().endswith('.TAR.BZ2')):
(upload_status,
msg) = handle_package_upload(local_filename, remote_filename,
client_id, configuration,
submit_mrslfiles,
os.path.dirname(local_filename))
if upload_status:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append({
'object_type':
'error_text',
'text':
'Error in submit: %s' % msg
})
else:
submitstatuslist = msg
else:
output_objects.append({
'object_type': 'text',
'text': msg
})
else:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append({
'object_type':
'error_text',
'text':
'Error in unpack: %s' % msg
})
else:
submitstatuslist = msg
else:
output_objects.append({
'object_type':
'error_text',
'text':
'Problems unpacking: %s' % msg
})
else:
# output_objects.append({"object_type":"text", "text":msg})
# a "normal" (non-package) file was uploaded
try:
output_objects.append({
'object_type':
'text',
'text':
'File saved: %s' % remote_filename
})
except Exception, err:
output_objects.append({
'object_type':
'error_text',
'text':
'File seems to be saved, but could not get file size %s'
% err
})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['size'] = os.path.getsize(local_filename)
fileuploadobj['name'] = remote_filename
# Check if the extension is .mRSL
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
# A .mrsl file was uploaded!
# output_objects.append({"object_type":"text", "text":
# "File name on MiG server: %s"
# % (remote_filename)})
mrslfiles_to_parse.append(local_filename)
else:
# mrsl file created by html controls. create filename. Loop until
# a filename that do not exits is created
html_generated_mrsl_dir = base_dir + 'html_generated_mrsl'
if os.path.exists(html_generated_mrsl_dir)\
and not os.path.isdir(html_generated_mrsl_dir):
# oops, user might have created a file with the same name
output_objects.append({
'object_type':
'error_text',
'text':
'Please make sure %s does not exist or is a directory!' %
'html_generated_mrsl/'
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(html_generated_mrsl_dir):
os.mkdir(html_generated_mrsl_dir)
while True:
time_c = time.gmtime()
timestamp = '%s_%s_%s__%s_%s_%s' % (
time_c[1],
time_c[2],
time_c[0],
time_c[3],
time_c[4],
time_c[5],
)
local_filename = html_generated_mrsl_dir\
+ '/TextAreaAt_' + timestamp + '.mRSL'
if not os.path.isfile(local_filename):
break
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'Could not write: %s' % local_filename
})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['name'] = os.sep\
+ 'html_generated_mrsl/TextAreaAt_' + timestamp\
+ '.mRSL'
fileuploadobj['size'] = os.path.getsize(local_filename)
mrslfiles_to_parse.append(local_filename)
fileuploadobjs.append(fileuploadobj)
# Submit selected file(s)
for mrslfile in mrslfiles_to_parse:
# do not reveal full path of mrsl file to client
relative_filename = os.sep + mrslfile.replace(base_dir, '')
submitstatus = {
'object_type': 'submitstatus',
'name': relative_filename
}
(new_status, newmsg, job_id) = new_job(mrslfile, client_id,
configuration, False, True)
if not new_status:
# output_objects.append({"object_type":"error_text", "text":"%s"
# % newmsg})
submitstatus['status'] = False
submitstatus['message'] = newmsg
else:
# return (output_objects, returnvalues.CLIENT_ERROR)
submitstatus['status'] = True
submitstatus['job_id'] = job_id
# output_objects.append({"object_type":"text", "text":"%s"
# % newmsg})
submitstatuslist.append(submitstatus)
# prepare next loop
filenumber += 1
def gen_job_script(
job_dictionary, resource_config, configuration, localjobname, path_without_extension, client_dir, exe, logger
):
"""Generate job script from job_dictionary before handout to resource"""
script_language = resource_config["SCRIPTLANGUAGE"]
if not script_language in configuration.scriptlanguages:
print "Unknown script language! (conflict with scriptlanguages in " + "configuration?) %s not in %s" % (
script_language,
configuration.scriptlanguages,
)
return False
if script_language == "python":
generator = genjobscriptpython.GenJobScriptPython(
job_dictionary,
resource_config,
exe,
configuration.migserver_https_sid_url,
localjobname,
path_without_extension,
)
elif script_language == "sh":
generator = genjobscriptsh.GenJobScriptSh(
job_dictionary,
resource_config,
exe,
configuration.migserver_https_sid_url,
localjobname,
path_without_extension,
)
elif script_language == "java":
generator = genjobscriptjava.GenJobScriptJava(
job_dictionary, resource_config, configuration.migserver_https_sid_url, localjobname, path_without_extension
)
else:
print "Unknown script language! (is in configuration but not in " + "jobscriptgenerator) %s " % script_language
return False
# String concatenation in python: [X].join is much faster
# than repeated use of s += strings
getinputfiles_array = []
getinputfiles_array.append(generator.script_init())
getinputfiles_array.append(generator.comment("print start"))
getinputfiles_array.append(generator.print_start("get input files"))
getinputfiles_array.append(generator.comment("init log"))
getinputfiles_array.append(generator.init_io_log())
getinputfiles_array.append(generator.comment("get special inputfiles"))
getinputfiles_array.append(generator.get_special_input_files("get_special_status"))
getinputfiles_array.append(generator.log_io_status("get_special_input_files", "get_special_status"))
getinputfiles_array.append(
generator.print_on_error("get_special_status", "0", "failed to fetch special input files!")
)
getinputfiles_array.append(generator.comment("get input files"))
getinputfiles_array.append(generator.get_input_files("get_input_status"))
getinputfiles_array.append(generator.log_io_status("get_input_files", "get_input_status"))
getinputfiles_array.append(generator.print_on_error("get_input_status", "0", "failed to fetch input files!"))
getinputfiles_array.append(generator.comment("get executables"))
getinputfiles_array.append(generator.get_executables("get_executables_status"))
getinputfiles_array.append(generator.log_io_status("get_executables", "get_executables_status"))
getinputfiles_array.append(
generator.print_on_error("get_executables_status", "0", "failed to fetch executable files!")
)
# client_dir equals empty_job_name for sleep jobs
getinputfiles_array.append(
generator.generate_output_filelists(client_dir != configuration.empty_job_name, "generate_output_filelists")
)
getinputfiles_array.append(
generator.print_on_error("generate_output_filelists", "0", "failed to generate output filelists!")
)
getinputfiles_array.append(generator.generate_input_filelist("generate_input_filelist"))
getinputfiles_array.append(
generator.print_on_error("generate_input_filelist", "0", "failed to generate input filelist!")
)
getinputfiles_array.append(generator.generate_iosessionid_file("generate_iosessionid_file"))
getinputfiles_array.append(
generator.print_on_error("generate_iosessionid_file", "0", "failed to generate iosessionid file!")
)
getinputfiles_array.append(generator.generate_mountsshprivatekey_file("generate_mountsshprivatekey_file"))
getinputfiles_array.append(
generator.print_on_error("generate_mountsshprivatekey_file", "0", "failed to generate mountsshprivatekey file!")
)
getinputfiles_array.append(generator.generate_mountsshknownhosts_file("generate_mountsshknownhosts_file"))
getinputfiles_array.append(
generator.print_on_error("generate_mountsshknownhosts_file", "0", "failed to generate mountsshknownhosts file!")
)
getinputfiles_array.append(
generator.total_status(
["get_special_status", "get_input_status", "get_executables_status", "generate_output_filelists"],
"total_status",
)
)
getinputfiles_array.append(generator.exit_on_error("total_status", "0", "total_status"))
getinputfiles_array.append(generator.comment("exit script"))
getinputfiles_array.append(generator.exit_script("0", "get input files"))
job_array = []
job_array.append(generator.script_init())
job_array.append(generator.set_core_environments())
job_array.append(generator.print_start("job"))
job_array.append(generator.comment("TODO: switch to job directory here"))
job_array.append(generator.comment("make sure job status files exist"))
job_array.append(
generator.create_files(
[
job_dictionary["JOB_ID"] + ".stdout",
job_dictionary["JOB_ID"] + ".stderr",
job_dictionary["JOB_ID"] + ".status",
]
)
)
job_array.append(generator.init_status())
job_array.append(generator.comment("chmod +x"))
job_array.append(generator.chmod_executables("chmod_status"))
job_array.append(generator.print_on_error("chmod_status", "0", "failed to make one or more EXECUTABLES executable"))
job_array.append(generator.log_on_error("chmod_status", "0", "system: chmod"))
job_array.append(generator.comment("set environments"))
job_array.append(generator.set_environments("env_status"))
job_array.append(generator.print_on_error("env_status", "0", "failed to initialize one or more ENVIRONMENTs"))
job_array.append(generator.log_on_error("env_status", "0", "system: set environments"))
job_array.append(generator.comment("set runtimeenvironments"))
job_array.append(generator.set_runtime_environments(resource_config["RUNTIMEENVIRONMENT"], "re_status"))
job_array.append(generator.print_on_error("re_status", "0", "failed to initialize one or more RUNTIMEENVIRONMENTs"))
job_array.append(generator.log_on_error("re_status", "0", "system: set RUNTIMEENVIRONMENTs"))
job_array.append(generator.comment("enforce some basic job limits"))
job_array.append(generator.set_limits())
if job_dictionary.get("MOUNT", []) != []:
job_array.append(generator.comment("Mount job home"))
job_array.append(
generator.mount(
job_dictionary["SESSIONID"],
configuration.user_sftp_show_address,
configuration.user_sftp_show_port,
"mount_status",
)
)
job_array.append(generator.print_on_error("mount_status", "0", "failded to mount job home"))
job_array.append(generator.log_on_error("mount_status", "0", "system: mount"))
job_array.append(generator.comment("execute!"))
job_array.append(generator.execute("EXECUTING: ", "--Exit code:"))
if job_dictionary.get("MOUNT", []) != []:
job_array.append(generator.comment("Unmount job home"))
job_array.append(generator.umount("umount_status"))
job_array.append(generator.print_on_error("umount_status", "0", "failded to umount job home"))
job_array.append(generator.log_on_error("umount_status", "0", "system: umount"))
job_array.append(generator.comment("exit script"))
job_array.append(generator.exit_script("0", "job"))
getupdatefiles_array = []
# We need to make sure that curl failures lead to retry while
# missing output (from say a failed job) is logged but
# ignored in relation to getupdatefiles success.
getupdatefiles_array.append(generator.print_start("get update files"))
getupdatefiles_array.append(generator.init_io_log())
getupdatefiles_array.append(generator.comment("get io files"))
getupdatefiles_array.append(generator.get_io_files("get_io_status"))
getupdatefiles_array.append(generator.log_io_status("get_io_files", "get_io_status"))
getupdatefiles_array.append(generator.print_on_error("get_io_status", "0", "failed to get one or more IO files"))
getupdatefiles_array.append(generator.exit_on_error("get_io_status", "0", "get_io_status"))
getupdatefiles_array.append(generator.comment("exit script"))
getupdatefiles_array.append(generator.exit_script("0", "get update files"))
sendoutputfiles_array = []
# We need to make sure that curl failures lead to retry while
# missing output (from say a failed job) is logged but
# ignored in relation to sendoutputfiles success.
sendoutputfiles_array.append(generator.print_start("send output files"))
sendoutputfiles_array.append(generator.init_io_log())
sendoutputfiles_array.append(generator.comment("check output files"))
sendoutputfiles_array.append(generator.output_files_missing("missing_counter"))
sendoutputfiles_array.append(generator.log_io_status("output_files_missing", "missing_counter"))
sendoutputfiles_array.append(generator.print_on_error("missing_counter", "0", "missing output files"))
sendoutputfiles_array.append(generator.comment("send output files"))
sendoutputfiles_array.append(generator.send_output_files("send_output_status"))
sendoutputfiles_array.append(generator.log_io_status("send_output_files", "send_output_status"))
sendoutputfiles_array.append(
generator.print_on_error("send_output_status", "0", "failed to send one or more outputfiles")
)
sendoutputfiles_array.append(generator.exit_on_error("send_output_status", "0", "send_output_status"))
sendoutputfiles_array.append(generator.comment("send io files"))
sendoutputfiles_array.append(generator.send_io_files("send_io_status"))
sendoutputfiles_array.append(generator.log_io_status("send_io_files", "send_io_status"))
sendoutputfiles_array.append(generator.print_on_error("send_io_status", "0", "failed to send one or more IO files"))
sendoutputfiles_array.append(generator.exit_on_error("send_io_status", "0", "send_io_status"))
sendoutputfiles_array.append(generator.comment("send status files"))
sendoutputfiles_array.append(
generator.send_status_files([job_dictionary["JOB_ID"] + ".io-status"], "send_io_status_status")
)
sendoutputfiles_array.append(
generator.print_on_error("send_io_status_status", "0", "failed to send io-status file")
)
sendoutputfiles_array.append(generator.exit_on_error("send_io_status_status", "0", "send_io_status_status"))
# Please note that .status upload marks the end of the
# session and thus it must be the last uploaded file.
sendoutputfiles_array.append(
generator.send_status_files([job_dictionary["JOB_ID"] + ".status"], "send_status_status")
)
sendoutputfiles_array.append(generator.print_on_error("send_status_status", "0", "failed to send status file"))
sendoutputfiles_array.append(generator.exit_on_error("send_status_status", "0", "send_status_status"))
# Note that ID.sendouputfiles is called from frontend_script
# so exit on failure can be handled there.
sendoutputfiles_array.append(generator.comment("exit script"))
sendoutputfiles_array.append(generator.exit_script("0", "send output files"))
sendupdatefiles_array = []
# We need to make sure that curl failures lead to retry while
# missing output (from say a failed job) is logged but
# ignored in relation to sendupdatefiles success.
sendupdatefiles_array.append(generator.print_start("send update files"))
sendupdatefiles_array.append(generator.init_io_log())
sendupdatefiles_array.append(generator.comment("send io files"))
sendupdatefiles_array.append(generator.send_io_files("send_io_status"))
sendupdatefiles_array.append(generator.log_io_status("send_io_files", "send_io_status"))
sendupdatefiles_array.append(generator.print_on_error("send_io_status", "0", "failed to send one or more IO files"))
sendupdatefiles_array.append(generator.exit_on_error("send_io_status", "0", "send_io_status"))
sendupdatefiles_array.append(generator.comment("exit script"))
sendupdatefiles_array.append(generator.exit_script("0", "send update files"))
# clean up must be done with SSH (when the .status file
# has been uploaded): Job script can't safely/reliably clean up
# after itself because of possible user interference.
if job_dictionary.has_key("JOBTYPE") and job_dictionary["JOBTYPE"].lower() == "interactive":
# interactive jobs have a .job file just containing a curl
# call to the MiG servers cgi-sid/requestinteractivejob
# and the usual .job is instead called .interactivejob and
# is SCP'ed and started by SSH in the requestinteractive.py
# script
logger.error("jobtype: interactive")
interactivejobfile = (
generator.script_init()
+ "\n"
+ generator.request_interactive()
+ "\n"
+ generator.exit_script("0", "interactive job")
)
# write the small file containing the requestinteractivejob.py
# call as .job
write_file(interactivejobfile, configuration.mig_system_files + job_dictionary["JOB_ID"] + ".job", logger)
# write the usual .job file as .interactivejob
write_file(
"\n".join(job_array), configuration.mig_system_files + job_dictionary["JOB_ID"] + ".interactivejob", logger
)
print interactivejobfile
else:
# write files
write_file("\n".join(job_array), configuration.mig_system_files + job_dictionary["JOB_ID"] + ".job", logger)
write_file("\n".join(getinputfiles_array), path_without_extension + ".getinputfiles", logger)
write_file(
"\n".join(getupdatefiles_array),
configuration.mig_system_files + job_dictionary["JOB_ID"] + ".getupdatefiles",
logger,
)
write_file(
"\n".join(sendoutputfiles_array),
configuration.mig_system_files + job_dictionary["JOB_ID"] + ".sendoutputfiles",
logger,
)
write_file(
"\n".join(sendupdatefiles_array),
configuration.mig_system_files + job_dictionary["JOB_ID"] + ".sendupdatefiles",
logger,
)
return True
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
logger.debug("User: %s executing %s" % (client_id, op_name))
if not configuration.site_enable_jupyter:
output_objects.append({
'object_type':
'error_text',
'text':
'The Jupyter service is not enabled on the system'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not configuration.site_enable_sftp_subsys and not \
configuration.site_enable_sftp:
output_objects.append({
'object_type':
'error_text',
'text':
'The required sftp service is not enabled on the system'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if configuration.site_enable_sftp:
sftp_port = configuration.user_sftp_port
if configuration.site_enable_sftp_subsys:
sftp_port = configuration.user_sftp_subsys_port
requested_service = accepted['service'][-1]
service = {
k: v
for options in configuration.jupyter_services
for k, v in options.items()
if options['service_name'] == requested_service
}
if not service:
valid_services = [
options['name'] for options in configuration.jupyter_services
]
output_objects.append({
'object_type':
'error_text',
'text':
'%s is not a valid jupyter service, '
'allowed include %s' % (requested_service, valid_services)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
valid_service = valid_jupyter_service(configuration, service)
if not valid_service:
output_objects.append({
'object_type':
'error_text',
'text':
'The service %s appears to be misconfigured, '
'please contact a system administrator about this issue' %
requested_service
})
return (output_objects, returnvalues.SYSTEM_ERROR)
host = get_host_from_service(configuration, service)
# Get an active jupyterhost
if host is None:
logger.error("No active jupyterhub host could be found")
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to establish connection to the %s Jupyter service' %
service['service_name']
})
output_objects.append({
'object_type': 'link',
'destination': 'jupyter.py',
'text': 'Back to Jupyter services overview'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
remote_user = unescape(os.environ.get('REMOTE_USER', '')).strip()
if not remote_user:
logger.error("Can't connect to jupyter with an empty REMOTE_USER "
"environment variable")
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to establish connection to the Jupyter service'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Ensure the remote_user dict can be http posted
remote_user = str(remote_user)
# TODO, activate admin info
# remote_user = {'USER': username, 'IS_ADMIN': is_admin(client_id,
# configuration,
# logger)}
# Regular sftp path
mnt_path = os.path.join(configuration.jupyter_mount_files_dir, client_dir)
# Subsys sftp path
subsys_path = os.path.join(configuration.mig_system_files, 'jupyter_mount')
# sftp session path
link_home = configuration.sessid_to_jupyter_mount_link_home
user_home_dir = os.path.join(configuration.user_home, client_dir)
# Preparing prerequisites
if not os.path.exists(mnt_path):
os.makedirs(mnt_path)
if not os.path.exists(link_home):
os.makedirs(link_home)
if configuration.site_enable_sftp_subsys:
if not os.path.exists(subsys_path):
os.makedirs(subsys_path)
# Make sure ssh daemon does not complain
tighten_key_perms(configuration, client_id)
url_base = '/' + service['service_name']
url_home = url_base + '/home'
url_auth = host + url_base + '/hub/login'
url_data = host + url_base + '/hub/user-data'
# Does the client home dir contain an active mount key
# If so just keep on using it.
jupyter_mount_files = [
os.path.join(mnt_path, jfile) for jfile in os.listdir(mnt_path)
if jfile.endswith('.jupyter_mount')
]
logger.info("User: %s mount files: %s" %
(client_id, "\n".join(jupyter_mount_files)))
logger.debug("Remote-User %s" % remote_user)
active_mounts = []
for jfile in jupyter_mount_files:
jupyter_dict = unpickle(jfile, logger)
if not jupyter_dict:
# Remove failed unpickle
logger.error("Failed to unpickle %s removing it" % jfile)
remove_jupyter_mount(jfile, configuration)
else:
# Mount has been timed out
if not is_active(jupyter_dict):
remove_jupyter_mount(jfile, configuration)
else:
# Valid mount
active_mounts.append({'path': jfile, 'state': jupyter_dict})
logger.debug(
"User: %s active keys: %s" %
(client_id, "\n".join([mount['path'] for mount in active_mounts])))
# If multiple are active, remove oldest
active_mount, old_mounts = get_newest_mount(active_mounts)
for mount in old_mounts:
remove_jupyter_mount(mount['path'], configuration)
# A valid active key is already present redirect straight to the jupyter
# service, pass most recent mount information
if active_mount is not None:
mount_dict = mig_to_mount_adapt(active_mount['state'])
user_dict = mig_to_user_adapt(active_mount['state'])
logger.debug("Existing header values, Mount: %s User: %s" %
(mount_dict, user_dict))
auth_header = {'Remote-User': remote_user}
json_data = {'data': {'Mount': mount_dict, 'User': user_dict}}
if configuration.site_enable_workflows:
workflows_dict = mig_to_workflows_adapt(active_mount['state'])
if not workflows_dict:
# No cached workflows session could be found -> refresh with a
# one
workflow_session_id = get_workflow_session_id(
configuration, client_id)
if not workflow_session_id:
workflow_session_id = create_workflow_session_id(
configuration, client_id)
# TODO get this dynamically
url = configuration.migserver_https_sid_url + \
'/cgi-sid/workflowsjsoninterface.py?output_format=json'
workflows_dict = {
'WORKFLOWS_URL': url,
'WORKFLOWS_SESSION_ID': workflow_session_id
}
logger.debug("Existing header values, Workflows: %s" %
workflows_dict)
json_data['workflows_data'] = {'Session': workflows_dict}
with requests.session() as session:
# Authenticate and submit data
response = session.post(url_auth, headers=auth_header)
if response.status_code == 200:
response = session.post(url_data, json=json_data)
if response.status_code != 200:
logger.error(
"Jupyter: User %s failed to submit data %s to %s" %
(client_id, json_data, url_data))
else:
logger.error(
"Jupyter: User %s failed to authenticate against %s" %
(client_id, url_auth))
# Redirect client to jupyterhub
return jupyter_host(configuration, output_objects, remote_user,
url_home)
# Create a new keyset
# Create login session id
session_id = generate_random_ascii(2 * session_id_bytes,
charset='0123456789abcdef')
# Generate private/public keys
(mount_private_key,
mount_public_key) = generate_ssh_rsa_key_pair(encode_utf8=True)
# Known hosts
sftp_addresses = socket.gethostbyname_ex(
configuration.user_sftp_show_address or socket.getfqdn())
# Subsys sftp support
if configuration.site_enable_sftp_subsys:
# Restrict possible mount agent
auth_content = []
restrict_opts = 'no-agent-forwarding,no-port-forwarding,no-pty,'
restrict_opts += 'no-user-rc,no-X11-forwarding'
restrictions = '%s' % restrict_opts
auth_content.append('%s %s\n' % (restrictions, mount_public_key))
# Write auth file
write_file('\n'.join(auth_content),
os.path.join(subsys_path, session_id + '.authorized_keys'),
logger,
umask=027)
logger.debug("User: %s - Creating a new jupyter mount keyset - "
"private_key: %s public_key: %s " %
(client_id, mount_private_key, mount_public_key))
jupyter_dict = {
'MOUNT_HOST': configuration.short_title,
'SESSIONID': session_id,
'USER_CERT': client_id,
# don't need fraction precision, also not all systems provide fraction
# precision.
'CREATED_TIMESTAMP': int(time.time()),
'MOUNTSSHPRIVATEKEY': mount_private_key,
'MOUNTSSHPUBLICKEY': mount_public_key,
# Used by the jupyterhub to know which host to mount against
'TARGET_MOUNT_ADDR': "@" + sftp_addresses[0] + ":",
'PORT': sftp_port
}
client_email = extract_field(client_id, 'email')
if client_email:
jupyter_dict.update({'USER_EMAIL': client_email})
if configuration.site_enable_workflows:
workflow_session_id = get_workflow_session_id(configuration, client_id)
if not workflow_session_id:
workflow_session_id = create_workflow_session_id(
configuration, client_id)
# TODO get this dynamically
url = configuration.migserver_https_sid_url + \
'/cgi-sid/workflowsjsoninterface.py?output_format=json'
jupyter_dict.update({
'WORKFLOWS_URL': url,
'WORKFLOWS_SESSION_ID': workflow_session_id
})
# Only post the required keys, adapt to API expectations
mount_dict = mig_to_mount_adapt(jupyter_dict)
user_dict = mig_to_user_adapt(jupyter_dict)
workflows_dict = mig_to_workflows_adapt(jupyter_dict)
logger.debug("User: %s Mount header: %s" % (client_id, mount_dict))
logger.debug("User: %s User header: %s" % (client_id, user_dict))
if workflows_dict:
logger.debug("User: %s Workflows header: %s" %
(client_id, workflows_dict))
# Auth and pass a new set of valid mount keys
auth_header = {'Remote-User': remote_user}
json_data = {'data': {'Mount': mount_dict, 'User': user_dict}}
if workflows_dict:
json_data['workflows_data'] = {'Session': workflows_dict}
# First login
with requests.session() as session:
# Authenticate
response = session.post(url_auth, headers=auth_header)
if response.status_code == 200:
response = session.post(url_data, json=json_data)
if response.status_code != 200:
logger.error(
"Jupyter: User %s failed to submit data %s to %s" %
(client_id, json_data, url_data))
else:
logger.error("Jupyter: User %s failed to authenticate against %s" %
(client_id, url_auth))
# Update pickle with the new valid key
jupyter_mount_state_path = os.path.join(mnt_path,
session_id + '.jupyter_mount')
pickle(jupyter_dict, jupyter_mount_state_path, logger)
# Link jupyter pickle state file
linkdest_new_jupyter_mount = os.path.join(mnt_path,
session_id + '.jupyter_mount')
linkloc_new_jupyter_mount = os.path.join(link_home,
session_id + '.jupyter_mount')
make_symlink(linkdest_new_jupyter_mount, linkloc_new_jupyter_mount, logger)
# Link userhome
linkloc_user_home = os.path.join(link_home, session_id)
make_symlink(user_home_dir, linkloc_user_home, logger)
return jupyter_host(configuration, output_objects, remote_user, url_home)
if __name__ == '__main__':
print 'starting translation test. Args: ' , len(sys.argv)
logger.debug('translation for file ' + sys.argv[1] + ' starts')
if len(sys.argv) > 1:
fname = sys.argv[1]
parsed = '.'.join([fname,'parsed'])
translated = '.'.join([parsed,'xrsl'])
try:
import shared.mrslparser as p
import shared.fileio as fileio
(presult,errors) = p.parse(fname, 'test-id',
'+No+Client+Id',None,parsed)
if not presult:
print 'Errors:\n%s' % errors
else:
print 'Parsing OK, now translating'
mrsl_dict = fileio.unpickle(parsed,logger)
(xrsl,script,name) = translate(mrsl_dict,'test-name')
print '\n'.join(['Job name',name,'script',script,'XRSL'])
fileio.write_file(script, "test-id.sh", logger)
print (format_xrsl(xrsl))
fileio.write_file("%s" % xrsl, translated, logger)
print 'done'
except Exception, err:
print 'Error.'
print err.__str__()
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
unique_resource_name = accepted['unique_resource_name'][-1]
resconfig = accepted['resconfig'][-1]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type': 'header',
'text': 'Trying to Update resource configuration'
})
if not is_owner(client_id, unique_resource_name,
configuration.resource_home, logger):
logger.error(client_id + ' is not an owner of ' +
unique_resource_name + ': update rejected!')
output_objects.append({
'object_type':
'error_text',
'text':
'You must be an owner of ' + unique_resource_name +
' to update the configuration!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# TODO: race if two confs are uploaded concurrently!
host_url, host_identifier = unique_resource_name.rsplit('.', 1)
pending_file = os.path.join(configuration.resource_home,
unique_resource_name, 'config.tmp')
# write new proposed config file to disk
try:
logger.info('write to file: %s' % pending_file)
if not write_file(resconfig, pending_file, logger):
output_objects.append({
'object_type': 'error_text',
'text': 'Could not write: %s' % pending_file
})
return (output_objects, returnvalues.SYSTEM_ERROR)
except Exception, err:
logger.error('Resource conf %s could not be written: %s' % \
(pending_file, err))
output_objects.append({
'object_type': 'error_text',
'text': 'Could not write configuration!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
except Exception, err:
# error during translation, pass a message
logger.error('Error during xRSL translation: %s' % err.__str__())
return (None, err.__str__())
# we submit directly from here (the other version above does
# copyFileToResource and gen_job_script generates all files)
# we have to put the generated script somewhere..., and submit from there.
# inputfiles are given by the user as relative paths from his home,
# so we should use that location (and clean up afterwards).
# write script (to user home)
user_home = os.path.join(configuration.user_home, client_dir)
script_path = os.path.abspath(os.path.join(user_home, script_name))
write_file(script, script_path, logger)
os.chdir(user_home)
try:
logger.debug('submitting job to ARC')
session = arc.Ui(user_home)
arc_job_ids = session.submit(xrsl)
# if no exception occurred, we are done:
job_dict['ARCID'] = arc_job_ids[0]
job_dict['SESSIONID'] = sessionid
msg = 'OK'
result = job_dict
})
return (output_objects, returnvalues.CLIENT_ERROR)
# create directory to store vgrid public_base files
try:
os.mkdir(public_base_dir)
pub_readme = os.path.join(public_base_dir, 'README')
if not os.path.exists(pub_readme):
write_file("""= Public Web Page =
This directory is used for hosting the public web page for the %s %s.
It is accessible by the public from the %ss page or directly using the URL
%s/vgrid/%s/
Just update the index.html file to suit your wishes for an entry page. It can
link to any other material in this folder or subfolders with relative
addresses. So it is possible to create a full web site with multiple pages and
rich content like on other web hosting services. However, there's no support
for server side scripting with Python, ASP or PHP for security reasons.
""" % (vgrid_name, configuration.site_vgrid_label,
configuration.site_vgrid_label, configuration.migserver_http_url,
vgrid_name),
pub_readme, logger)
pub_entry_page = os.path.join(public_base_dir, 'index.html')
if not os.path.exists(pub_entry_page):
write_file("""<!DOCTYPE html>
<html>
<head>
<meta http-equiv='Content-Type' content='text/html;charset=utf-8'/>
<title>Public entry page not created yet..</title>
</head>
<body>
status = False
continue
# TODO: can we detect and ignore symlinks?
# Zip format is horribly designed/documented:
# http://www.pkware.com/documents/casestudies/APPNOTE.TXT
# I haven't managed to find a way to detect symlinks. Thus
# they are simply created as files containing the name they
# were supposed to link to: This is inconsistent but safe :-S
# write file - symbolic links are written as files! (good for
# security).
# NB: Needs to use undecoded filename here
if not write_file(zip_data, local_zip_entry_name, logger) and \
not os.path.exists(local_zip_entry_name):
msg += 'Error unpacking %s to disk! ' % entry_filename
status = False
continue
# get the size as the OS sees it
try:
__ = os.path.getsize(local_zip_entry_name)
except Exception, exc:
logger.warning("unpack may have failed: %s" % exc)
msg += \
'File %s unpacked, but could not get file size %s! '\
% (entry_filename, exc)
status = False
logger.error("create_frozen_archive: failed: %s" % msg)
remove_rec(frozen_dir, configuration)
return (False, 'Error writing frozen archive')
else:
(status, msg) = move_file(real_source, freeze_path, configuration)
if not status:
logger.error("create_frozen_archive: failed: %s" % msg)
remove_rec(frozen_dir, configuration)
return (False, 'Error writing frozen archive')
logger.info("create_frozen_archive: save %s for %s" % \
([i[0] for i in freeze_upload], freeze_id))
for (filename, contents) in freeze_upload:
freeze_path = os.path.join(frozen_dir, filename)
frozen_files.append(filename)
logger.debug("create_frozen_archive: write %s" % freeze_path)
if not write_file(contents, freeze_path, logger):
logger.error("create_frozen_archive: failed: %s" % err)
remove_rec(frozen_dir, configuration)
return (False, 'Error writing frozen archive')
if freeze_dict['PUBLISH']:
published_id = public_freeze_id(freeze_dict)
public_meta = [('CREATOR', 'Owner'), ('NAME', 'Name'),
('DESCRIPTION', 'Description'),
('CREATED_TIMESTAMP', 'Date')]
# Use the default preamle to get style, skin and so on right
contents = get_cgi_html_preamble(configuration, "Public Archive: %s" % \
published_id, "", widgets=False)
