def get_ps3_resource():
log_msg = 'ps3live'
# Identify sandboxkey
sandboxkey = fieldstorage.getfirst('sandboxkey', None)
if not sandboxkey:
# No sandboxkey provided,
log_msg = log_msg + ', Remote IP: %s, provided no sandboxkey.'\
% os.getenv('REMOTE_ADDR')
return (False, log_msg)
if not os.path.exists(configuration.sandbox_home + sandboxkey):
# Create resource
unique_resource_name = create_ps3_resource(sandboxkey)
log_msg = log_msg + ' Created resource: %s'\
% unique_resource_name
# Make symbolic link from
# sandbox_home/sandboxkey to resource_home/resource_name
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(os.path.join(configuration.resource_home,
unique_resource_name))
make_symlink(resource_path, sandbox_link, logger)
else:
(status, unique_resource_name) = get_resource_name(sandboxkey,
logger)
if not status:
return (False, unique_resource_name)
# If resource has a jobrequest pending, remove it.
job_pending_file = os.path.join(configuration.resource_home,
unique_resource_name,
'jobrequest_pending.ps3')
if os.path.exists(job_pending_file):
os.remove(job_pending_file)
log_msg = log_msg + ', Remote IP: %s, Key: %s'\
% (os.getenv('REMOTE_ADDR'), sandboxkey)
o.internal('''
%s
''' % log_msg)
return (True, unique_resource_name)
def get_ps3_resource(configuration):
log_msg = 'ps3live'
# Identify sandboxkey
sandboxkey = fieldstorage.getfirst('sandboxkey', None)
if not sandboxkey:
# No sandboxkey provided,
log_msg = log_msg + ', Remote IP: %s, provided no sandboxkey.'\
% os.getenv('REMOTE_ADDR')
return (False, log_msg)
if not os.path.exists(configuration.sandbox_home + sandboxkey):
# Create resource
unique_resource_name = create_ps3_resource(configuration, sandboxkey)
log_msg = log_msg + ' Created resource: %s'\
% unique_resource_name
# Make symbolic link from
# sandbox_home/sandboxkey to resource_home/resource_name
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(
os.path.join(configuration.resource_home, unique_resource_name))
make_symlink(resource_path, sandbox_link, logger)
else:
(status, unique_resource_name) = get_resource_name(sandboxkey, logger)
if not status:
return (False, unique_resource_name)
# If resource has a jobrequest pending, remove it.
job_pending_file = os.path.join(configuration.resource_home,
unique_resource_name,
'jobrequest_pending.ps3')
if os.path.exists(job_pending_file):
os.remove(job_pending_file)
log_msg = log_msg + ', Remote IP: %s, Key: %s'\
% (os.getenv('REMOTE_ADDR'), sandboxkey)
o.internal('''
%s
''' % log_msg)
return (True, unique_resource_name)
def save_twofactor_session(configuration,
client_id,
session_key,
user_addr,
user_agent,
session_start,
session_end=-1):
"""Save twofactor session dict for client_id"""
_logger = configuration.logger
if configuration.site_enable_gdp:
client_id = get_base_client_id(configuration,
client_id,
expand_oid_alias=False)
session_path = os.path.join(configuration.twofactor_home, session_key)
if session_end < 0:
session_end = session_start + twofactor_cookie_ttl
session_data = {
'client_id': client_id,
'session_key': session_key,
'user_addr': user_addr,
'user_agent': user_agent,
'session_start': session_start,
'session_end': session_end
}
status = pickle(session_data, session_path, configuration.logger)
if status and configuration.site_twofactor_strict_address:
session_path_link = os.path.join(configuration.twofactor_home,
"%s_%s" % (user_addr, session_key))
status = \
make_symlink(session_key, session_path_link, _logger, force=False)
if not status:
delete_file(session_path, _logger)
return status
output_objects.append(
{'object_type': 'error_text', 'text'
: ('Could not create needed dirs on %s server! %s'
% (configuration.short_title, exc))})
logger.error('%s when looking for dir %s.' % (exc, share_dir))
return (output_objects, returnvalues.SYSTEM_ERROR)
# create symlink from users home directory to vgrid file directory
# unless member of parent vgrid so that it is included already
link_src = os.path.abspath(configuration.vgrid_files_home + os.sep
+ vgrid_name) + os.sep
link_dst = user_dir + vgrid_name
if not inherit_vgrid_member and \
not make_symlink(link_src, link_dst, logger):
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create link to %s share!' % \
configuration.site_vgrid_label})
logger.error('Could not create link to %s files (%s -> %s)'
% (configuration.site_vgrid_label, link_src, link_dst))
return (output_objects, returnvalues.SYSTEM_ERROR)
public_base_dst = user_public_base + vgrid_name
# create symlink for public_base files
if not make_symlink(public_base_dir, public_base_dst, logger):
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create link to public_base dir!'
})
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
defaults = signature()[1]
title_entry = find_entry(output_objects, 'title')
label = "%s" % configuration.site_vgrid_label
title_entry['text'] = "Add %s Member" % label
output_objects.append({'object_type': 'header', 'text':
'Add %s Member(s)' % label})
status = returnvalues.OK
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
vgrid_name = accepted['vgrid_name'][-1].strip()
cert_id_list = accepted['cert_id']
request_name = unhexlify(accepted['request_name'][-1])
rank_list = accepted['rank'] + ['' for _ in cert_id_list]
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append(
{'object_type': 'error_text', 'text': '''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
user_map = get_full_user_map(configuration)
user_dict = user_map.get(client_id, None)
# Optional site-wide limitation of manage vgrid permission
if not user_dict or \
not vgrid_manage_allowed(configuration, user_dict):
logger.warning("user %s is not allowed to manage vgrids!" % client_id)
output_objects.append(
{'object_type': 'error_text', 'text':
'Only privileged users can manage %ss' % label})
return (output_objects, returnvalues.CLIENT_ERROR)
# make sure vgrid settings allow this owner to edit members
(allow_status, allow_msg) = allow_members_adm(configuration, vgrid_name,
client_id)
if not allow_status:
output_objects.append({'object_type': 'error_text', 'text': allow_msg})
return (output_objects, returnvalues.CLIENT_ERROR)
cert_id_added = []
for (cert_id, rank_str) in zip(cert_id_list, rank_list):
cert_id = cert_id.strip()
cert_dir = client_id_dir(cert_id)
try:
rank = int(rank_str)
except ValueError:
rank = None
# Allow openid alias as subject if openid with alias is enabled
if configuration.user_openid_providers and configuration.user_openid_alias:
cert_id = expand_openid_alias(cert_id, configuration)
# Validity of user and vgrid names is checked in this init function so
# no need to worry about illegal directory traversal through variables
(ret_val, msg, _) = \
init_vgrid_script_add_rem(vgrid_name, client_id, cert_id,
'member', configuration)
if not ret_val:
output_objects.append({'object_type': 'error_text', 'text':
msg})
status = returnvalues.CLIENT_ERROR
continue
# don't add if already an owner
if vgrid_is_owner(vgrid_name, cert_id, configuration):
output_objects.append(
{'object_type': 'error_text', 'text':
'%s is already an owner of %s or a parent %s.' %
(cert_id, vgrid_name, label)})
status = returnvalues.CLIENT_ERROR
continue
# don't add if already a member unless rank is given
if rank is None and vgrid_is_member(vgrid_name, cert_id, configuration):
output_objects.append(
{'object_type': 'error_text', 'text':
'''%s is already a member of %s or a parent %s. Please remove
the person first and then try this operation again.''' %
(cert_id, vgrid_name, label)
})
status = returnvalues.CLIENT_ERROR
continue
# owner or member of subvgrid?
(list_status, subvgrids) = vgrid_list_subvgrids(vgrid_name,
configuration)
if not list_status:
output_objects.append({'object_type': 'error_text', 'text':
'Error getting list of sub%ss: %s' %
(label, subvgrids)})
status = returnvalues.SYSTEM_ERROR
continue
# TODO: we DO allow ownership of sub vgrids with parent membership so we
# should support the (cumbersome) relinking of vgrid shares here. Leave it
# to user to do it manually for now with temporary removal of ownership
skip_entity = False
for subvgrid in subvgrids:
if vgrid_is_owner(subvgrid, cert_id, configuration, recursive=False):
output_objects.append(
{'object_type': 'error_text', 'text':
"""%(cert_id)s is already an owner of a
sub-%(vgrid_label)s ('%(subvgrid)s'). While we DO support members being owners
of sub-%(vgrid_label)ss, we do not support adding parent %(vgrid_label)s
members at the moment. Please (temporarily) remove the person as owner of all
sub-%(vgrid_label)ss first and then try this operation again.""" %
{'cert_id': cert_id, 'subvgrid': subvgrid,
'vgrid_label': label}})
status = returnvalues.CLIENT_ERROR
skip_entity = True
break
if vgrid_is_member(subvgrid, cert_id, configuration, recursive=False):
output_objects.append(
{'object_type': 'error_text', 'text':
"""%s is already a member of a sub-%s ('%s'). Please
remove the person first and then try this operation again.""" %
(cert_id, label, subvgrid)})
status = returnvalues.CLIENT_ERROR
skip_entity = True
break
if skip_entity:
continue
# Check if only rank change was requested and apply if so
if rank is not None:
(add_status, add_msg) = vgrid_add_members(configuration,
vgrid_name,
[cert_id], rank=rank)
if not add_status:
output_objects.append(
{'object_type': 'error_text', 'text': add_msg})
status = returnvalues.SYSTEM_ERROR
else:
output_objects.append({'object_type': 'text', 'text':
'changed %s to member %d' % (cert_id,
rank)})
# No further action after rank change as everything else exists
continue
# Getting here means cert_id is neither owner or member of any parent or
# sub-vgrids.
# Please note that base_dir must end in slash to avoid access to other
# vgrid dirs when own name is a prefix of another name
base_dir = os.path.abspath(os.path.join(configuration.vgrid_home,
vgrid_name)) + os.sep
user_dir = os.path.abspath(os.path.join(configuration.user_home,
cert_dir)) + os.sep
# make sure all dirs can be created (that a file or directory with the same
# name do not exist prior to adding the member)
if os.path.exists(user_dir + vgrid_name):
output_objects.append(
{'object_type': 'error_text', 'text':
'''Could not add member, a file or directory in the home
directory called %s exists! (%s)''' % (vgrid_name, user_dir + vgrid_name)})
status = returnvalues.CLIENT_ERROR
continue
# Add
(add_status, add_msg) = vgrid_add_members(configuration, vgrid_name,
[cert_id])
if not add_status:
output_objects.append(
{'object_type': 'error_text', 'text': add_msg})
status = returnvalues.SYSTEM_ERROR
continue
vgrid_name_parts = vgrid_name.split('/')
is_subvgrid = len(vgrid_name_parts) > 1
if is_subvgrid:
try:
# vgrid_name = IMADA/STUD/BACH
# vgrid_name_last_fragment = BACH
vgrid_name_last_fragment = \
vgrid_name_parts[len(vgrid_name_parts)
- 1].strip()
# vgrid_name_without_last_fragment = IMADA/STUD/
vgrid_name_without_last_fragment = \
('/'.join(vgrid_name_parts[0:len(vgrid_name_parts)
- 1]) + os.sep).strip()
# create dirs if they do not exist
dir1 = user_dir + vgrid_name_without_last_fragment
if not os.path.isdir(dir1):
os.makedirs(dir1)
except Exception, exc:
# out of range? should not be possible due to is_subvgrid check
output_objects.append(
{'object_type': 'error_text', 'text':
('Could not create needed dirs on %s server! %s'
% (configuration.short_title, exc))})
logger.error('%s when looking for dir %s.' % (exc, dir1))
status = returnvalues.SYSTEM_ERROR
continue
# create symlink from users home directory to vgrid file directory
link_src = os.path.abspath(configuration.vgrid_files_home + os.sep
+ vgrid_name) + os.sep
link_dst = user_dir + vgrid_name
# create symlink to vgrid files
if not make_symlink(link_src, link_dst, logger):
output_objects.append({'object_type': 'error_text', 'text':
'Could not create link to %s files!' %
label
})
logger.error('Could not create link to %s files! (%s -> %s)' %
(label, link_src, link_dst))
status = returnvalues.SYSTEM_ERROR
continue
cert_id_added.append(cert_id)
output_objects.append({
'object_type':
'error_text',
'text':
'Could not update sandbox database: %s' % exc
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.debug('building resource specific files for %s' % unique_host_name)
# create sandboxlink
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(configuration.resource_home +
unique_host_name)
make_symlink(resource_path, sandbox_link, logger)
# change dir to sss_home
old_path = os.getcwd()
# log_dir = "log/"
# read pickled resource conf file (needed to create
# master_node_script.sh)
msg = ''
(status, resource_config) = \
get_resource_configuration(configuration.resource_home,
unique_host_name, logger)
logger.debug('got resource conf %s' % resource_config)
def create_arc_job(job, configuration, logger):
"""Analog to create_job_script for ARC jobs:
Creates symLinks for receiving result files, translates job dict to ARC
xrsl, and stores resulting job script (xrsl + sh script) for submitting.
We do _not_ create a separate job_dict with copies and SESSIONID inside,
as opposed to create_job_script, all we need is the link from
webserver_home / sessionID into the user's home directory
("job_output/job['JOB_ID']" is added to the result upload URLs in the
translation).
Returns message (ARC job ID if no error) and sessionid (None if error)
"""
if not configuration.arc_clusters:
return (None, "No ARC support!")
if not job["JOBTYPE"] == "arc":
return (None, "Error. This is not an ARC job")
# Deep copy job for local changes
job_dict = deepcopy(job)
# Finally expand reserved job variables like +JOBID+ and +JOBNAME+
job_dict = expand_variables(job_dict)
# ... no more changes to job_dict from here on
client_id = str(job_dict["USER_CERT"])
# we do not want to see empty jobs here. Test as done in create_job_script.
if client_id == configuration.empty_job_name:
return (None, "Error. empty job for ARC?")
# generate random session ID:
sessionid = hexlify(open("/dev/urandom").read(32))
logger.debug("session ID (for creating links): %s" % sessionid)
client_dir = client_id_dir(client_id)
# make symbolic links inside webserver_home:
#
# we need: link to owner's dir. to receive results,
# job mRSL inside sessid_to_mrsl_link_home
linklist = [
(configuration.user_home + client_dir, configuration.webserver_home + sessionid),
(
configuration.mrsl_files_dir + client_dir + "/" + str(job_dict["JOB_ID"]) + ".mRSL",
configuration.sessid_to_mrsl_link_home + sessionid + ".mRSL",
),
]
for (dest, loc) in linklist:
make_symlink(dest, loc, logger)
# the translation generates an xRSL object which specifies to execute
# a shell script with script_name. If sessionid != None, results will
# be uploaded to sid_redirect/sessionid/job_output/job_id
try:
(xrsl, script, script_name) = mrsltoxrsl.translate(job_dict, sessionid)
logger.debug("translated to xRSL: %s" % xrsl)
logger.debug("script:\n %s" % script)
except Exception, err:
# error during translation, pass a message
logger.error("Error during xRSL translation: %s" % err.__str__())
return (None, err.__str__())
def create_job_script(unique_resource_name, exe, job, resource_config, localjobname, configuration, logger):
"""Helper to create actual jobs for handout to a resource.
Returns tuple with job dict on success and None otherwise.
The job dict includes random generated sessionid and a I/O session id.
"""
job_dict = {"": ""}
sessionid = hexlify(open("/dev/urandom").read(32))
iosessionid = hexlify(open("/dev/urandom").read(32))
helper_dict_filename = os.path.join(
configuration.resource_home, unique_resource_name, "empty_job_helper_dict.%s" % exe
)
# Deep copy job for local changes
job_dict = deepcopy(job)
job_dict["SESSIONID"] = sessionid
job_dict["IOSESSIONID"] = iosessionid
# Create ssh rsa keys and known_hosts for job mount
mount_private_key = ""
mount_public_key = ""
mount_known_hosts = ""
if job_dict.get("MOUNT", []) != []:
# Generate public/private key pair for sshfs
(mount_private_key, mount_public_key) = generate_ssh_rsa_key_pair()
# Generate known_hosts
if not os.path.exists(configuration.user_sftp_key_pub):
msg = "job generation failed:"
msg = "%s user_sftp_key_pub: '%s' -> File _NOT_ found" % (msg, configuration.user_sftp_key_pub)
print msg
logger.error(msg)
return (None, msg)
sftp_address = configuration.user_sftp_show_address
sftp_addresses = socket.gethostbyname_ex(sftp_address or socket.getfqdn())
sftp_port = configuration.user_sftp_show_port
mount_known_hosts = "%s,[%s]:%s" % (sftp_addresses[0], sftp_addresses[0], sftp_port)
for list_idx in xrange(1, len(sftp_addresses)):
for sftp_address in sftp_addresses[list_idx]:
mount_known_hosts += ",%s,[%s]:%s" % (sftp_address, sftp_address, sftp_port)
fd = open(configuration.user_sftp_key_pub, "r")
mount_known_hosts = "%s %s" % (mount_known_hosts, fd.read())
fd.close()
job_dict["MOUNTSSHPUBLICKEY"] = mount_public_key
job_dict["MOUNTSSHPRIVATEKEY"] = mount_private_key
job_dict["MOUNTSSHKNOWNHOSTS"] = mount_known_hosts
if not job_dict.has_key("MAXPRICE"):
job_dict["MAXPRICE"] = "0"
# Finally expand reserved job variables like +JOBID+ and +JOBNAME+
job_dict = expand_variables(job_dict)
# ... no more changes to job_dict from here on
client_id = str(job_dict["USER_CERT"])
client_dir = client_id_dir(client_id)
# if not job:
if client_id == configuration.empty_job_name:
# create link to empty job
linkdest_empty_job = helper_dict_filename
linkloc_empty_job = configuration.sessid_to_mrsl_link_home + sessionid + ".mRSL"
make_symlink(linkdest_empty_job, linkloc_empty_job, logger)
else:
# link sessionid to mrsl file
linkdest1 = configuration.mrsl_files_dir + client_dir + "/" + str(job_dict["JOB_ID"]) + ".mRSL"
linkloc1 = configuration.sessid_to_mrsl_link_home + sessionid + ".mRSL"
make_symlink(linkdest1, linkloc1, logger)
# link sessionid to job owners home directory
linkdest2 = configuration.user_home + client_dir
linkloc2 = configuration.webserver_home + sessionid
make_symlink(linkdest2, linkloc2, logger)
# link iosessionid to job owners home directory
linkdest3 = configuration.user_home + client_dir
linkloc3 = configuration.webserver_home + iosessionid
make_symlink(linkdest3, linkloc3, logger)
# link sessionid to .job file
linkdest4 = configuration.mig_system_files + str(job_dict["JOB_ID"]) + ".job"
linkloc4 = configuration.webserver_home + sessionid + ".job"
make_symlink(linkdest4, linkloc4, logger)
# link sessionid to .getupdatefiles file
linkdest5 = configuration.mig_system_files + str(job_dict["JOB_ID"]) + ".getupdatefiles"
linkloc5 = configuration.webserver_home + sessionid + ".getupdatefiles"
make_symlink(linkdest5, linkloc5, logger)
# link sessionid to .sendoutputfiles file
linkdest4 = configuration.mig_system_files + str(job_dict["JOB_ID"]) + ".sendoutputfiles"
linkloc4 = configuration.webserver_home + sessionid + ".sendoutputfiles"
make_symlink(linkdest4, linkloc4, logger)
# link sessionid to .sendupdatefiles file
linkdest5 = configuration.mig_system_files + str(job_dict["JOB_ID"]) + ".sendupdatefiles"
linkloc5 = configuration.webserver_home + sessionid + ".sendupdatefiles"
make_symlink(linkdest5, linkloc5, logger)
path_without_extension = os.path.join(configuration.resource_home, unique_resource_name, localjobname)
gen_res = gen_job_script(
job_dict, resource_config, configuration, localjobname, path_without_extension, client_dir, exe, logger
)
if not gen_res:
msg = "job scripts were not generated. Perhaps you have specified " + "an invalid SCRIPTLANGUAGE ? "
print msg
logger.error(msg)
return (None, msg)
inputfiles_path = path_without_extension + ".getinputfiles"
# hack to ensure that a resource has a sandbox keyword
if resource_config.get("SANDBOX", False):
# Move file to webserver_home for download as we can't push it to
# sandboxes
try:
# RA TODO: change download filename to something that
# includes sessionid
webserver_path = os.path.join(configuration.webserver_home, localjobname + ".getinputfiles")
os.rename(inputfiles_path, webserver_path)
# ########## ATTENTION HACK TO MAKE JVM SANDBOXES WORK ############
# This should be changed to use the (to be developed) RE pre/post
# processing framework. For now the user must have a jvm dir in his
# home dir where the classfiles is located this should be changed
# so that the execution homepath can be specified in the mRSL
# jobfile
# Martin Rehr 08/09/06
# If this is a oneclick job link the users jvm dir to
# webserver_home/sandboxkey.oneclick
# This is done because the client applet uses the
# codebase from which it is originaly loaded
# Therefore the codebase must be dynamicaly changed
# for every job
if resource_config.has_key("PLATFORM") and resource_config["PLATFORM"] == "ONE-CLICK":
# A two step link is made.
# First sandboxkey.oneclick is made to point to
# sessiondid.jvm
# Second sessionid.jvm is set to point to
# USER_HOME/jvm
# This is done for security and easy cleanup,
# sessionid.jvm is cleaned up
# by the server upon job finish/timeout and
# thereby leaving no open entryes to the users
# jvm dir.
linkintermediate = configuration.webserver_home + sessionid + ".jvm"
if client_dir == configuration.empty_job_name:
linkdest = os.path.abspath(configuration.javabin_home)
else:
linkdest = configuration.user_home + client_dir + os.sep + "jvm"
# Make link sessionid.jvm -> USER_HOME/jvm
make_symlink(linkdest, linkintermediate, logger)
linkloc = configuration.webserver_home + resource_config["SANDBOXKEY"] + ".oneclick"
# Remove previous symlink
# This must be done in a try/catch as the symlink,
# may be a dead link and 'if os.path.exists(linkloc):'
# will then return false, even though the link exists.
try:
os.remove(linkloc)
except:
pass
# Make link sandboxkey.oneclick -> sessionid.jvm
make_symlink(linkintermediate, linkloc, logger)
except Exception, err:
# ######### End JVM SANDBOX HACK ###########
msg = "File '%s' was not copied to the webserver home." % inputfiles_path
print "\nERROR: " + str(err)
logger.error(msg)
return (None, msg)
return (job_dict, "OK")
def modify_share_links(action,
share_dict,
client_id,
configuration,
share_map=None):
"""Modify share links with given action and share_dict for client_id.
In practice this a shared helper to add or remove share links from the
saved dictionary. The optional share_map argument can be used to pass an
already loaded dictionary of saved share links to avoid reloading.
"""
logger = configuration.logger
share_id = share_dict['share_id']
if share_map is None:
(load_status, share_map) = load_share_links(configuration, client_id)
if not load_status:
logger.error("modify_share_links failed in load: %s" % share_map)
return (load_status, share_map)
share_dict.update(share_map.get(share_id, {}))
rel_path = share_dict['path'].lstrip(os.sep)
access = share_dict['access']
if 'read' in access and 'write' in access:
access_dir = 'read-write'
elif 'read' in access:
access_dir = 'read-only'
elif 'write' in access:
access_dir = 'write-only'
else:
logger.error("modify_share_links invalid access: %s" % access)
return (load_status, share_map)
symlink_path = os.path.join(configuration.sharelink_home, access_dir,
share_id)
target_path = os.path.join(configuration.user_home,
client_id_dir(client_id), rel_path)
if action == "create":
if not make_symlink(target_path, symlink_path, configuration.logger,
False):
logger.error("could not make share symlink: %s (already exists?)" %
symlink_path)
return (False, share_map)
share_dict.update({
'created_timestamp': datetime.datetime.now(),
'owner': client_id,
})
share_map[share_id] = share_dict
elif action == "modify":
if not make_symlink(target_path, symlink_path, configuration.logger,
True):
logger.error("could not update share symlink: %s" % symlink_path)
return (False, share_map)
share_dict['created_timestamp'] = datetime.datetime.now()
share_map[share_id].update(share_dict)
elif action == "delete":
if not delete_symlink(symlink_path, configuration.logger):
logger.error("could not delete share symlink: %s (missing?)" %
symlink_path)
return (False, share_map)
del share_map[share_id]
else:
return (False, "Invalid action %s on share links" % action)
try:
sharelinks_path = os.path.join(configuration.user_settings,
client_id_dir(client_id),
sharelinks_filename)
dump(share_map, sharelinks_path)
except Exception, err:
logger.error("modify_share_links failed: %s" % err)
return (False, 'Error updating share links: %s' % err)
save_sandbox_db(userdb, configuration)
except Exception, exc:
output_objects.append({'object_type': 'error_text', 'text':
'Could not update sandbox database: %s' % exc
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.debug('building resource specific files for %s'
% unique_host_name)
# create sandboxlink
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(configuration.resource_home
+ unique_host_name)
make_symlink(resource_path, sandbox_link, logger)
# change dir to sss_home
old_path = os.getcwd()
# log_dir = "log/"
# read pickled resource conf file (needed to create
# master_node_script.sh)
msg = ''
(status, resource_config) = \
get_resource_configuration(configuration.resource_home,
unique_host_name, logger)
logger.debug('got resource conf %s' % resource_config)
def create_arc_job(
job,
configuration,
logger,
):
"""Analog to create_job_script for ARC jobs:
Creates symLinks for receiving result files, translates job dict to ARC
xrsl, and stores resulting job script (xrsl + sh script) for submitting.
We do _not_ create a separate job_dict with copies and SESSIONID inside,
as opposed to create_job_script, all we need is the link from
webserver_home / sessionID into the user's home directory
("job_output/job['JOB_ID']" is added to the result upload URLs in the
translation).
Returns message (ARC job ID if no error) and sessionid (None if error)
"""
if not configuration.arc_clusters:
return (None, 'No ARC support!')
if not job['JOBTYPE'] == 'arc':
return (None, 'Error. This is not an ARC job')
# Deep copy job for local changes
job_dict = deepcopy(job)
# Finally expand reserved job variables like +JOBID+ and +JOBNAME+
job_dict = expand_variables(job_dict)
# ... no more changes to job_dict from here on
client_id = str(job_dict['USER_CERT'])
# we do not want to see empty jobs here. Test as done in create_job_script.
if client_id == configuration.empty_job_name:
return (None, 'Error. empty job for ARC?')
# generate random session ID:
sessionid = hexlify(open('/dev/urandom').read(session_id_bytes))
logger.debug('session ID (for creating links): %s' % sessionid)
client_dir = client_id_dir(client_id)
# make symbolic links inside webserver_home:
#
# we need: link to owner's dir. to receive results,
# job mRSL inside sessid_to_mrsl_link_home
linklist = [(configuration.user_home + client_dir,
configuration.webserver_home + sessionid),
(configuration.mrsl_files_dir + client_dir + '/' +
str(job_dict['JOB_ID']) + '.mRSL',
configuration.sessid_to_mrsl_link_home + sessionid + '.mRSL')
]
for (dest, loc) in linklist:
make_symlink(dest, loc, logger)
# the translation generates an xRSL object which specifies to execute
# a shell script with script_name. If sessionid != None, results will
# be uploaded to sid_redirect/sessionid/job_output/job_id
try:
(xrsl, script, script_name) = mrsltoxrsl.translate(job_dict, sessionid)
logger.debug('translated to xRSL: %s' % xrsl)
logger.debug('script:\n %s' % script)
except Exception, err:
# error during translation, pass a message
logger.error('Error during xRSL translation: %s' % err.__str__())
return (None, err.__str__())
def create_job_script(
unique_resource_name,
exe,
job,
resource_config,
localjobname,
configuration,
logger,
):
"""Helper to create actual jobs for handout to a resource.
Returns tuple with job dict on success and None otherwise.
The job dict includes random generated sessionid and a I/O session id.
"""
job_dict = {'': ''}
# TODO: hexlify is an awfully space wasting URL-safe encoding.
# We should just use something like the proposed secure method from
# http://stackoverflow.com/a/23728630/2213647
sessionid = hexlify(open('/dev/urandom').read(session_id_bytes))
iosessionid = hexlify(open('/dev/urandom').read(session_id_bytes))
helper_dict_filename = os.path.join(configuration.resource_home,
unique_resource_name,
'empty_job_helper_dict.%s' % exe)
# Deep copy job for local changes
job_dict = deepcopy(job)
# Bump requested values to any resource specs requested in MAXFILL
job_maxfill = job_dict.get('MAXFILL', [])
if keyword_all in job_maxfill:
job_maxfill = maxfill_fields
for name in maxfill_fields:
if name in job_maxfill:
job_dict[name] = resource_config[name]
job_dict['SESSIONID'] = sessionid
job_dict['IOSESSIONID'] = iosessionid
# Create ssh rsa keys and known_hosts for job mount
mount_private_key = ""
mount_public_key = ""
mount_known_hosts = ""
if job_dict.get('MOUNT', []) != []:
# Generate public/private key pair for sshfs
(mount_private_key, mount_public_key) = generate_ssh_rsa_key_pair()
# Generate known_hosts
if not os.path.exists(configuration.user_sftp_key_pub):
msg = "job generation failed:"
msg = "%s user_sftp_key_pub: '%s' -> File NOT found" % \
(msg, configuration.user_sftp_key_pub)
print msg
logger.error(msg)
return (None, msg)
# Use best available sftp implementation - configuration picks it
sftp_address = configuration.user_sftp_show_address
sftp_port = configuration.user_sftp_show_port
sftp_addresses = socket.gethostbyname_ex(sftp_address or
socket.getfqdn())
mount_known_hosts = "%s,[%s]:%s" % (sftp_addresses[0],
sftp_addresses[0], sftp_port)
for list_idx in xrange(1, len(sftp_addresses)):
for sftp_address in sftp_addresses[list_idx]:
mount_known_hosts += ",%s,[%s]:%s" % (sftp_address,
sftp_address,
sftp_port)
fd = open(configuration.user_sftp_key_pub, 'r')
mount_known_hosts = "%s %s" % (mount_known_hosts, fd.read())
fd.close()
job_dict['MOUNTSSHPUBLICKEY'] = mount_public_key
job_dict['MOUNTSSHPRIVATEKEY'] = mount_private_key
job_dict['MOUNTSSHKNOWNHOSTS'] = mount_known_hosts
if not job_dict.has_key('MAXPRICE'):
job_dict['MAXPRICE'] = '0'
# Finally expand reserved job variables like +JOBID+ and +JOBNAME+
job_dict = expand_variables(job_dict)
# ... no more changes to job_dict from here on
client_id = str(job_dict['USER_CERT'])
client_dir = client_id_dir(client_id)
# if not job:
if client_id == configuration.empty_job_name:
# create link to empty job
linkdest_empty_job = helper_dict_filename
linkloc_empty_job = configuration.sessid_to_mrsl_link_home + \
sessionid + '.mRSL'
make_symlink(linkdest_empty_job, linkloc_empty_job, logger)
else:
# link sessionid to mrsl file
linkdest1 = configuration.mrsl_files_dir + client_dir + '/' + \
str(job_dict['JOB_ID']) + '.mRSL'
linkloc1 = configuration.sessid_to_mrsl_link_home + sessionid + '.mRSL'
make_symlink(linkdest1, linkloc1, logger)
# link sessionid to job owners home directory
linkdest2 = configuration.user_home + client_dir
linkloc2 = configuration.webserver_home + sessionid
make_symlink(linkdest2, linkloc2, logger)
# link iosessionid to job owners home directory
linkdest3 = configuration.user_home + client_dir
linkloc3 = configuration.webserver_home + iosessionid
make_symlink(linkdest3, linkloc3, logger)
# link sessionid to .job file
linkdest4 = configuration.mig_system_files + str(job_dict['JOB_ID']) + \
'.job'
linkloc4 = configuration.webserver_home + sessionid + '.job'
make_symlink(linkdest4, linkloc4, logger)
# link sessionid to .getupdatefiles file
linkdest5 = configuration.mig_system_files + str(job_dict['JOB_ID']) + \
'.getupdatefiles'
linkloc5 = configuration.webserver_home + sessionid + \
'.getupdatefiles'
make_symlink(linkdest5, linkloc5, logger)
# link sessionid to .sendoutputfiles file
linkdest4 = configuration.mig_system_files + str(job_dict['JOB_ID']) + \
'.sendoutputfiles'
linkloc4 = configuration.webserver_home + sessionid + \
'.sendoutputfiles'
make_symlink(linkdest4, linkloc4, logger)
# link sessionid to .sendupdatefiles file
linkdest5 = configuration.mig_system_files + str(job_dict['JOB_ID']) + \
'.sendupdatefiles'
linkloc5 = configuration.webserver_home + sessionid + \
'.sendupdatefiles'
make_symlink(linkdest5, linkloc5, logger)
path_without_extension = os.path.join(configuration.resource_home,
unique_resource_name, localjobname)
gen_res = gen_job_script(
job_dict,
resource_config,
configuration,
localjobname,
path_without_extension,
client_dir,
exe,
logger,
)
if not gen_res:
msg = \
'job scripts were not generated. Perhaps you have specified ' + \
'an invalid SCRIPTLANGUAGE ? '
print msg
logger.error(msg)
return (None, msg)
inputfiles_path = path_without_extension + '.getinputfiles'
# hack to ensure that a resource has a sandbox keyword
if resource_config.get('SANDBOX', False):
# Move file to webserver_home for download as we can't push it to
# sandboxes
try:
# RA TODO: change download filename to something that
# includes sessionid
webserver_path = os.path.join(configuration.webserver_home,
localjobname + '.getinputfiles')
os.rename(inputfiles_path, webserver_path)
# ########## ATTENTION HACK TO MAKE JVM SANDBOXES WORK ############
# This should be changed to use the (to be developed) RE pre/post
# processing framework. For now the user must have a jvm dir in his
# home dir where the classfiles is located this should be changed
# so that the execution homepath can be specified in the mRSL
# jobfile
# Martin Rehr 08/09/06
# If this is a oneclick job link the users jvm dir to
# webserver_home/sandboxkey.oneclick
# This is done because the client applet uses the
# codebase from which it is originaly loaded
# Therefore the codebase must be dynamicaly changed
# for every job
if resource_config.has_key('PLATFORM') and \
resource_config['PLATFORM'] == 'ONE-CLICK':
# A two step link is made.
# First sandboxkey.oneclick is made to point to
# sessiondid.jvm
# Second sessionid.jvm is set to point to
# USER_HOME/jvm
# This is done for security and easy cleanup,
# sessionid.jvm is cleaned up
# by the server upon job finish/timeout and
# thereby leaving no open entryes to the users
# jvm dir.
linkintermediate = configuration.webserver_home + \
sessionid + '.jvm'
if client_dir == configuration.empty_job_name:
linkdest = os.path.abspath(configuration.javabin_home)
else:
linkdest = configuration.user_home + client_dir + \
os.sep + 'jvm'
# Make link sessionid.jvm -> USER_HOME/jvm
make_symlink(linkdest, linkintermediate, logger)
linkloc = configuration.webserver_home + \
resource_config['SANDBOXKEY'] + '.oneclick'
# Remove previous symlink
# This must be done in a try/catch as the symlink,
# may be a dead link and 'if os.path.exists(linkloc):'
# will then return false, even though the link exists.
try:
os.remove(linkloc)
except:
pass
# Make link sandboxkey.oneclick -> sessionid.jvm
make_symlink(linkintermediate, linkloc, logger)
except Exception, err:
# ######### End JVM SANDBOX HACK ###########
msg = "File '%s' was not copied to the webserver home." % \
inputfiles_path
print '\nERROR: ' + str(err)
logger.error(msg)
return (None, msg)
return (job_dict, 'OK')
vgrid_name, [])
if not trigger_status:
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not save trigger list: %s' % \
trigger_msg})
return (output_objects, returnvalues.SYSTEM_ERROR)
if new_base_vgrid:
# create sym link from creators (client_id) home directory to directory
# containing the vgrid files
src = vgrid_files_dir
dst = os.path.join(configuration.user_home, client_dir,
vgrid_name)
if not make_symlink(src, dst, logger):
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create link to %s files!' % \
configuration.site_vgrid_label
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# make sure public_base dir exists in users home dir
user_public_base = os.path.join(configuration.user_home,
client_dir, 'public_base')
try:
os.mkdir(user_public_base)
except:
logger.warning("could not create %s. Probably already exists." % \
user_public_base)
def get_resource(client_id, configuration, logger):
cookie = None
sandboxkey = None
cputime = 1000000
log_msg = 'oneclick:'
__MIG_ONECLICK_COOKIE__ = 'MiGOneClickSandboxKey'
# If user with identifing cookie use cookie infomation
if os.getenv('HTTP_COOKIE') and os.getenv('HTTP_COOKIE'
).count(__MIG_ONECLICK_COOKIE__) > 0:
cookie_arr = os.getenv('HTTP_COOKIE').split(';')
for elm in cookie_arr:
if elm.count(__MIG_ONECLICK_COOKIE__) > 0:
sandboxkey = elm.split('=')[1]
break
# If we don't know user, generate an identification key
# and a new resource for him.
# The key is send to him as a cookie
if not sandboxkey or not os.path.exists(configuration.sandbox_home
+ sandboxkey):
# Generate key, and set cookie
sandboxkey = hexlify(open('/dev/urandom').read(32))
cookie = 'Set-Cookie: ' + __MIG_ONECLICK_COOKIE__ + '='\
+ sandboxkey + '; '\
+ 'expires=Thu 31-Jan-2099 12:00:00 GMT; path=/; '\
+ 'domain=' + configuration.server_fqdn + '; secure'
# Create resource
(status, msg) = create_oneclick_resource(sandboxkey, cputime,
configuration, logger)
if not status:
return (status, msg)
resource_name = msg
log_msg += ' Created resource: %s' % resource_name
# Make symbolic link from
# sandbox_home/sandboxkey to resource_home/resource_name
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(configuration.resource_home
+ resource_name)
make_symlink(resource_path, sandbox_link, logger)
else:
# Retrieve resource_name from sandboxkey symbolic link
sandbox_link = configuration.sandbox_home + sandboxkey
resource_name = os.path.basename(os.path.realpath(sandbox_link))
# If resource has a jobrequest pending, remove it.
job_pending_file = configuration.resource_home + resource_name\
+ os.sep + 'jobrequest_pending.jvm'
if os.path.exists(job_pending_file):
os.remove(job_pending_file)
log_msg += ', Remote IP: %s, Key: %s' % (os.getenv('REMOTE_ADDR'),
sandboxkey)
# Make symbolic link from webserver_home to javabin_home
codebase_link = configuration.webserver_home + sandboxkey\
+ '.oneclick'
codebase_path = os.path.abspath(configuration.javabin_home)
# Remove symbolic link if it allready exists.
# This must be done in a try/catch as the symlink,
# may be a dead link and 'if os.path.exists(linkloc):'
# will then return false, even though the link exists.
try:
os.remove(codebase_link)
except:
pass
make_symlink(codebase_path, codebase_link, logger)
logger.info(log_msg)
return (True, (sandboxkey, resource_name, cookie, cputime))
def get_resource(client_id, configuration, logger):
cookie = None
sandboxkey = None
cputime = 1000000
log_msg = 'oneclick:'
__MIG_ONECLICK_COOKIE__ = 'MiGOneClickSandboxKey'
# If user with identifing cookie use cookie infomation
if os.getenv('HTTP_COOKIE') and os.getenv('HTTP_COOKIE').count(
__MIG_ONECLICK_COOKIE__) > 0:
cookie_arr = os.getenv('HTTP_COOKIE').split(';')
for elm in cookie_arr:
if elm.count(__MIG_ONECLICK_COOKIE__) > 0:
sandboxkey = elm.split('=')[1]
break
# If we don't know user, generate an identification key
# and a new resource for him.
# The key is send to him as a cookie
if not sandboxkey or not os.path.exists(configuration.sandbox_home +
sandboxkey):
# Generate key, and set cookie
sandboxkey = hexlify(open('/dev/urandom').read(32))
cookie = 'Set-Cookie: ' + __MIG_ONECLICK_COOKIE__ + '='\
+ sandboxkey + '; '\
+ 'expires=Thu 31-Jan-2099 12:00:00 GMT; path=/; '\
+ 'domain=' + configuration.server_fqdn + '; secure'
# Create resource
(status, msg) = create_oneclick_resource(sandboxkey, cputime,
configuration, logger)
if not status:
return (status, msg)
resource_name = msg
log_msg += ' Created resource: %s' % resource_name
# Make symbolic link from
# sandbox_home/sandboxkey to resource_home/resource_name
sandbox_link = configuration.sandbox_home + sandboxkey
resource_path = os.path.abspath(configuration.resource_home +
resource_name)
make_symlink(resource_path, sandbox_link, logger)
else:
# Retrieve resource_name from sandboxkey symbolic link
sandbox_link = configuration.sandbox_home + sandboxkey
resource_name = os.path.basename(os.path.realpath(sandbox_link))
# If resource has a jobrequest pending, remove it.
job_pending_file = configuration.resource_home + resource_name\
+ os.sep + 'jobrequest_pending.jvm'
if os.path.exists(job_pending_file):
os.remove(job_pending_file)
log_msg += ', Remote IP: %s, Key: %s' % (os.getenv('REMOTE_ADDR'),
sandboxkey)
# Make symbolic link from webserver_home to javabin_home
codebase_link = configuration.webserver_home + sandboxkey\
+ '.oneclick'
codebase_path = os.path.abspath(configuration.javabin_home)
# Remove symbolic link if it allready exists.
# This must be done in a try/catch as the symlink,
# may be a dead link and 'if os.path.exists(linkloc):'
# will then return false, even though the link exists.
try:
os.remove(codebase_link)
except:
pass
make_symlink(codebase_path, codebase_link, logger)
logger.info(log_msg)
return (True, (sandboxkey, resource_name, cookie, cputime))
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
logger.debug("User: %s executing %s" % (client_id, op_name))
if not configuration.site_enable_jupyter:
output_objects.append({
'object_type':
'error_text',
'text':
'The Jupyter service is not enabled on the system'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if not configuration.site_enable_sftp_subsys and not \
configuration.site_enable_sftp:
output_objects.append({
'object_type':
'error_text',
'text':
'The required sftp service is not enabled on the system'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
if configuration.site_enable_sftp:
sftp_port = configuration.user_sftp_port
if configuration.site_enable_sftp_subsys:
sftp_port = configuration.user_sftp_subsys_port
requested_service = accepted['service'][-1]
service = {
k: v
for options in configuration.jupyter_services
for k, v in options.items()
if options['service_name'] == requested_service
}
if not service:
valid_services = [
options['name'] for options in configuration.jupyter_services
]
output_objects.append({
'object_type':
'error_text',
'text':
'%s is not a valid jupyter service, '
'allowed include %s' % (requested_service, valid_services)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
valid_service = valid_jupyter_service(configuration, service)
if not valid_service:
output_objects.append({
'object_type':
'error_text',
'text':
'The service %s appears to be misconfigured, '
'please contact a system administrator about this issue' %
requested_service
})
return (output_objects, returnvalues.SYSTEM_ERROR)
host = get_host_from_service(configuration, service)
# Get an active jupyterhost
if host is None:
logger.error("No active jupyterhub host could be found")
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to establish connection to the %s Jupyter service' %
service['service_name']
})
output_objects.append({
'object_type': 'link',
'destination': 'jupyter.py',
'text': 'Back to Jupyter services overview'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
remote_user = unescape(os.environ.get('REMOTE_USER', '')).strip()
if not remote_user:
logger.error("Can't connect to jupyter with an empty REMOTE_USER "
"environment variable")
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to establish connection to the Jupyter service'
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Ensure the remote_user dict can be http posted
remote_user = str(remote_user)
# TODO, activate admin info
# remote_user = {'USER': username, 'IS_ADMIN': is_admin(client_id,
# configuration,
# logger)}
# Regular sftp path
mnt_path = os.path.join(configuration.jupyter_mount_files_dir, client_dir)
# Subsys sftp path
subsys_path = os.path.join(configuration.mig_system_files, 'jupyter_mount')
# sftp session path
link_home = configuration.sessid_to_jupyter_mount_link_home
user_home_dir = os.path.join(configuration.user_home, client_dir)
# Preparing prerequisites
if not os.path.exists(mnt_path):
os.makedirs(mnt_path)
if not os.path.exists(link_home):
os.makedirs(link_home)
if configuration.site_enable_sftp_subsys:
if not os.path.exists(subsys_path):
os.makedirs(subsys_path)
# Make sure ssh daemon does not complain
tighten_key_perms(configuration, client_id)
url_base = '/' + service['service_name']
url_home = url_base + '/home'
url_auth = host + url_base + '/hub/login'
url_data = host + url_base + '/hub/user-data'
# Does the client home dir contain an active mount key
# If so just keep on using it.
jupyter_mount_files = [
os.path.join(mnt_path, jfile) for jfile in os.listdir(mnt_path)
if jfile.endswith('.jupyter_mount')
]
logger.info("User: %s mount files: %s" %
(client_id, "\n".join(jupyter_mount_files)))
logger.debug("Remote-User %s" % remote_user)
active_mounts = []
for jfile in jupyter_mount_files:
jupyter_dict = unpickle(jfile, logger)
if not jupyter_dict:
# Remove failed unpickle
logger.error("Failed to unpickle %s removing it" % jfile)
remove_jupyter_mount(jfile, configuration)
else:
# Mount has been timed out
if not is_active(jupyter_dict):
remove_jupyter_mount(jfile, configuration)
else:
# Valid mount
active_mounts.append({'path': jfile, 'state': jupyter_dict})
logger.debug(
"User: %s active keys: %s" %
(client_id, "\n".join([mount['path'] for mount in active_mounts])))
# If multiple are active, remove oldest
active_mount, old_mounts = get_newest_mount(active_mounts)
for mount in old_mounts:
remove_jupyter_mount(mount['path'], configuration)
# A valid active key is already present redirect straight to the jupyter
# service, pass most recent mount information
if active_mount is not None:
mount_dict = mig_to_mount_adapt(active_mount['state'])
user_dict = mig_to_user_adapt(active_mount['state'])
logger.debug("Existing header values, Mount: %s User: %s" %
(mount_dict, user_dict))
auth_header = {'Remote-User': remote_user}
json_data = {'data': {'Mount': mount_dict, 'User': user_dict}}
if configuration.site_enable_workflows:
workflows_dict = mig_to_workflows_adapt(active_mount['state'])
if not workflows_dict:
# No cached workflows session could be found -> refresh with a
# one
workflow_session_id = get_workflow_session_id(
configuration, client_id)
if not workflow_session_id:
workflow_session_id = create_workflow_session_id(
configuration, client_id)
# TODO get this dynamically
url = configuration.migserver_https_sid_url + \
'/cgi-sid/workflowsjsoninterface.py?output_format=json'
workflows_dict = {
'WORKFLOWS_URL': url,
'WORKFLOWS_SESSION_ID': workflow_session_id
}
logger.debug("Existing header values, Workflows: %s" %
workflows_dict)
json_data['workflows_data'] = {'Session': workflows_dict}
with requests.session() as session:
# Authenticate and submit data
response = session.post(url_auth, headers=auth_header)
if response.status_code == 200:
response = session.post(url_data, json=json_data)
if response.status_code != 200:
logger.error(
"Jupyter: User %s failed to submit data %s to %s" %
(client_id, json_data, url_data))
else:
logger.error(
"Jupyter: User %s failed to authenticate against %s" %
(client_id, url_auth))
# Redirect client to jupyterhub
return jupyter_host(configuration, output_objects, remote_user,
url_home)
# Create a new keyset
# Create login session id
session_id = generate_random_ascii(2 * session_id_bytes,
charset='0123456789abcdef')
# Generate private/public keys
(mount_private_key,
mount_public_key) = generate_ssh_rsa_key_pair(encode_utf8=True)
# Known hosts
sftp_addresses = socket.gethostbyname_ex(
configuration.user_sftp_show_address or socket.getfqdn())
# Subsys sftp support
if configuration.site_enable_sftp_subsys:
# Restrict possible mount agent
auth_content = []
restrict_opts = 'no-agent-forwarding,no-port-forwarding,no-pty,'
restrict_opts += 'no-user-rc,no-X11-forwarding'
restrictions = '%s' % restrict_opts
auth_content.append('%s %s\n' % (restrictions, mount_public_key))
# Write auth file
write_file('\n'.join(auth_content),
os.path.join(subsys_path, session_id + '.authorized_keys'),
logger,
umask=027)
logger.debug("User: %s - Creating a new jupyter mount keyset - "
"private_key: %s public_key: %s " %
(client_id, mount_private_key, mount_public_key))
jupyter_dict = {
'MOUNT_HOST': configuration.short_title,
'SESSIONID': session_id,
'USER_CERT': client_id,
# don't need fraction precision, also not all systems provide fraction
# precision.
'CREATED_TIMESTAMP': int(time.time()),
'MOUNTSSHPRIVATEKEY': mount_private_key,
'MOUNTSSHPUBLICKEY': mount_public_key,
# Used by the jupyterhub to know which host to mount against
'TARGET_MOUNT_ADDR': "@" + sftp_addresses[0] + ":",
'PORT': sftp_port
}
client_email = extract_field(client_id, 'email')
if client_email:
jupyter_dict.update({'USER_EMAIL': client_email})
if configuration.site_enable_workflows:
workflow_session_id = get_workflow_session_id(configuration, client_id)
if not workflow_session_id:
workflow_session_id = create_workflow_session_id(
configuration, client_id)
# TODO get this dynamically
url = configuration.migserver_https_sid_url + \
'/cgi-sid/workflowsjsoninterface.py?output_format=json'
jupyter_dict.update({
'WORKFLOWS_URL': url,
'WORKFLOWS_SESSION_ID': workflow_session_id
})
# Only post the required keys, adapt to API expectations
mount_dict = mig_to_mount_adapt(jupyter_dict)
user_dict = mig_to_user_adapt(jupyter_dict)
workflows_dict = mig_to_workflows_adapt(jupyter_dict)
logger.debug("User: %s Mount header: %s" % (client_id, mount_dict))
logger.debug("User: %s User header: %s" % (client_id, user_dict))
if workflows_dict:
logger.debug("User: %s Workflows header: %s" %
(client_id, workflows_dict))
# Auth and pass a new set of valid mount keys
auth_header = {'Remote-User': remote_user}
json_data = {'data': {'Mount': mount_dict, 'User': user_dict}}
if workflows_dict:
json_data['workflows_data'] = {'Session': workflows_dict}
# First login
with requests.session() as session:
# Authenticate
response = session.post(url_auth, headers=auth_header)
if response.status_code == 200:
response = session.post(url_data, json=json_data)
if response.status_code != 200:
logger.error(
"Jupyter: User %s failed to submit data %s to %s" %
(client_id, json_data, url_data))
else:
logger.error("Jupyter: User %s failed to authenticate against %s" %
(client_id, url_auth))
# Update pickle with the new valid key
jupyter_mount_state_path = os.path.join(mnt_path,
session_id + '.jupyter_mount')
pickle(jupyter_dict, jupyter_mount_state_path, logger)
# Link jupyter pickle state file
linkdest_new_jupyter_mount = os.path.join(mnt_path,
session_id + '.jupyter_mount')
linkloc_new_jupyter_mount = os.path.join(link_home,
session_id + '.jupyter_mount')
make_symlink(linkdest_new_jupyter_mount, linkloc_new_jupyter_mount, logger)
# Link userhome
linkloc_user_home = os.path.join(link_home, session_id)
make_symlink(user_home_dir, linkloc_user_home, logger)
return jupyter_host(configuration, output_objects, remote_user, url_home)
output_objects.append(
{'object_type': 'error_text', 'text'
: ('Could not create needed dirs on %s server! %s'
% (configuration.short_title, exc))})
logger.error('%s when looking for dir %s.' % (exc, dir1))
return (output_objects, returnvalues.SYSTEM_ERROR)
# create symlink from users home directory to vgrid file directory
link_src = os.path.abspath(configuration.vgrid_files_home + os.sep
+ vgrid_name) + os.sep
link_dst = user_dir + vgrid_name
# create symlink to vgrid files
if not make_symlink(link_src, link_dst, logger):
output_objects.append({'object_type': 'error_text', 'text'
: 'Could not create link to %s files!' % \
configuration.site_vgrid_label
})
logger.error('Could not create link to %s files! (%s -> %s)'
% (configuration.site_vgrid_label, link_src, link_dst))
return (output_objects, returnvalues.SYSTEM_ERROR)
output_objects.append({'object_type': 'text', 'text'
: 'New member %s successfully added to %s %s!'
% (cert_id, vgrid_name,
configuration.site_vgrid_label)})
output_objects.append({'object_type': 'html_form', 'text'
: """
<form method='post' action='sendrequestaction.py'>
for (meta_key, meta_label) in public_meta:
meta_value = freeze_dict.get(meta_key, '')
if meta_value:
contents += """%s: %s<br/>
""" % (meta_label, meta_value)
contents += """
<h2 class='staticpage'>Archive Files</h2>
"""
for rel_path in frozen_files:
contents += """<a href='%s'>%s</a><br/>
""" % (rel_path, rel_path)
contents += """
</div>
%s
""" % get_cgi_html_footer(configuration, widgets=False)
if not make_symlink(frozen_dir, real_pub_dir, logger) or \
not write_file(contents, real_pub_index, configuration.logger):
logger.error("create_frozen_archive: publish failed")
remove_rec(frozen_dir, configuration)
return (False, 'Error publishing frozen archive')
return (True, freeze_id)
def delete_frozen_archive(freeze_id, configuration):
"""Delete an existing frozen archive without checking ownership or
persistance of frozen archives.
"""
frozen_dir = os.path.join(configuration.freeze_home, freeze_id)
if remove_rec(frozen_dir, configuration):
return (True, '')
else:
return (False, 'Error deleting frozen archive "%s"' % freeze_id)
