def _get_jsurl(self, pilot):
'''
get job service endpoint and hop URL for the pilot's target resource.
'''
resrc = pilot['description']['resource']
schema = pilot['description']['access_schema']
rcfg = self.get_resource_config(resrc, schema)
js_url = rs.Url(rcfg.get('job_manager_endpoint'))
js_hop = rs.Url(rcfg.get('job_manager_hop', js_url))
# make sure the js_hop url points to an interactive access
# TODO: this is an unreliable heuristics - we should require the js_hop
# URL to be specified in the resource configs.
if '+gsissh' in js_hop.schema or \
'gsissh+' in js_hop.schema :
js_hop.schema = 'gsissh'
elif '+ssh' in js_hop.schema or \
'ssh+' in js_hop.schema :
js_hop.schema = 'ssh'
else:
js_hop.schema = 'fork'
return js_url, js_hop
def upload (self, source, target, flags) :
'''Uploads a file from the LOCAL, PHYSICAL filesystem to
the replica management system.
@param source: URL (should be file:// or local path) of local file
@param target: Optional param containing ?resource=myresource query
This will upload the file to a specified iRODS
resource or group.
'''
# TODO: Make sure that the source URL is a local/file:// URL
complete_path = rs.Url(source).get_path()
# extract the path from the LogicalFile object, excluding
# the filename
destination_path = self._url.get_path()[0:string.rfind(
self._url.get_path(), "/") + 1]
try:
# note we're uploading
self._logger.debug("Beginning upload operation "
"will register file in logical dir: %s"
% destination_path)
# the query holds our target resource
query = rs.Url(target).get_query()
# list of args we will generate
arg_list = ""
# parse flags + generate args
if flags:
if flags & rs.namespace.OVERWRITE:
arg_list += "-f "
# was no resource selected?
if not query:
self._logger.debug("Attempting to upload to default resource")
ret, out, _ = self.shell.run_sync("iput %s %s %s" %
(arg_list, complete_path, destination_path))
# resource was selected, have to parse it and supply to iput -R
else:
# TODO: Verify correctness
resource = query.split("=")[1]
self._logger.debug("upload to %s" % resource)
ret, out, _ = self.shell.run_sync("iput -R %s %s %s %s" %
(resource, arg_list, complete_path,
destination_path))
if ret:
raise rs.NoSuccess("Could not upload file %s, errorcode %s: %s"
% (complete_path, str(ret), out))
except Exception, ex:
# couldn't upload for unspecificed reason
raise rs.NoSuccess._log (self._logger, "upload failed: %s" % ex)
def test_jobid_viability():
""" Test if jobid represents job """
# The job id for the fork shell adaptor should return a pid which represents
# the actual job instance. We test by killing that pid and checking state.
try:
import os
cfg = config()
js_url = rs.Url(cfg.job_service_url)
if js_url.schema.lower() not in ['fork', 'local', 'ssh']:
# test not supported for other backends
return
if js_url.host.lower() not in [None, '', 'localhost']:
# test not supported for other backends
return
js = rs.job.Service('fork:///')
j = js.run_job("/bin/sleep 100")
jid = j.id
js_part, j_part = jid.split('-', 1)
pid = j_part[1:-3]
# kill the children (i.e. the only child) of the pid, which is the
# actual job
os.system(('ps -ef | cut -c 8-21 | grep " %s " | cut -c 1-8 '
'| grep -v " %s " | xargs -r kill') % (pid, pid))
assert (j.state == rs.job.FAILED), 'job.state: %s' % j.state
except rs.SagaException as se:
assert False, "Unexpected exception: %s" % se
def remove(self, path, flags):
'''This method is called upon logicaldir.remove() '''
complete_path = rs.Url(path).get_path()
try:
self._logger.debug("Executing: irm -r %s" % complete_path)
ret, out, _ = self.shell.run_sync("irm -r %s" % complete_path)
if ret != 0:
raise rs.NoSuccess("Could not remove directory %s [%s]: %s" %
(complete_path, str(ret), out))
except Exception as ex:
# was there no directory to delete?
if "does not exist" in str(ex):
raise rs.DoesNotExist("Directory %s does not exist." %
complete_path)
# couldn't delete for unspecificed reason
raise rs.NoSuccess("Couldn't delete directory %s because %s" %
(complete_path, ex))
return
def get_js_shell(self, resource, schema):
if resource not in self._cache['js_shells']:
self._cache['js_shells'][resource] = dict()
if schema not in self._cache['js_shells'][resource]:
rcfg = self.get_resource_config(resource, schema)
js_url = rcfg['job_manager_endpoint']
js_url = rcfg.get('job_manager_hop', js_url)
js_url = rs.Url(js_url)
elems = js_url.schema.split('+')
if 'ssh' in elems: js_url.schema = 'ssh'
elif 'gsissh' in elems: js_url.schema = 'gsissh'
elif 'fork' in elems: js_url.schema = 'fork'
elif len(elems) == 1 : js_url.schema = 'fork'
else: raise Exception("invalid schema: %s" % js_url.schema)
if js_url.schema == 'fork':
js_url.hostname = 'localhost'
self._log.debug("rsup.PTYShell('%s')", js_url)
shell = rsup.PTYShell(js_url, self)
self._cache['js_shells'][resource][schema] = shell
return self._cache['js_shells'][resource][schema]
def benchmark_pre (tid, test_cfg, bench_cfg, session) :
if not 'job_service_url' in test_cfg :
raise saga.NoSuccess ('no job service URL configured')
if not 'load' in bench_cfg :
raise saga.NoSuccess ('no benchmark load configured')
host = saga.Url(test_cfg['job_service_url']).host
n_j = int(bench_cfg['iterations'])
load = int(bench_cfg['load'])
exe = '/bin/sleep'
ssh = subprocess ("ssh %s" % host, stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
# find the ssh prompt
stdin = ssh.communicate[0]
stdout = ssh.communicate[1]
while <stdin> ~! /'>$'/io :
time.sleep (0.1)
# setup is done
return {'ssh' : ssh, 'cmd' : "%s %s" % (executable, load)}
def replicate (self, target, flags):
'''This method is called upon logicaldir.replicate()
'''
# path to file we are replicating on iRODS
complete_path = self._url.get_path()
# TODO: Verify Correctness in the way the resource is grabbed
query = rs.Url(target).get_query()
resource = query.split("=")[1]
self._logger.debug("Attempting to replicate logical file %s to "
"resource/resource group %s"
% (complete_path, resource))
try:
self._logger.debug("Executing: irepl -R %s %s"
% (resource, complete_path))
ret, out, _ = self.shell.run_sync("irepl -R %s %s"
% (resource, complete_path))
if ret:
raise Exception("Could not replicate logical file %s to"
"resource/resource group %s, errorcode %s: %s"
% (complete_path, resource, str(ret), out))
except Exception, ex:
raise rs.NoSuccess._log(self._logger, "replicate failed: %s" % ex)
def test_job_service_invalid_url(self):
""" Test if a non-resolvable hostname results in a proper exception
"""
tmp_js = None
try:
invalid_url = rs.Url(self.cfg['job_service_url'])
invalid_url.schema = "ssh"
invalid_url.host = "does.not.exist"
tmp_js = rs.job.Service(invalid_url, self.session)
assert False, "Expected BadParameter exception but got none."
except rs.BadParameter:
assert True
# we don't check DNS anymore, as that can take *ages* -- so we now also
# see Timeout and NoSuccess exceptions...
except (rs.NoSuccess, rs.Timeout):
assert True
# other exceptions should never occur
except rs.SagaException as ex:
assert False, 'Expected BadParameter, Timeout or NoSuccess' \
'exception, but got %s (%s)' % (type(ex), ex)
finally:
_silent_close_js(tmp_js)
def test_file_copy_1(self):
""" Testing if we can copy an existing file.
"""
try:
pass
tc = config()
filename1 = deepcopy(rs.Url(tc.filesystem_url))
filename1.path += "/%s" % self.uniquefilename1
f1 = rs.filesystem.File(filename1, rs.filesystem.CREATE)
filename2 = deepcopy(rs.Url(tc.filesystem_url))
filename2.path += "/%s" % self.uniquefilename2
f1.copy(filename2)
f2 = rs.filesystem.File(filename2)
assert f2.size == 0 # this should fail if the file doesn't exist!
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_nonexisting_file_create_open(self):
""" Testing if opening a non-existing file with the 'create' flag set works.
"""
try:
pass
tc = config()
nonex_file = deepcopy(rs.Url(tc.filesystem_url))
nonex_file.path += "/%s" % self.uniquefilename1
f = rs.filesystem.File(nonex_file, rs.filesystem.CREATE)
assert f.size == 0 # this should fail if the file doesn't exist!
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_file_copy_invalid_tgt_scheme(self):
""" Testing if get an exception if we try to copy an unsupported target scheme.
"""
try:
tc = config()
# Create the source file
source_file = deepcopy(rs.Url(tc.filesystem_url))
source_file.path += "/%s" % self.uniquefilename1
f1 = rs.filesystem.File(source_file, rs.filesystem.CREATE)
target_url = deepcopy(rs.Url(tc.filesystem_url))
target_url.scheme = "crapscheme"
f1.copy(target_url)
assert False, "Expected BadParameter exception but got none."
except rs.BadParameter:
assert True
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def _get_pilot_sandbox(self, pilot):
# FIXME: this should get 'pid, resource, schema=None' as parameters
pilot_sandbox = pilot.get('pilot_sandbox')
if str(pilot_sandbox):
return rs.Url(pilot_sandbox)
pid = pilot['uid']
with self._cache_lock:
if pid in self._cache['pilot_sandbox']:
return self._cache['pilot_sandbox'][pid]
# cache miss
session_sandbox = self._get_session_sandbox(pilot)
pilot_sandbox = rs.Url(session_sandbox)
pilot_sandbox.path += '/%s/' % pilot['uid']
with self._cache_lock:
self._cache['pilot_sandbox'][pid] = pilot_sandbox
return pilot_sandbox
def test_ptyshell_nok():
""" Test pty_shell which runs command unsuccessfully """
conf = config()
shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)
txt = "______1______2_____3_____"
ret, out, _ = shell.run_sync("printf \"%s\" ; false" % txt)
assert (ret == 1), "%s" % (repr(ret))
assert (out == txt), "%s == %s" % (repr(out), repr(txt))
assert (shell.alive())
shell.finalize(True)
assert (not shell.alive())
def test_nonexisting_host_file_open(self):
""" Testing if opening a file on a non-existing host causes an exception.
"""
try:
tc = config()
invalid_url = deepcopy(rs.Url(tc.filesystem_url))
invalid_url.host = "does.not.exist"
_ = rs.filesystem.File(invalid_url)
assert False, "Expected BadParameter exception but got none."
except rs.BadParameter:
assert True
except rs.SagaException as ex:
assert False, "Expected BadParameter exception, but got %s" % ex
def test_existing_file_open(self):
""" Testing if we can open an existing file.
"""
try:
tc = config()
filename = deepcopy(rs.Url(tc.filesystem_url))
filename.path += "/%s" % self.uniquefilename1
f = rs.filesystem.File(filename, rs.filesystem.CREATE)
f2 = rs.filesystem.File(f.url)
assert f2.size == 0 # this should fail if the file doesn't exist!
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_ptyshell_file_stage():
""" Test pty_shell file staging """
conf = config()
shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)
txt = "______1______2_____3_____"
shell.write_to_remote(txt, "/tmp/saga-test-staging")
out = shell.read_from_remote("/tmp/saga-test-staging")
assert (txt == out), "%s == %s" % (repr(out), repr(txt))
ret, out, _ = shell.run_sync("rm /tmp/saga-test-staging")
assert (ret == 0), "%s" % (repr(ret))
assert (out == ""), "%s == ''" % (repr(out))
def test_nonexisting_file_open(self):
""" Testing if opening a non-existing file causes an exception.
"""
try:
pass
tc = config()
nonex_file = deepcopy(rs.Url(tc.filesystem_url))
nonex_file.path += "/file.does.not.exist"
_ = rs.filesystem.File(nonex_file)
assert False, "Expected DoesNotExist exception but got none."
except rs.DoesNotExist:
assert True
except rs.SagaException as ex:
assert False, "Expected DoesNotExist exception, but got %s" % ex
def download (self, name, source, flags) :
'''Downloads a file from the REMOTE REPLICA FILESYSTEM to a local
directory.
@param target: param containing a local path/filename
to save the file to
@param source: Optional param containing a remote replica to retrieve
the file from (not evaluated, yet)
'''
target = name
# TODO: Make sure that the target URL is a local/file:// URL
# extract the path from the LogicalFile object, excluding
# the filename
logical_path = self._url.get_path()
# fill in our local path if one was specified
local_path = ""
if target:
local_path = rs.Url(target).get_path()
try:
# var to hold our command result, placed here to keep in scope
ret = out = _ = 0
# mark that this is experimental/may not be part of official API
self._logger.debug("Beginning download operation "
"will download logical file: %s, "
"specified local target is %s"
% (logical_path, target))
if target: cmd = "iget %s %s" % (logical_path, local_path)
else : cmd = "iget %s" % logical_path
self._logger.debug("Executing: %s" % cmd)
ret, out, _ = self.shell.run_sync(cmd)
# check our result
if ret:
raise rs.NoSuccess("download failed: %s [%s]: %s"
% (logical_path, str(ret), out))
except Exception, ex:
# couldn't download for unspecificed reason
raise rs.NoSuccess ("Couldn't download file. %s" % ex)
def test_file_copy_remote_local(self):
""" Testing if we can copy a file from remote -> local
"""
try:
pass
tc = config()
filename1 = deepcopy(rs.Url(tc.filesystem_url))
filename1.path = "/etc/passwd"
f1 = rs.filesystem.File(filename1)
filename2 = "file://localhost/tmp/%s" % self.uniquefilename2
f1.copy(filename2)
f2 = rs.filesystem.File(filename2)
assert f2.size != 0 # this should fail if the file doesn't exist!
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_file_get_session(self):
""" Testing if the correct session is being used
"""
try:
tc = config()
session = tc.session or rs.Session()
filename = deepcopy(rs.Url(tc.filesystem_url))
filename.path += "/%s" % self.uniquefilename1
f = rs.filesystem.File(filename, rs.filesystem.CREATE,
session=session)
assert f.get_session() == session, 'Failed setting the session'
f2 = rs.filesystem.File(f.url, session=session)
assert f2.get_session() == session, 'Failed setting the session'
except rs.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_ptyshell_async():
""" Test pty_shell which runs command successfully """
conf = config()
shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)
txt = "______1______2_____3_____\n"
shell.run_async("cat <<EOT")
shell.send(txt)
shell.send('EOT\n')
ret, out = shell.find_prompt()
assert (ret == 0), "%s" % (repr(ret))
assert (out == txt), "%s == %s" % (repr(out), repr(txt))
assert (shell.alive())
shell.finalize(True)
assert (not shell.alive())
def make_dir (self, path, flags) :
# complete_path = dir_obj._url.path
complete_path = rs.Url(path).get_path()
# attempt to run iRODS mkdir command
try:
ret, out, _ = self.shell.run_sync("imkdir %s" % complete_path)
if ret != 0:
raise rs.NoSuccess("Could not create dir %s, [%s]: %s"
% (complete_path, str(ret), out))
except Exception, ex:
# did the directory already exist?
if "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME" in str(ex):
raise rs.AlreadyExists ("Directory already exists.")
# couldn't create for unspecificed reason
raise rs.NoSuccess ("Couldn't create directory. %s" % ex)
def _prepare_job(self, job):
"""
Prepare job on remote host
- Create remote working dir
- Upload job input files
"""
job.logger.debug('Prepared job in ShellAdapter')
try:
work_dir = self.job_work_dir(job, saga.filesystem.CREATE_PARENTS)
for input_file in job.input_files:
wrapper = saga.filesystem.File(
saga.Url('file://localhost/%s' %
join(job.working_dir, input_file.value)))
wrapper.copy(work_dir.get_url())
job.logger.debug("Uploaded file %s to %s",
join(job.working_dir, input_file.value),
work_dir.get_url())
return job
except saga.SagaException as exc:
raise AdaptorJobException(exc.message)
def move_self (self, target, flags) :
'''This method is called upon logicaldir.move() '''
# path to file we are moving on iRODS
source_path = self._url.get_path()
dest_path = rs.Url(target).get_path()
self._logger.debug("Attempting to move logical file %s to location %s"
% (source_path, dest_path))
try:
ret, out, _ = self.shell.run_sync("imv %s %s"
% (source_path, dest_path))
if ret:
raise rs.NoSuccess("Could not move logical file %s to location"
"%s, errorcode %s: %s"
% (source_path, dest_path, str(ret), out))
except Exception, ex:
raise rs.NoSuccess ("Couldn't move file: %s" % ex)
def test_ptyshell_prompt():
""" Test pty_shell with prompt change """
conf = config()
shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)
txt = "______1______2_____3_____"
ret, out, _ = shell.run_sync("printf \"%s\"" % txt)
assert (ret == 0), "%s" % (repr(ret))
assert (out == txt), "%s == %s" % (repr(out), repr(txt))
shell.run_sync('export PS1="HALLO-(\\$?)-PROMPT>"',
new_prompt='HALLO-\((\d)\)-PROMPT>')
txt = "______1______2_____3_____"
ret, out, _ = shell.run_sync("printf \"%s\"" % txt)
assert (ret == 0), "%s" % (repr(ret))
assert (out == txt), "%s == %s" % (repr(out), repr(txt))
assert (shell.alive())
shell.finalize(True)
assert (not shell.alive())
def _get_session_sandbox(self, pilot):
# FIXME: this should get 'resource, schema=None' as parameters
resource = pilot['description'].get('resource')
if not resource:
raise ValueError('Cannot get session sandbox w/o resource target')
with self._cache_lock:
if resource not in self._cache['session_sandbox']:
# cache miss
resource_sandbox = self._get_resource_sandbox(pilot)
session_sandbox = rs.Url(resource_sandbox)
session_sandbox.path += '/%s' % self.uid
self._cache['session_sandbox'][resource] = session_sandbox
return self._cache['session_sandbox'][resource]
def main():
tmp_dir = None
try:
tmp_dir = tempfile.mkdtemp(prefix='saga-test-',
suffix='-%s' % TEST_NAME,
dir=os.path.expanduser('~/tmp'))
print('tmpdir: %s' % tmp_dir)
ctx = saga.Context("x509")
ctx.user_proxy = '/Users/mark/proj/myproxy/xsede.x509'
session = saga.Session()
session.add_context(ctx)
source_url = saga.Url()
source_url.schema = 'go'
source_url.host = SOURCE
source_url.path = tmp_dir
target_url = saga.Url()
target_url.schema = 'go'
target_url.host = TARGET
target_url.path = os.path.join('~/saga-tests/',
os.path.basename(tmp_dir))
print("Point to local Directory through GO ...")
d = saga.filesystem.Directory(source_url)
print("And check ...")
assert d.is_dir() == True
assert d.is_file() == False
assert d.is_link() == False
d.close()
print("Point to remote Directory through GO ...")
d = saga.filesystem.Directory(target_url,
flags=saga.filesystem.CREATE_PARENTS)
print("And check ...")
assert d.is_dir() == True
assert d.is_file() == False
assert d.is_link() == False
d.close()
print("Point to local file through GO, before creation ...")
caught = False
try:
saga.filesystem.File(os.path.join(str(source_url), FILE_A_level_0))
except saga.DoesNotExist:
caught = True
assert caught == True
print("Create actual file ...")
touch(tmp_dir, FILE_A_level_0)
print("Try again ...")
f = saga.filesystem.File(os.path.join(str(source_url), FILE_A_level_0))
assert f.is_file() == True
assert f.is_dir() == False
assert f.is_link() == False
f.close()
print("Copy local file to remote, using different filename ...")
d = saga.filesystem.Directory(target_url,
flags=saga.filesystem.CREATE_PARENTS)
d.copy(os.path.join(str(source_url), FILE_A_level_0),
FILE_A_level_0 + COPIED_SUFFIX)
d.close()
f = saga.filesystem.File(
os.path.join(str(target_url), FILE_A_level_0 + COPIED_SUFFIX))
assert f.is_file() == True
assert f.is_dir() == False
assert f.is_link() == False
f.close()
print("Copy local file to remote, keeping filename in tact ...")
d = saga.filesystem.Directory(target_url,
flags=saga.filesystem.CREATE_PARENTS)
d.copy(os.path.join(str(source_url), FILE_A_level_0), FILE_A_level_0)
d.close()
f = saga.filesystem.File(os.path.join(str(target_url), FILE_A_level_0))
assert f.is_file() == True
assert f.is_dir() == False
assert f.is_link() == False
f.close()
print('Create file in level 1 ...')
tree = LEVEL_1
os.mkdir(os.path.join(tmp_dir, tree))
touch(os.path.join(tmp_dir, tree), FILE_A_level_1)
print("Test local file ...")
f = saga.filesystem.File(
os.path.join(str(source_url), tree, FILE_A_level_1))
assert f.is_file() == True
assert f.is_dir() == False
assert f.is_link() == False
f.close()
print("Copy local file to remote, keeping filename in tact ...")
d = saga.filesystem.Directory(os.path.join(str(target_url), tree),
flags=saga.filesystem.CREATE_PARENTS)
d.copy(os.path.join(str(source_url), tree, FILE_A_level_1),
FILE_A_level_1)
d.close()
print("Test file after transfer ...")
f = saga.filesystem.File(
os.path.join(str(target_url), tree, FILE_A_level_1))
assert f.is_file() == True
assert f.is_dir() == False
assert f.is_link() == False
f.close()
print(
"Copy non-existent local file to remote, keeping filename in tact ..."
)
d = saga.filesystem.Directory(str(target_url),
flags=saga.filesystem.CREATE_PARENTS)
try:
d.copy(os.path.join(str(source_url), NON_EXISTING_FILE),
NON_EXISTING_FILE)
except saga.DoesNotExist:
caught = True
assert caught == True
print("Test file after (non-)transfer ...")
caught = False
try:
saga.filesystem.File(
os.path.join(str(target_url), NON_EXISTING_FILE))
except saga.DoesNotExist:
caught = True
assert caught == True
# destination = "go://gridftp.stampede.tacc.xsede.org/~/tmp/"
# #destination = "go://oasis-dm.sdsc.xsede.org/~/tmp/"
# #destination = "go://ncsa#BlueWaters/~/tmp/"
# #destination = "go://marksant#netbook/Users/mark/tmp/go/"
# src_filename = "my_file"
# dst_filename = "my_file_"
# rt_filename = "my_file__"
#
# # open home directory on a remote machine
# source_dir = saga.filesystem.Directory(source)
#
# # copy .bash_history to /tmp/ on the local machine
# source_dir.copy(src_filename, os.path.join(destination, dst_filename))
#
# # list 'm*' in local /tmp/ directory
# dest_dir = saga.filesystem.Directory(destination)
# for entry in dest_dir.list(pattern='%s*' % src_filename[0]):
# print entry
#
# dest_file = saga.filesystem.File(os.path.join(destination, dst_filename))
# assert dest_file.is_file() == True
# assert dest_file.is_link() == False
# assert dest_file.is_dir() == False
# print 'Size: %d' % dest_file.get_size()
#
# dest_file.copy(source)
#
# dest_file.copy(os.path.join(source+'broken', rt_filename))
print("Before return 0")
return 0
except saga.SagaException as ex:
# Catch all saga exceptions
print("An exception occurred: (%s) %s " % (ex.type, (str(ex))))
# Trace back the exception. That can be helpful for debugging.
print(" \n*** Backtrace:\n %s" % ex.traceback)
print("before return -1")
return -1
finally:
print("and finally ...")
if CLEANUP and tmp_dir:
shutil.rmtree(tmp_dir)
# ------------------------------------------------------------------------------
#
# set up the connection to EC2
#
if not 'EC2_URL' in os.environ: usage("no %s in environment" % 'EC2_URL')
if not 'EC2_ACCESS_KEY' in os.environ:
usage("no %s in environment" % 'EC2_ACCESS_KEY')
if not 'EC2_SECRET_KEY' in os.environ:
usage("no %s in environment" % 'EC2_SECRET_KEY')
if not 'EC2_KEYPAIR_ID' in os.environ:
usage("no %s in environment" % 'EC2_KEYPAIR_ID')
if not 'EC2_KEYPAIR' in os.environ:
usage("no %s in environment" % 'EC2_KEYPAIR')
server = rs.Url(os.environ['EC2_URL'])
# in order to connect to EC2, we need an EC2 ID and KEY
c1 = rs.Context('ec2')
c1.user_id = os.environ['EC2_ACCESS_KEY']
c1.user_key = os.environ['EC2_SECRET_KEY']
c1.server = server
# in order to access a created VM, we additionally need to point to the ssh
# key which is used for EC2 VM contextualization, i.e. as EC2 'keypair'.
# If the keypair is not yet registered on EC2, it will be registered by SAGA
# -- but then a user_key *must* be specified (only the public key is ever
# transfererd to EC2).
c2 = rs.Context('ec2_keypair')
c2.token = os.environ['EC2_KEYPAIR_ID']
c2.user_cert = os.environ['EC2_KEYPAIR']
def job_work_dir(self, job, mode=saga.filesystem.READ):
""" Setup remote host working dir """
return saga.filesystem.Directory(saga.Url(
'%s/%s' % (self.remote_dir, str(job.slug))),
mode,
session=self.session)
def _get_resource_sandbox(self, pilot):
'''
for a given pilot dict, determine the global RP sandbox, based on the
pilot's 'resource' attribute.
'''
# FIXME: this should get 'resource, schema=None' as parameters
resource = pilot['description'].get('resource')
schema = pilot['description'].get('access_schema')
if not resource:
raise ValueError('Cannot get pilot sandbox w/o resource target')
# the global sandbox will be the same for all pilots on any resource, so
# we cache it
with self._cache_lock:
if resource not in self._cache['resource_sandbox']:
# cache miss -- determine sandbox and fill cache
rcfg = self.get_resource_config(resource, schema)
fs_url = rs.Url(rcfg['filesystem_endpoint'])
# Get the sandbox from either the pilot_desc or resource conf
sandbox_raw = pilot['description'].get('sandbox')
if not sandbox_raw:
sandbox_raw = rcfg.get('default_remote_workdir', "$PWD")
# we may need to replace pat elements with data from the pilot
# description
if '%' in sandbox_raw:
# expand from pilot description
expand = dict()
for k,v in pilot['description'].items():
if v is None:
v = ''
expand['pd.%s' % k] = v
if isinstance(v, str):
expand['pd.%s' % k.upper()] = v.upper()
expand['pd.%s' % k.lower()] = v.lower()
else:
expand['pd.%s' % k.upper()] = v
expand['pd.%s' % k.lower()] = v
sandbox_raw = sandbox_raw % expand
if '_' in sandbox_raw and 'summit' in resource:
sandbox_raw = sandbox_raw.split('_')[0]
# If the sandbox contains expandables, we need to resolve those
# remotely.
#
# NOTE: this will only work for (gsi)ssh or similar shell
# based access mechanisms
if '$' not in sandbox_raw:
# no need to expand further
sandbox_base = sandbox_raw
else:
shell = self.get_js_shell(resource, schema)
ret, out, err = shell.run_sync(' echo "WORKDIR: %s"'
% sandbox_raw)
if ret or 'WORKDIR:' not in out:
raise RuntimeError("Couldn't get remote workdir.")
sandbox_base = out.split(":")[1].strip()
self._log.debug("sandbox base %s", sandbox_base)
# at this point we have determined the remote 'pwd' - the
# global sandbox is relative to it.
fs_url.path = "%s/radical.pilot.sandbox" % sandbox_base
# before returning, keep the URL string in cache
self._cache['resource_sandbox'][resource] = fs_url
return self._cache['resource_sandbox'][resource]
