def _get_jsurl(self, pilot): ''' get job service endpoint and hop URL for the pilot's target resource. ''' resrc = pilot['description']['resource'] schema = pilot['description']['access_schema'] rcfg = self.get_resource_config(resrc, schema) js_url = rs.Url(rcfg.get('job_manager_endpoint')) js_hop = rs.Url(rcfg.get('job_manager_hop', js_url)) # make sure the js_hop url points to an interactive access # TODO: this is an unreliable heuristics - we should require the js_hop # URL to be specified in the resource configs. if '+gsissh' in js_hop.schema or \ 'gsissh+' in js_hop.schema : js_hop.schema = 'gsissh' elif '+ssh' in js_hop.schema or \ 'ssh+' in js_hop.schema : js_hop.schema = 'ssh' else: js_hop.schema = 'fork' return js_url, js_hop
def upload (self, source, target, flags) : '''Uploads a file from the LOCAL, PHYSICAL filesystem to the replica management system. @param source: URL (should be file:// or local path) of local file @param target: Optional param containing ?resource=myresource query This will upload the file to a specified iRODS resource or group. ''' # TODO: Make sure that the source URL is a local/file:// URL complete_path = rs.Url(source).get_path() # extract the path from the LogicalFile object, excluding # the filename destination_path = self._url.get_path()[0:string.rfind( self._url.get_path(), "/") + 1] try: # note we're uploading self._logger.debug("Beginning upload operation " "will register file in logical dir: %s" % destination_path) # the query holds our target resource query = rs.Url(target).get_query() # list of args we will generate arg_list = "" # parse flags + generate args if flags: if flags & rs.namespace.OVERWRITE: arg_list += "-f " # was no resource selected? if not query: self._logger.debug("Attempting to upload to default resource") ret, out, _ = self.shell.run_sync("iput %s %s %s" % (arg_list, complete_path, destination_path)) # resource was selected, have to parse it and supply to iput -R else: # TODO: Verify correctness resource = query.split("=")[1] self._logger.debug("upload to %s" % resource) ret, out, _ = self.shell.run_sync("iput -R %s %s %s %s" % (resource, arg_list, complete_path, destination_path)) if ret: raise rs.NoSuccess("Could not upload file %s, errorcode %s: %s" % (complete_path, str(ret), out)) except Exception, ex: # couldn't upload for unspecificed reason raise rs.NoSuccess._log (self._logger, "upload failed: %s" % ex)
def test_jobid_viability(): """ Test if jobid represents job """ # The job id for the fork shell adaptor should return a pid which represents # the actual job instance. We test by killing that pid and checking state. try: import os cfg = config() js_url = rs.Url(cfg.job_service_url) if js_url.schema.lower() not in ['fork', 'local', 'ssh']: # test not supported for other backends return if js_url.host.lower() not in [None, '', 'localhost']: # test not supported for other backends return js = rs.job.Service('fork:///') j = js.run_job("/bin/sleep 100") jid = j.id js_part, j_part = jid.split('-', 1) pid = j_part[1:-3] # kill the children (i.e. the only child) of the pid, which is the # actual job os.system(('ps -ef | cut -c 8-21 | grep " %s " | cut -c 1-8 ' '| grep -v " %s " | xargs -r kill') % (pid, pid)) assert (j.state == rs.job.FAILED), 'job.state: %s' % j.state except rs.SagaException as se: assert False, "Unexpected exception: %s" % se
def remove(self, path, flags): '''This method is called upon logicaldir.remove() ''' complete_path = rs.Url(path).get_path() try: self._logger.debug("Executing: irm -r %s" % complete_path) ret, out, _ = self.shell.run_sync("irm -r %s" % complete_path) if ret != 0: raise rs.NoSuccess("Could not remove directory %s [%s]: %s" % (complete_path, str(ret), out)) except Exception as ex: # was there no directory to delete? if "does not exist" in str(ex): raise rs.DoesNotExist("Directory %s does not exist." % complete_path) # couldn't delete for unspecificed reason raise rs.NoSuccess("Couldn't delete directory %s because %s" % (complete_path, ex)) return
def get_js_shell(self, resource, schema): if resource not in self._cache['js_shells']: self._cache['js_shells'][resource] = dict() if schema not in self._cache['js_shells'][resource]: rcfg = self.get_resource_config(resource, schema) js_url = rcfg['job_manager_endpoint'] js_url = rcfg.get('job_manager_hop', js_url) js_url = rs.Url(js_url) elems = js_url.schema.split('+') if 'ssh' in elems: js_url.schema = 'ssh' elif 'gsissh' in elems: js_url.schema = 'gsissh' elif 'fork' in elems: js_url.schema = 'fork' elif len(elems) == 1 : js_url.schema = 'fork' else: raise Exception("invalid schema: %s" % js_url.schema) if js_url.schema == 'fork': js_url.hostname = 'localhost' self._log.debug("rsup.PTYShell('%s')", js_url) shell = rsup.PTYShell(js_url, self) self._cache['js_shells'][resource][schema] = shell return self._cache['js_shells'][resource][schema]
def benchmark_pre (tid, test_cfg, bench_cfg, session) : if not 'job_service_url' in test_cfg : raise saga.NoSuccess ('no job service URL configured') if not 'load' in bench_cfg : raise saga.NoSuccess ('no benchmark load configured') host = saga.Url(test_cfg['job_service_url']).host n_j = int(bench_cfg['iterations']) load = int(bench_cfg['load']) exe = '/bin/sleep' ssh = subprocess ("ssh %s" % host, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) # find the ssh prompt stdin = ssh.communicate[0] stdout = ssh.communicate[1] while <stdin> ~! /'>$'/io : time.sleep (0.1) # setup is done return {'ssh' : ssh, 'cmd' : "%s %s" % (executable, load)}
def replicate (self, target, flags): '''This method is called upon logicaldir.replicate() ''' # path to file we are replicating on iRODS complete_path = self._url.get_path() # TODO: Verify Correctness in the way the resource is grabbed query = rs.Url(target).get_query() resource = query.split("=")[1] self._logger.debug("Attempting to replicate logical file %s to " "resource/resource group %s" % (complete_path, resource)) try: self._logger.debug("Executing: irepl -R %s %s" % (resource, complete_path)) ret, out, _ = self.shell.run_sync("irepl -R %s %s" % (resource, complete_path)) if ret: raise Exception("Could not replicate logical file %s to" "resource/resource group %s, errorcode %s: %s" % (complete_path, resource, str(ret), out)) except Exception, ex: raise rs.NoSuccess._log(self._logger, "replicate failed: %s" % ex)
def test_job_service_invalid_url(self): """ Test if a non-resolvable hostname results in a proper exception """ tmp_js = None try: invalid_url = rs.Url(self.cfg['job_service_url']) invalid_url.schema = "ssh" invalid_url.host = "does.not.exist" tmp_js = rs.job.Service(invalid_url, self.session) assert False, "Expected BadParameter exception but got none." except rs.BadParameter: assert True # we don't check DNS anymore, as that can take *ages* -- so we now also # see Timeout and NoSuccess exceptions... except (rs.NoSuccess, rs.Timeout): assert True # other exceptions should never occur except rs.SagaException as ex: assert False, 'Expected BadParameter, Timeout or NoSuccess' \ 'exception, but got %s (%s)' % (type(ex), ex) finally: _silent_close_js(tmp_js)
def test_file_copy_1(self): """ Testing if we can copy an existing file. """ try: pass tc = config() filename1 = deepcopy(rs.Url(tc.filesystem_url)) filename1.path += "/%s" % self.uniquefilename1 f1 = rs.filesystem.File(filename1, rs.filesystem.CREATE) filename2 = deepcopy(rs.Url(tc.filesystem_url)) filename2.path += "/%s" % self.uniquefilename2 f1.copy(filename2) f2 = rs.filesystem.File(filename2) assert f2.size == 0 # this should fail if the file doesn't exist! except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def test_nonexisting_file_create_open(self): """ Testing if opening a non-existing file with the 'create' flag set works. """ try: pass tc = config() nonex_file = deepcopy(rs.Url(tc.filesystem_url)) nonex_file.path += "/%s" % self.uniquefilename1 f = rs.filesystem.File(nonex_file, rs.filesystem.CREATE) assert f.size == 0 # this should fail if the file doesn't exist! except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def test_file_copy_invalid_tgt_scheme(self): """ Testing if get an exception if we try to copy an unsupported target scheme. """ try: tc = config() # Create the source file source_file = deepcopy(rs.Url(tc.filesystem_url)) source_file.path += "/%s" % self.uniquefilename1 f1 = rs.filesystem.File(source_file, rs.filesystem.CREATE) target_url = deepcopy(rs.Url(tc.filesystem_url)) target_url.scheme = "crapscheme" f1.copy(target_url) assert False, "Expected BadParameter exception but got none." except rs.BadParameter: assert True except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def _get_pilot_sandbox(self, pilot): # FIXME: this should get 'pid, resource, schema=None' as parameters pilot_sandbox = pilot.get('pilot_sandbox') if str(pilot_sandbox): return rs.Url(pilot_sandbox) pid = pilot['uid'] with self._cache_lock: if pid in self._cache['pilot_sandbox']: return self._cache['pilot_sandbox'][pid] # cache miss session_sandbox = self._get_session_sandbox(pilot) pilot_sandbox = rs.Url(session_sandbox) pilot_sandbox.path += '/%s/' % pilot['uid'] with self._cache_lock: self._cache['pilot_sandbox'][pid] = pilot_sandbox return pilot_sandbox
def test_ptyshell_nok(): """ Test pty_shell which runs command unsuccessfully """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\" ; false" % txt) assert (ret == 1), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def test_nonexisting_host_file_open(self): """ Testing if opening a file on a non-existing host causes an exception. """ try: tc = config() invalid_url = deepcopy(rs.Url(tc.filesystem_url)) invalid_url.host = "does.not.exist" _ = rs.filesystem.File(invalid_url) assert False, "Expected BadParameter exception but got none." except rs.BadParameter: assert True except rs.SagaException as ex: assert False, "Expected BadParameter exception, but got %s" % ex
def test_existing_file_open(self): """ Testing if we can open an existing file. """ try: tc = config() filename = deepcopy(rs.Url(tc.filesystem_url)) filename.path += "/%s" % self.uniquefilename1 f = rs.filesystem.File(filename, rs.filesystem.CREATE) f2 = rs.filesystem.File(f.url) assert f2.size == 0 # this should fail if the file doesn't exist! except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def test_ptyshell_file_stage(): """ Test pty_shell file staging """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" shell.write_to_remote(txt, "/tmp/saga-test-staging") out = shell.read_from_remote("/tmp/saga-test-staging") assert (txt == out), "%s == %s" % (repr(out), repr(txt)) ret, out, _ = shell.run_sync("rm /tmp/saga-test-staging") assert (ret == 0), "%s" % (repr(ret)) assert (out == ""), "%s == ''" % (repr(out))
def test_nonexisting_file_open(self): """ Testing if opening a non-existing file causes an exception. """ try: pass tc = config() nonex_file = deepcopy(rs.Url(tc.filesystem_url)) nonex_file.path += "/file.does.not.exist" _ = rs.filesystem.File(nonex_file) assert False, "Expected DoesNotExist exception but got none." except rs.DoesNotExist: assert True except rs.SagaException as ex: assert False, "Expected DoesNotExist exception, but got %s" % ex
def download (self, name, source, flags) : '''Downloads a file from the REMOTE REPLICA FILESYSTEM to a local directory. @param target: param containing a local path/filename to save the file to @param source: Optional param containing a remote replica to retrieve the file from (not evaluated, yet) ''' target = name # TODO: Make sure that the target URL is a local/file:// URL # extract the path from the LogicalFile object, excluding # the filename logical_path = self._url.get_path() # fill in our local path if one was specified local_path = "" if target: local_path = rs.Url(target).get_path() try: # var to hold our command result, placed here to keep in scope ret = out = _ = 0 # mark that this is experimental/may not be part of official API self._logger.debug("Beginning download operation " "will download logical file: %s, " "specified local target is %s" % (logical_path, target)) if target: cmd = "iget %s %s" % (logical_path, local_path) else : cmd = "iget %s" % logical_path self._logger.debug("Executing: %s" % cmd) ret, out, _ = self.shell.run_sync(cmd) # check our result if ret: raise rs.NoSuccess("download failed: %s [%s]: %s" % (logical_path, str(ret), out)) except Exception, ex: # couldn't download for unspecificed reason raise rs.NoSuccess ("Couldn't download file. %s" % ex)
def test_file_copy_remote_local(self): """ Testing if we can copy a file from remote -> local """ try: pass tc = config() filename1 = deepcopy(rs.Url(tc.filesystem_url)) filename1.path = "/etc/passwd" f1 = rs.filesystem.File(filename1) filename2 = "file://localhost/tmp/%s" % self.uniquefilename2 f1.copy(filename2) f2 = rs.filesystem.File(filename2) assert f2.size != 0 # this should fail if the file doesn't exist! except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def test_file_get_session(self): """ Testing if the correct session is being used """ try: tc = config() session = tc.session or rs.Session() filename = deepcopy(rs.Url(tc.filesystem_url)) filename.path += "/%s" % self.uniquefilename1 f = rs.filesystem.File(filename, rs.filesystem.CREATE, session=session) assert f.get_session() == session, 'Failed setting the session' f2 = rs.filesystem.File(f.url, session=session) assert f2.get_session() == session, 'Failed setting the session' except rs.SagaException as ex: assert False, "Unexpected exception: %s" % ex
def test_ptyshell_async(): """ Test pty_shell which runs command successfully """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____\n" shell.run_async("cat <<EOT") shell.send(txt) shell.send('EOT\n') ret, out = shell.find_prompt() assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def make_dir (self, path, flags) : # complete_path = dir_obj._url.path complete_path = rs.Url(path).get_path() # attempt to run iRODS mkdir command try: ret, out, _ = self.shell.run_sync("imkdir %s" % complete_path) if ret != 0: raise rs.NoSuccess("Could not create dir %s, [%s]: %s" % (complete_path, str(ret), out)) except Exception, ex: # did the directory already exist? if "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME" in str(ex): raise rs.AlreadyExists ("Directory already exists.") # couldn't create for unspecificed reason raise rs.NoSuccess ("Couldn't create directory. %s" % ex)
def _prepare_job(self, job): """ Prepare job on remote host - Create remote working dir - Upload job input files """ job.logger.debug('Prepared job in ShellAdapter') try: work_dir = self.job_work_dir(job, saga.filesystem.CREATE_PARENTS) for input_file in job.input_files: wrapper = saga.filesystem.File( saga.Url('file://localhost/%s' % join(job.working_dir, input_file.value))) wrapper.copy(work_dir.get_url()) job.logger.debug("Uploaded file %s to %s", join(job.working_dir, input_file.value), work_dir.get_url()) return job except saga.SagaException as exc: raise AdaptorJobException(exc.message)
def move_self (self, target, flags) : '''This method is called upon logicaldir.move() ''' # path to file we are moving on iRODS source_path = self._url.get_path() dest_path = rs.Url(target).get_path() self._logger.debug("Attempting to move logical file %s to location %s" % (source_path, dest_path)) try: ret, out, _ = self.shell.run_sync("imv %s %s" % (source_path, dest_path)) if ret: raise rs.NoSuccess("Could not move logical file %s to location" "%s, errorcode %s: %s" % (source_path, dest_path, str(ret), out)) except Exception, ex: raise rs.NoSuccess ("Couldn't move file: %s" % ex)
def test_ptyshell_prompt(): """ Test pty_shell with prompt change """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\"" % txt) assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) shell.run_sync('export PS1="HALLO-(\\$?)-PROMPT>"', new_prompt='HALLO-\((\d)\)-PROMPT>') txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\"" % txt) assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def _get_session_sandbox(self, pilot): # FIXME: this should get 'resource, schema=None' as parameters resource = pilot['description'].get('resource') if not resource: raise ValueError('Cannot get session sandbox w/o resource target') with self._cache_lock: if resource not in self._cache['session_sandbox']: # cache miss resource_sandbox = self._get_resource_sandbox(pilot) session_sandbox = rs.Url(resource_sandbox) session_sandbox.path += '/%s' % self.uid self._cache['session_sandbox'][resource] = session_sandbox return self._cache['session_sandbox'][resource]
def main(): tmp_dir = None try: tmp_dir = tempfile.mkdtemp(prefix='saga-test-', suffix='-%s' % TEST_NAME, dir=os.path.expanduser('~/tmp')) print('tmpdir: %s' % tmp_dir) ctx = saga.Context("x509") ctx.user_proxy = '/Users/mark/proj/myproxy/xsede.x509' session = saga.Session() session.add_context(ctx) source_url = saga.Url() source_url.schema = 'go' source_url.host = SOURCE source_url.path = tmp_dir target_url = saga.Url() target_url.schema = 'go' target_url.host = TARGET target_url.path = os.path.join('~/saga-tests/', os.path.basename(tmp_dir)) print("Point to local Directory through GO ...") d = saga.filesystem.Directory(source_url) print("And check ...") assert d.is_dir() == True assert d.is_file() == False assert d.is_link() == False d.close() print("Point to remote Directory through GO ...") d = saga.filesystem.Directory(target_url, flags=saga.filesystem.CREATE_PARENTS) print("And check ...") assert d.is_dir() == True assert d.is_file() == False assert d.is_link() == False d.close() print("Point to local file through GO, before creation ...") caught = False try: saga.filesystem.File(os.path.join(str(source_url), FILE_A_level_0)) except saga.DoesNotExist: caught = True assert caught == True print("Create actual file ...") touch(tmp_dir, FILE_A_level_0) print("Try again ...") f = saga.filesystem.File(os.path.join(str(source_url), FILE_A_level_0)) assert f.is_file() == True assert f.is_dir() == False assert f.is_link() == False f.close() print("Copy local file to remote, using different filename ...") d = saga.filesystem.Directory(target_url, flags=saga.filesystem.CREATE_PARENTS) d.copy(os.path.join(str(source_url), FILE_A_level_0), FILE_A_level_0 + COPIED_SUFFIX) d.close() f = saga.filesystem.File( os.path.join(str(target_url), FILE_A_level_0 + COPIED_SUFFIX)) assert f.is_file() == True assert f.is_dir() == False assert f.is_link() == False f.close() print("Copy local file to remote, keeping filename in tact ...") d = saga.filesystem.Directory(target_url, flags=saga.filesystem.CREATE_PARENTS) d.copy(os.path.join(str(source_url), FILE_A_level_0), FILE_A_level_0) d.close() f = saga.filesystem.File(os.path.join(str(target_url), FILE_A_level_0)) assert f.is_file() == True assert f.is_dir() == False assert f.is_link() == False f.close() print('Create file in level 1 ...') tree = LEVEL_1 os.mkdir(os.path.join(tmp_dir, tree)) touch(os.path.join(tmp_dir, tree), FILE_A_level_1) print("Test local file ...") f = saga.filesystem.File( os.path.join(str(source_url), tree, FILE_A_level_1)) assert f.is_file() == True assert f.is_dir() == False assert f.is_link() == False f.close() print("Copy local file to remote, keeping filename in tact ...") d = saga.filesystem.Directory(os.path.join(str(target_url), tree), flags=saga.filesystem.CREATE_PARENTS) d.copy(os.path.join(str(source_url), tree, FILE_A_level_1), FILE_A_level_1) d.close() print("Test file after transfer ...") f = saga.filesystem.File( os.path.join(str(target_url), tree, FILE_A_level_1)) assert f.is_file() == True assert f.is_dir() == False assert f.is_link() == False f.close() print( "Copy non-existent local file to remote, keeping filename in tact ..." ) d = saga.filesystem.Directory(str(target_url), flags=saga.filesystem.CREATE_PARENTS) try: d.copy(os.path.join(str(source_url), NON_EXISTING_FILE), NON_EXISTING_FILE) except saga.DoesNotExist: caught = True assert caught == True print("Test file after (non-)transfer ...") caught = False try: saga.filesystem.File( os.path.join(str(target_url), NON_EXISTING_FILE)) except saga.DoesNotExist: caught = True assert caught == True # destination = "go://gridftp.stampede.tacc.xsede.org/~/tmp/" # #destination = "go://oasis-dm.sdsc.xsede.org/~/tmp/" # #destination = "go://ncsa#BlueWaters/~/tmp/" # #destination = "go://marksant#netbook/Users/mark/tmp/go/" # src_filename = "my_file" # dst_filename = "my_file_" # rt_filename = "my_file__" # # # open home directory on a remote machine # source_dir = saga.filesystem.Directory(source) # # # copy .bash_history to /tmp/ on the local machine # source_dir.copy(src_filename, os.path.join(destination, dst_filename)) # # # list 'm*' in local /tmp/ directory # dest_dir = saga.filesystem.Directory(destination) # for entry in dest_dir.list(pattern='%s*' % src_filename[0]): # print entry # # dest_file = saga.filesystem.File(os.path.join(destination, dst_filename)) # assert dest_file.is_file() == True # assert dest_file.is_link() == False # assert dest_file.is_dir() == False # print 'Size: %d' % dest_file.get_size() # # dest_file.copy(source) # # dest_file.copy(os.path.join(source+'broken', rt_filename)) print("Before return 0") return 0 except saga.SagaException as ex: # Catch all saga exceptions print("An exception occurred: (%s) %s " % (ex.type, (str(ex)))) # Trace back the exception. That can be helpful for debugging. print(" \n*** Backtrace:\n %s" % ex.traceback) print("before return -1") return -1 finally: print("and finally ...") if CLEANUP and tmp_dir: shutil.rmtree(tmp_dir)
# ------------------------------------------------------------------------------ # # set up the connection to EC2 # if not 'EC2_URL' in os.environ: usage("no %s in environment" % 'EC2_URL') if not 'EC2_ACCESS_KEY' in os.environ: usage("no %s in environment" % 'EC2_ACCESS_KEY') if not 'EC2_SECRET_KEY' in os.environ: usage("no %s in environment" % 'EC2_SECRET_KEY') if not 'EC2_KEYPAIR_ID' in os.environ: usage("no %s in environment" % 'EC2_KEYPAIR_ID') if not 'EC2_KEYPAIR' in os.environ: usage("no %s in environment" % 'EC2_KEYPAIR') server = rs.Url(os.environ['EC2_URL']) # in order to connect to EC2, we need an EC2 ID and KEY c1 = rs.Context('ec2') c1.user_id = os.environ['EC2_ACCESS_KEY'] c1.user_key = os.environ['EC2_SECRET_KEY'] c1.server = server # in order to access a created VM, we additionally need to point to the ssh # key which is used for EC2 VM contextualization, i.e. as EC2 'keypair'. # If the keypair is not yet registered on EC2, it will be registered by SAGA # -- but then a user_key *must* be specified (only the public key is ever # transfererd to EC2). c2 = rs.Context('ec2_keypair') c2.token = os.environ['EC2_KEYPAIR_ID'] c2.user_cert = os.environ['EC2_KEYPAIR']
def job_work_dir(self, job, mode=saga.filesystem.READ): """ Setup remote host working dir """ return saga.filesystem.Directory(saga.Url( '%s/%s' % (self.remote_dir, str(job.slug))), mode, session=self.session)
def _get_resource_sandbox(self, pilot): ''' for a given pilot dict, determine the global RP sandbox, based on the pilot's 'resource' attribute. ''' # FIXME: this should get 'resource, schema=None' as parameters resource = pilot['description'].get('resource') schema = pilot['description'].get('access_schema') if not resource: raise ValueError('Cannot get pilot sandbox w/o resource target') # the global sandbox will be the same for all pilots on any resource, so # we cache it with self._cache_lock: if resource not in self._cache['resource_sandbox']: # cache miss -- determine sandbox and fill cache rcfg = self.get_resource_config(resource, schema) fs_url = rs.Url(rcfg['filesystem_endpoint']) # Get the sandbox from either the pilot_desc or resource conf sandbox_raw = pilot['description'].get('sandbox') if not sandbox_raw: sandbox_raw = rcfg.get('default_remote_workdir', "$PWD") # we may need to replace pat elements with data from the pilot # description if '%' in sandbox_raw: # expand from pilot description expand = dict() for k,v in pilot['description'].items(): if v is None: v = '' expand['pd.%s' % k] = v if isinstance(v, str): expand['pd.%s' % k.upper()] = v.upper() expand['pd.%s' % k.lower()] = v.lower() else: expand['pd.%s' % k.upper()] = v expand['pd.%s' % k.lower()] = v sandbox_raw = sandbox_raw % expand if '_' in sandbox_raw and 'summit' in resource: sandbox_raw = sandbox_raw.split('_')[0] # If the sandbox contains expandables, we need to resolve those # remotely. # # NOTE: this will only work for (gsi)ssh or similar shell # based access mechanisms if '$' not in sandbox_raw: # no need to expand further sandbox_base = sandbox_raw else: shell = self.get_js_shell(resource, schema) ret, out, err = shell.run_sync(' echo "WORKDIR: %s"' % sandbox_raw) if ret or 'WORKDIR:' not in out: raise RuntimeError("Couldn't get remote workdir.") sandbox_base = out.split(":")[1].strip() self._log.debug("sandbox base %s", sandbox_base) # at this point we have determined the remote 'pwd' - the # global sandbox is relative to it. fs_url.path = "%s/radical.pilot.sandbox" % sandbox_base # before returning, keep the URL string in cache self._cache['resource_sandbox'][resource] = fs_url return self._cache['resource_sandbox'][resource]