def run_bfast(jobno, session, jobservice): bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast") try: run_id = str(int(time.time())) # create a working directory on the remote machine workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, run_id) basedir = saga.filesystem.Directory(bfast_base_dir, session=session) basedir.make_dir(workdir) jd = saga.job.Description() jd.wall_time_limit = "0:05:00" jd.total_cpu_count = 1 jd.environment = {'BFAST_DIR': bfast_base_dir.path} jd.working_directory = workdir jd.output = 'bfast.out' jd.error = 'bfast.err' jd.executable = '$BFAST_DIR/bin/bfast' jd.arguments = [ 'match', '-A 1', '-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq', '-f $BFAST_DIR/data/small/reference/hg_2122.fa' ] myjob = js.create_job(jd) myjob.run() start = time.time() print "\nJob #%s started with ID '%s' and working directory: '%s'"\ % (jobno, myjob.jobid, workdir) myjob.wait() diff = time.time() - start print "Job #%s with ID '%s' finished (RC: %s). Elapsed time: %.0fs"\ % (jobno, myjob.jobid, myjob.exitcode, diff) # copy output and error files back to the local machine local_file = saga.Url('sftp://localhost//tmp/bfast.out.' + run_id) basedir.copy(workdir + '/bfast.out', local_file) print "Job #%s output copied to local machine: %s (%s bytes)" \ % (jobno, local_file, basedir.get_size(workdir+'/bfast.out')) return diff except saga.Exception, ex: print "Oh, snap! An error occured: %s" % (str(ex))
def test_url_issue_49(self): url = saga.Url("scheme://*****:*****@host:123/dir/file?query#fragment") url.set_host('remote.host.net') url.set_scheme('sftp') url.set_path('/tmp/data') if str(url) != "sftp://*****:*****@remote.host.net:123/tmp/data": self.fail("unexpected url")
def __init__(self, resource_url, security_context=None, pilot_data_description=None): self.resource_url = saga.Url(resource_url) self.bucket_name = self.__get_bucket_name(resource_url) self.__state = State.New self.pilot_data_description = pilot_data_description aws_access_key_id = None aws_secret_access_key = None # try to recover key from pilot_data_description if self.pilot_data_description!=None and\ self.pilot_data_description.has_key("access_key_id") and \ self.pilot_data_description.has_key("secret_access_key"): aws_access_key_id = self.pilot_data_description["access_key_id"] aws_secret_access_key = self.pilot_data_description[ "secret_access_key"] # try to recover key from security context if security_context != None: logger.debug( "Attempt to restore credentials from security context: " + str(security_context)) security_context = eval(security_context) aws_access_key_id = security_context["aws_access_key_id"] aws_secret_access_key = security_context["aws_secret_access_key"] self.s3_conn = None if self.resource_url.scheme == "walrus" or self.resource_url.scheme == "swift": calling_format = OrdinaryCallingFormat() logger.debug("Access Key: %s Secret: %s Host: %s" % (aws_access_key_id, aws_secret_access_key, self.resource_url.host)) port = 8773 if self.resource_url.port != None: port = self.resource_url.port path = "/" if self.resource_url.scheme == "walrus": path = "/services/Walrus" self.s3_conn = S3Connection( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, is_secure=False, host=self.resource_url.host, port=port, calling_format=calling_format, path=path) else: # s3:// urls self.s3_conn = S3Connection(aws_access_key_id, aws_secret_access_key)
def __get_bucket_name(self, resource_url): surl = saga.Url(resource_url) if surl.scheme.startswith("s3"): bucket_name = resource_url.replace("s3://", "") try: bucket_name = bucket_name[:bucket_name.index("/")] except: pass #bucket_name = bucket_name.replace("/", "") else: bucket_name = surl.path[1:] return bucket_name
def __get_key_name(self, resource_url): surl = saga.Url(resource_url) # get path out of URL if surl.scheme.startswith("s3"): bucket_name = resource_url.replace("s3://", "") else: bucket_name = surl.path[1:] # get key path out of URL try: key_name = bucket_name[bucket_name.index("/") + 1:] except: pass return key_name
def run_bfast_p(numjobs, session, jobservice): bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast") try: container = saga.job.Container(jobservice) for jobno in range(0, numjobs): workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, str(int(time.time()))) basedir = saga.filesystem.Directory(bfast_base_dir, session=session) basedir.make_dir(workdir) jd = saga.job.Description() jd.wall_time_limit = "0:05:00" jd.total_cpu_count = 1 jd.environment = {'BFAST_DIR': bfast_base_dir.path} jd.working_directory = workdir jd.executable = '$BFAST_DIR/bin/bfast' jd.arguments = [ 'match', '-A 1', '-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq', '-f $BFAST_DIR/data/small/reference/hg_2122.fa' ] container.add(js.create_job(jd)) print "Added job #%s with working directory '%s' to container"\ % (jobno, workdir) cstart = time.time() container.run() cdiff = time.time() - cstart start = time.time() print "Container with %s jobs started. It took %.0fs." % (numjobs, cdiff) container.wait() diff = time.time() - start print "Container finished running. Elapsed time: %.0fs."\ % (diff) return diff except saga.Exception, ex: print "Oh, snap! An error occured: %s" % (str(ex))
def main(): try: c1 = saga.Context() c1.type = saga.Context.SSH c1.usercert = "/Users/s1063117/id_rsa.pub" c1.userkey = "/Users/s1063117/id_rsa" js = saga.job.Service("fork://localhost") js.get_session().add_context(c1) js.get_session().contexts.append(c1) jd = saga.job.Description() jd.set_attribute('Executable', '/bin/sleep') jd.set_vector_attribute('Arguments', ["10"]) xx = saga.Url("fork://localhost:8080") jd.arguments = ['4'] for att in jd.list_attributes(): print "%s %s %s %s" % (att, jd.attribute_is_vector(att), jd.attribute_is_readonly(att), jd.attribute_is_writeable(att)) for job in range(10): job = js.create_job(jd) job.run() print job.get_job_id() + " : " + job.get_state() time.sleep(2) for job in js.list(): rjob = js.get_job(job.get_job_id()) print rjob.get_job_id() + " : " + rjob.get_state() for job in js.list(): job.cancel() print job.get_job_id() + " : " + job.get_state() for job in js.list(): job.run() print job.get_job_id() + " : " + job.get_state() for job in js.list(): job.wait() print job.get_job_id() + " : " + job.get_state() except saga.Exception, ex: print str(ex)
def __init__(self, job_description, resource_url, pilot_compute_description): self.job_description = job_description logger.debug("URL: " + str(resource_url) + " Type: " + str(type(resource_url))) self.resource_url = saga.Url(str(resource_url)) self.pilot_compute_description = pilot_compute_description self.id = "bigjob-" + str(uuid.uuid1()) self.network_ip = None self.ec2_conn = None if self.resource_url.scheme == "euca+ssh" or self.resource_url.scheme == "nova+ssh": host = self.resource_url.host path = "/services/Eucalyptus" if self.resource_url.path != None: path = self.resource_url.path port = 8773 if self.resource_url.port != None: port = self.resource_url.port region = None logger.debug("Host: %s, Path: %s, Port: %d" % (host, path, port)) if self.resource_url.scheme == "euca+ssh": region = RegionInfo(name="eucalyptus", endpoint=host) elif self.resource_url.scheme == "nova+ssh": region = RegionInfo(name="openstack", endpoint=host) logger.debug("Access Key: %s Secret: %s" % (self.pilot_compute_description["access_key_id"], self.pilot_compute_description["secret_access_key"])) self.ec2_conn = EC2Connection( aws_access_key_id=self. pilot_compute_description["access_key_id"], aws_secret_access_key=self. pilot_compute_description["secret_access_key"], region=region, is_secure=False, port=port, path=path) else: self.ec2_conn = EC2Connection( aws_access_key_id=self. pilot_compute_description["access_key_id"], aws_secret_access_key=self. pilot_compute_description["secret_access_key"]) self.instance = None
def test_url_scheme_issue(self): u1 = saga.Url("unknownscheme://*****:*****@hostname.domain:9999/path") if u1.scheme != "unknownscheme": self.fail("unexpected value for scheme") if u1.username != "user": self.fail("unexpected value for username") if u1.password != "pwd": self.fail("unexpected value for password") if u1.host != "hostname.domain": self.fail("unexpected value for host") if u1.port != int(9999): self.fail("unexpected value for port")
def test_url_compatibility(self): u1 = saga.Url("ssh://*****:*****@hostname.domain:9999/path") if u1.scheme != "ssh": self.fail("unexpected value for scheme") if u1.username != "user": self.fail("unexpected value for username") if u1.password != "pwd": self.fail("unexpected value for password") if u1.host != "hostname.domain": self.fail("unexpected value for host") if u1.port != int(9999): self.fail("unexpected value for port")
def main(jobno, session, jobservice): bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast") try: workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, str(int( time.time()))) basedir = saga.filesystem.Directory(bfast_base_dir, session=session) basedir.make_dir(workdir) jd = saga.job.Description() jd.wall_time_limit = "0:05:00" jd.total_cpu_count = 1 jd.environment = {'BFAST_DIR': bfast_base_dir.path} jd.working_directory = workdir jd.executable = '$BFAST_DIR/bin/bfast' jd.arguments = [ 'match', '-A 1', '-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq', '-f $BFAST_DIR/data/small/reference/hg_2122.fa' ] myjob = js.create_job(jd) myjob.run() time.sleep(1) print "Job #%s started with ID '%s' and working directory: '%s'"\ % (jobno, myjob.jobid, workdir) myjob.wait() print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\ % (jobno, myjob.jobid, myjob.exitcode, workdir) # copy result to local machine: basedir.copy(...) #basedir.close() except saga.Exception, ex: print "Oh, snap! An error occured: %s" % (str(ex))
def main(jobno, session, jobservice): bfast_base_dir = saga.Url( "sftp://india.futuregrid.org/N/u/oweidner/software/bfast/") try: workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, str(int( time.time()))) basedir = saga.filesystem.Directory(bfast_base_dir, session=session) basedir.make_dir(workdir) jd = saga.job.Description() jd.wall_time_limit = 5 # wall-time in minutes jd.total_cpu_count = 1 jd.environment = {'BFAST_DIR': bfast_base_dir.path} jd.working_directory = workdir jd.executable = '$BFAST_DIR/bin/bfast' jd.arguments = [ 'match', '-A 1', '-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq', '-f $BFAST_DIR/data/small/reference/hg_2122.fa' ] myjob = js.create_job(jd) myjob.run() print "Job #%s started with ID '%s' and working directory: '%s'"\ % (jobno, myjob.jobid, workdir) myjob.wait() print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\ % (jobno, myjob.jobid, myjob.exitcode, workdir) basedir.close() except saga.Exception, ex: print "An error occured: %s" % (str(ex)) sys.exit(-1)
def main(): try: u = saga.Url('fork://localhost') #s = saga.job.Service(u) z = saga.job.Service(u) a = saga.job.Service(u) b = saga.job.Service("dummy://localhost") c = saga.job.Service("dummy://localhost") z = saga.job.Service(u) jd = saga.job.Description() jd.executable = "/bin/sleep" jd.arguments = ["10"] j1 = z.create_job(jd) j2 = z.create_job(jd) j3 = z.create_job(jd) j4 = z.create_job(jd) j1 = a.create_job(jd) j2 = a.create_job(jd) j3 = a.create_job(jd) j4 = a.create_job(jd) b.create_job(jd) k = c.create_job(jd) try: k.get_stdout() except saga.Exception, ex: assert (ex.error == saga.Error.NotImplemented) try: k.get_stderr() except saga.Exception, ex: assert (ex.error == saga.Error.NotImplemented)
basedir.copy(workdir+'/bfast.out', local_file) print "Job #%s output copied to local machine: %s (%s bytes)" \ % (jobno, local_file, basedir.get_size(workdir+'/bfast.out')) return diff except saga.Exception, ex: print "An error occured: %s" % (str(ex)) sys.exit(-1) if __name__ == "__main__": NUMJOBS = 32 execution_host = saga.Url("pbs+ssh://queenbee.loni.org") ctx = saga.Context() ctx.type = saga.Context.SSH ctx.userid = 'oweidner' # like 'ssh username@host ...' ctx.userkey = '/Users/s1063117/.ssh/id_rsa' # like ssh -i ...' session = saga.Session() session.contexts.append(ctx) js = saga.job.Service(execution_host, session) print "\n-------------------------------------" print "Submitting %s jobs sequentially" % NUMJOBS total_time = 0.0
print "Job #%s started with ID '%s' and working directory: '%s'"\ % (jobno, myjob.jobid, workdir) myjob.wait() print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\ % (jobno, myjob.jobid, myjob.exitcode, workdir) basedir.close() except saga.Exception, ex: print "An error occured: %s" % (str(ex)) sys.exit(-1) if __name__ == "__main__": execution_host = saga.Url("pbs+ssh://india.futuregrid.org") ctx = saga.Context() ctx.type = saga.Context.SSH ctx.userid = 's1063117' # like 'ssh username@host ...' ctx.userkey = '/Users/s1063117/.ssh/rsa_work' # like ssh -i ...' session = saga.Session() session.contexts.append(ctx) js = saga.job.Service(execution_host, session) for i in range(0, 4): main(i, session, js)
def test_url_properties(self): url = saga.Url("") if str(url) != "": self.fail("unexpected url") url.scheme = "scheme" if str(url) != "scheme://": self.fail("unexpected url") if url.get_scheme() != "scheme": self.fail("unexpected scheme") url.set_scheme("tscheme") if url.get_scheme() != "tscheme": self.fail("unexpected scheme") url.scheme = "scheme" url.host = "host" if str(url) != "scheme://host": self.fail("unexpected url") if url.get_host() != "host": self.fail("unexpected host") url.set_host("thost") if url.get_host() != "thost": self.fail("unexpected host") url.host = "host" url.port = 42 if str(url) != "scheme://*****:*****@host:42": self.fail("unexpected url") if url.get_username() != "username": self.fail("unexpected username") url.set_username("tusername") if url.get_username() != "tusername": self.fail("unexpected username") url.username = "******" url.password = "******" if str(url) != "scheme://*****:*****@host:42": self.fail("unexpected url") if url.get_password() != "password": self.fail("unexpected password") url.set_password("tpassword") if url.get_password() != "tpassword": self.fail("unexpected passsword") url.password = "******" url.path = "/path/" if str(url) != "scheme://*****:*****@host:42/path/": self.fail("unexpected url") if url.get_path() != "/path/": self.fail("unexpected path") url.set_path("tpath") if url.get_path() != "/tpath": self.fail("unexpected path") url.path = "/path/"
def test_url_issue_61(self): url = saga.Url("advert://localhost/?dbtype=sqlite3") if url.query != "dbtype=sqlite3": self.fail("unexpected url")
def run(url, username, queue, project): """Tests if a plug-in can handle both, username as part of the URL (i.e., oweidner@host) and username as part of a context. """ try: ctx = saga.Context() ctx.type = saga.Context.SSH ctx.userid = username # like 'ssh username@host ...' js = saga.job.Service(url) js.session.contexts.append(ctx) # describe our job jd = saga.job.Description() jd.queue = queue jd.project = project jd.wall_time_limit = 5 # minutes # environment, executable & arguments jd.environment = {'MYOUTPUT': '"Hello from Bliss"'} jd.executable = '/bin/echo' jd.arguments = ['$MYOUTPUT'] # output options jd.output = "bliss_job.01b.stdout" jd.error = "bliss_job.01b.stderr" # create the job (state: New) myjob = js.create_job(jd) print "Job ID : %s" % (myjob.jobid) print "Job State : %s" % (myjob.get_state()) print "\n...starting job...\n" # run the job (submit the job to PBS) myjob.run() print "Job ID : %s" % (myjob.jobid) print "Job State : %s" % (myjob.get_state()) print "\n...waiting for job...\n" # wait for the job to either finish or fail myjob.wait() print "Job State : %s" % (myjob.get_state()) print "Exitcode : %s" % (myjob.exitcode) failed = False why = "" if myjob.get_state() != saga.job.Job.Done: failed = True why = "Job 1 returned in state 'Failed'." ####################################################### url2 = saga.Url(url) url2.username = username js2 = saga.job.Service(url2) myjob2 = js.create_job(jd) print "Job ID : %s" % (myjob2.jobid) print "Job State : %s" % (myjob2.get_state()) print "\n...starting job...\n" # run the job (submit the job to PBS) myjob2.run() print "Job ID : %s" % (myjob2.jobid) print "Job State : %s" % (myjob2.get_state()) print "\n...waiting for job...\n" # wait for the job to either finish or fail myjob2.wait() print "Job State : %s" % (myjob2.get_state()) print "Exitcode : %s" % (myjob2.exitcode) failed = False why = "" if myjob2.get_state() != saga.job.Job.Done: failed = True why = "Job 2 returned in state 'Failed'." except saga.Exception, ex: failed = True why = str(ex)