def run_bfast(jobno, session, jobservice):
bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast")
try:
run_id = str(int(time.time()))
# create a working directory on the remote machine
workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, run_id)
basedir = saga.filesystem.Directory(bfast_base_dir, session=session)
basedir.make_dir(workdir)
jd = saga.job.Description()
jd.wall_time_limit = "0:05:00"
jd.total_cpu_count = 1
jd.environment = {'BFAST_DIR': bfast_base_dir.path}
jd.working_directory = workdir
jd.output = 'bfast.out'
jd.error = 'bfast.err'
jd.executable = '$BFAST_DIR/bin/bfast'
jd.arguments = [
'match', '-A 1',
'-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq',
'-f $BFAST_DIR/data/small/reference/hg_2122.fa'
]
myjob = js.create_job(jd)
myjob.run()
start = time.time()
print "\nJob #%s started with ID '%s' and working directory: '%s'"\
% (jobno, myjob.jobid, workdir)
myjob.wait()
diff = time.time() - start
print "Job #%s with ID '%s' finished (RC: %s). Elapsed time: %.0fs"\
% (jobno, myjob.jobid, myjob.exitcode, diff)
# copy output and error files back to the local machine
local_file = saga.Url('sftp://localhost//tmp/bfast.out.' + run_id)
basedir.copy(workdir + '/bfast.out', local_file)
print "Job #%s output copied to local machine: %s (%s bytes)" \
% (jobno, local_file, basedir.get_size(workdir+'/bfast.out'))
return diff
except saga.Exception, ex:
print "Oh, snap! An error occured: %s" % (str(ex))
def test_url_issue_49(self):
url = saga.Url("scheme://*****:*****@host:123/dir/file?query#fragment")
url.set_host('remote.host.net')
url.set_scheme('sftp')
url.set_path('/tmp/data')
if str(url) != "sftp://*****:*****@remote.host.net:123/tmp/data":
self.fail("unexpected url")
def __init__(self,
resource_url,
security_context=None,
pilot_data_description=None):
self.resource_url = saga.Url(resource_url)
self.bucket_name = self.__get_bucket_name(resource_url)
self.__state = State.New
self.pilot_data_description = pilot_data_description
aws_access_key_id = None
aws_secret_access_key = None
# try to recover key from pilot_data_description
if self.pilot_data_description!=None and\
self.pilot_data_description.has_key("access_key_id") and \
self.pilot_data_description.has_key("secret_access_key"):
aws_access_key_id = self.pilot_data_description["access_key_id"]
aws_secret_access_key = self.pilot_data_description[
"secret_access_key"]
# try to recover key from security context
if security_context != None:
logger.debug(
"Attempt to restore credentials from security context: " +
str(security_context))
security_context = eval(security_context)
aws_access_key_id = security_context["aws_access_key_id"]
aws_secret_access_key = security_context["aws_secret_access_key"]
self.s3_conn = None
if self.resource_url.scheme == "walrus" or self.resource_url.scheme == "swift":
calling_format = OrdinaryCallingFormat()
logger.debug("Access Key: %s Secret: %s Host: %s" %
(aws_access_key_id, aws_secret_access_key,
self.resource_url.host))
port = 8773
if self.resource_url.port != None:
port = self.resource_url.port
path = "/"
if self.resource_url.scheme == "walrus":
path = "/services/Walrus"
self.s3_conn = S3Connection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
is_secure=False,
host=self.resource_url.host,
port=port,
calling_format=calling_format,
path=path)
else: # s3:// urls
self.s3_conn = S3Connection(aws_access_key_id,
aws_secret_access_key)
def __get_bucket_name(self, resource_url):
surl = saga.Url(resource_url)
if surl.scheme.startswith("s3"):
bucket_name = resource_url.replace("s3://", "")
try:
bucket_name = bucket_name[:bucket_name.index("/")]
except:
pass
#bucket_name = bucket_name.replace("/", "")
else:
bucket_name = surl.path[1:]
return bucket_name
def __get_key_name(self, resource_url):
surl = saga.Url(resource_url)
# get path out of URL
if surl.scheme.startswith("s3"):
bucket_name = resource_url.replace("s3://", "")
else:
bucket_name = surl.path[1:]
# get key path out of URL
try:
key_name = bucket_name[bucket_name.index("/") + 1:]
except:
pass
return key_name
def run_bfast_p(numjobs, session, jobservice):
bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast")
try:
container = saga.job.Container(jobservice)
for jobno in range(0, numjobs):
workdir = "%s/tmp/run/%s" % (bfast_base_dir.path,
str(int(time.time())))
basedir = saga.filesystem.Directory(bfast_base_dir,
session=session)
basedir.make_dir(workdir)
jd = saga.job.Description()
jd.wall_time_limit = "0:05:00"
jd.total_cpu_count = 1
jd.environment = {'BFAST_DIR': bfast_base_dir.path}
jd.working_directory = workdir
jd.executable = '$BFAST_DIR/bin/bfast'
jd.arguments = [
'match', '-A 1',
'-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq',
'-f $BFAST_DIR/data/small/reference/hg_2122.fa'
]
container.add(js.create_job(jd))
print "Added job #%s with working directory '%s' to container"\
% (jobno, workdir)
cstart = time.time()
container.run()
cdiff = time.time() - cstart
start = time.time()
print "Container with %s jobs started. It took %.0fs." % (numjobs,
cdiff)
container.wait()
diff = time.time() - start
print "Container finished running. Elapsed time: %.0fs."\
% (diff)
return diff
except saga.Exception, ex:
print "Oh, snap! An error occured: %s" % (str(ex))
def main():
try:
c1 = saga.Context()
c1.type = saga.Context.SSH
c1.usercert = "/Users/s1063117/id_rsa.pub"
c1.userkey = "/Users/s1063117/id_rsa"
js = saga.job.Service("fork://localhost")
js.get_session().add_context(c1)
js.get_session().contexts.append(c1)
jd = saga.job.Description()
jd.set_attribute('Executable', '/bin/sleep')
jd.set_vector_attribute('Arguments', ["10"])
xx = saga.Url("fork://localhost:8080")
jd.arguments = ['4']
for att in jd.list_attributes():
print "%s %s %s %s" % (att, jd.attribute_is_vector(att),
jd.attribute_is_readonly(att),
jd.attribute_is_writeable(att))
for job in range(10):
job = js.create_job(jd)
job.run()
print job.get_job_id() + " : " + job.get_state()
time.sleep(2)
for job in js.list():
rjob = js.get_job(job.get_job_id())
print rjob.get_job_id() + " : " + rjob.get_state()
for job in js.list():
job.cancel()
print job.get_job_id() + " : " + job.get_state()
for job in js.list():
job.run()
print job.get_job_id() + " : " + job.get_state()
for job in js.list():
job.wait()
print job.get_job_id() + " : " + job.get_state()
except saga.Exception, ex:
print str(ex)
def __init__(self, job_description, resource_url,
pilot_compute_description):
self.job_description = job_description
logger.debug("URL: " + str(resource_url) + " Type: " +
str(type(resource_url)))
self.resource_url = saga.Url(str(resource_url))
self.pilot_compute_description = pilot_compute_description
self.id = "bigjob-" + str(uuid.uuid1())
self.network_ip = None
self.ec2_conn = None
if self.resource_url.scheme == "euca+ssh" or self.resource_url.scheme == "nova+ssh":
host = self.resource_url.host
path = "/services/Eucalyptus"
if self.resource_url.path != None:
path = self.resource_url.path
port = 8773
if self.resource_url.port != None:
port = self.resource_url.port
region = None
logger.debug("Host: %s, Path: %s, Port: %d" % (host, path, port))
if self.resource_url.scheme == "euca+ssh":
region = RegionInfo(name="eucalyptus", endpoint=host)
elif self.resource_url.scheme == "nova+ssh":
region = RegionInfo(name="openstack", endpoint=host)
logger.debug("Access Key: %s Secret: %s" %
(self.pilot_compute_description["access_key_id"],
self.pilot_compute_description["secret_access_key"]))
self.ec2_conn = EC2Connection(
aws_access_key_id=self.
pilot_compute_description["access_key_id"],
aws_secret_access_key=self.
pilot_compute_description["secret_access_key"],
region=region,
is_secure=False,
port=port,
path=path)
else:
self.ec2_conn = EC2Connection(
aws_access_key_id=self.
pilot_compute_description["access_key_id"],
aws_secret_access_key=self.
pilot_compute_description["secret_access_key"])
self.instance = None
def test_url_scheme_issue(self):
u1 = saga.Url("unknownscheme://*****:*****@hostname.domain:9999/path")
if u1.scheme != "unknownscheme":
self.fail("unexpected value for scheme")
if u1.username != "user":
self.fail("unexpected value for username")
if u1.password != "pwd":
self.fail("unexpected value for password")
if u1.host != "hostname.domain":
self.fail("unexpected value for host")
if u1.port != int(9999):
self.fail("unexpected value for port")
def test_url_compatibility(self):
u1 = saga.Url("ssh://*****:*****@hostname.domain:9999/path")
if u1.scheme != "ssh":
self.fail("unexpected value for scheme")
if u1.username != "user":
self.fail("unexpected value for username")
if u1.password != "pwd":
self.fail("unexpected value for password")
if u1.host != "hostname.domain":
self.fail("unexpected value for host")
if u1.port != int(9999):
self.fail("unexpected value for port")
def main(jobno, session, jobservice):
bfast_base_dir = saga.Url("sftp://queenbee.loni.org/work/oweidner/bfast")
try:
workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, str(int(
time.time())))
basedir = saga.filesystem.Directory(bfast_base_dir, session=session)
basedir.make_dir(workdir)
jd = saga.job.Description()
jd.wall_time_limit = "0:05:00"
jd.total_cpu_count = 1
jd.environment = {'BFAST_DIR': bfast_base_dir.path}
jd.working_directory = workdir
jd.executable = '$BFAST_DIR/bin/bfast'
jd.arguments = [
'match', '-A 1',
'-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq',
'-f $BFAST_DIR/data/small/reference/hg_2122.fa'
]
myjob = js.create_job(jd)
myjob.run()
time.sleep(1)
print "Job #%s started with ID '%s' and working directory: '%s'"\
% (jobno, myjob.jobid, workdir)
myjob.wait()
print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\
% (jobno, myjob.jobid, myjob.exitcode, workdir)
# copy result to local machine: basedir.copy(...)
#basedir.close()
except saga.Exception, ex:
print "Oh, snap! An error occured: %s" % (str(ex))
def main(jobno, session, jobservice):
bfast_base_dir = saga.Url(
"sftp://india.futuregrid.org/N/u/oweidner/software/bfast/")
try:
workdir = "%s/tmp/run/%s" % (bfast_base_dir.path, str(int(
time.time())))
basedir = saga.filesystem.Directory(bfast_base_dir, session=session)
basedir.make_dir(workdir)
jd = saga.job.Description()
jd.wall_time_limit = 5 # wall-time in minutes
jd.total_cpu_count = 1
jd.environment = {'BFAST_DIR': bfast_base_dir.path}
jd.working_directory = workdir
jd.executable = '$BFAST_DIR/bin/bfast'
jd.arguments = [
'match', '-A 1',
'-r $BFAST_DIR/data/small/reads_5K/reads.10.fastq',
'-f $BFAST_DIR/data/small/reference/hg_2122.fa'
]
myjob = js.create_job(jd)
myjob.run()
print "Job #%s started with ID '%s' and working directory: '%s'"\
% (jobno, myjob.jobid, workdir)
myjob.wait()
print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\
% (jobno, myjob.jobid, myjob.exitcode, workdir)
basedir.close()
except saga.Exception, ex:
print "An error occured: %s" % (str(ex))
sys.exit(-1)
def main():
try:
u = saga.Url('fork://localhost')
#s = saga.job.Service(u)
z = saga.job.Service(u)
a = saga.job.Service(u)
b = saga.job.Service("dummy://localhost")
c = saga.job.Service("dummy://localhost")
z = saga.job.Service(u)
jd = saga.job.Description()
jd.executable = "/bin/sleep"
jd.arguments = ["10"]
j1 = z.create_job(jd)
j2 = z.create_job(jd)
j3 = z.create_job(jd)
j4 = z.create_job(jd)
j1 = a.create_job(jd)
j2 = a.create_job(jd)
j3 = a.create_job(jd)
j4 = a.create_job(jd)
b.create_job(jd)
k = c.create_job(jd)
try:
k.get_stdout()
except saga.Exception, ex:
assert (ex.error == saga.Error.NotImplemented)
try:
k.get_stderr()
except saga.Exception, ex:
assert (ex.error == saga.Error.NotImplemented)
basedir.copy(workdir+'/bfast.out', local_file)
print "Job #%s output copied to local machine: %s (%s bytes)" \
% (jobno, local_file, basedir.get_size(workdir+'/bfast.out'))
return diff
except saga.Exception, ex:
print "An error occured: %s" % (str(ex))
sys.exit(-1)
if __name__ == "__main__":
NUMJOBS = 32
execution_host = saga.Url("pbs+ssh://queenbee.loni.org")
ctx = saga.Context()
ctx.type = saga.Context.SSH
ctx.userid = 'oweidner' # like 'ssh username@host ...'
ctx.userkey = '/Users/s1063117/.ssh/id_rsa' # like ssh -i ...'
session = saga.Session()
session.contexts.append(ctx)
js = saga.job.Service(execution_host, session)
print "\n-------------------------------------"
print "Submitting %s jobs sequentially" % NUMJOBS
total_time = 0.0
print "Job #%s started with ID '%s' and working directory: '%s'"\
% (jobno, myjob.jobid, workdir)
myjob.wait()
print "Job #%s with ID '%s' finished (RC: %s). Output available in: '%s'"\
% (jobno, myjob.jobid, myjob.exitcode, workdir)
basedir.close()
except saga.Exception, ex:
print "An error occured: %s" % (str(ex))
sys.exit(-1)
if __name__ == "__main__":
execution_host = saga.Url("pbs+ssh://india.futuregrid.org")
ctx = saga.Context()
ctx.type = saga.Context.SSH
ctx.userid = 's1063117' # like 'ssh username@host ...'
ctx.userkey = '/Users/s1063117/.ssh/rsa_work' # like ssh -i ...'
session = saga.Session()
session.contexts.append(ctx)
js = saga.job.Service(execution_host, session)
for i in range(0, 4):
main(i, session, js)
def test_url_properties(self):
url = saga.Url("")
if str(url) != "":
self.fail("unexpected url")
url.scheme = "scheme"
if str(url) != "scheme://":
self.fail("unexpected url")
if url.get_scheme() != "scheme":
self.fail("unexpected scheme")
url.set_scheme("tscheme")
if url.get_scheme() != "tscheme":
self.fail("unexpected scheme")
url.scheme = "scheme"
url.host = "host"
if str(url) != "scheme://host":
self.fail("unexpected url")
if url.get_host() != "host":
self.fail("unexpected host")
url.set_host("thost")
if url.get_host() != "thost":
self.fail("unexpected host")
url.host = "host"
url.port = 42
if str(url) != "scheme://*****:*****@host:42":
self.fail("unexpected url")
if url.get_username() != "username":
self.fail("unexpected username")
url.set_username("tusername")
if url.get_username() != "tusername":
self.fail("unexpected username")
url.username = "******"
url.password = "******"
if str(url) != "scheme://*****:*****@host:42":
self.fail("unexpected url")
if url.get_password() != "password":
self.fail("unexpected password")
url.set_password("tpassword")
if url.get_password() != "tpassword":
self.fail("unexpected passsword")
url.password = "******"
url.path = "/path/"
if str(url) != "scheme://*****:*****@host:42/path/":
self.fail("unexpected url")
if url.get_path() != "/path/":
self.fail("unexpected path")
url.set_path("tpath")
if url.get_path() != "/tpath":
self.fail("unexpected path")
url.path = "/path/"
def test_url_issue_61(self):
url = saga.Url("advert://localhost/?dbtype=sqlite3")
if url.query != "dbtype=sqlite3":
self.fail("unexpected url")
def run(url, username, queue, project):
"""Tests if a plug-in can handle both, username
as part of the URL (i.e., oweidner@host) and
username as part of a context.
"""
try:
ctx = saga.Context()
ctx.type = saga.Context.SSH
ctx.userid = username # like 'ssh username@host ...'
js = saga.job.Service(url)
js.session.contexts.append(ctx)
# describe our job
jd = saga.job.Description()
jd.queue = queue
jd.project = project
jd.wall_time_limit = 5 # minutes
# environment, executable & arguments
jd.environment = {'MYOUTPUT': '"Hello from Bliss"'}
jd.executable = '/bin/echo'
jd.arguments = ['$MYOUTPUT']
# output options
jd.output = "bliss_job.01b.stdout"
jd.error = "bliss_job.01b.stderr"
# create the job (state: New)
myjob = js.create_job(jd)
print "Job ID : %s" % (myjob.jobid)
print "Job State : %s" % (myjob.get_state())
print "\n...starting job...\n"
# run the job (submit the job to PBS)
myjob.run()
print "Job ID : %s" % (myjob.jobid)
print "Job State : %s" % (myjob.get_state())
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob.wait()
print "Job State : %s" % (myjob.get_state())
print "Exitcode : %s" % (myjob.exitcode)
failed = False
why = ""
if myjob.get_state() != saga.job.Job.Done:
failed = True
why = "Job 1 returned in state 'Failed'."
#######################################################
url2 = saga.Url(url)
url2.username = username
js2 = saga.job.Service(url2)
myjob2 = js.create_job(jd)
print "Job ID : %s" % (myjob2.jobid)
print "Job State : %s" % (myjob2.get_state())
print "\n...starting job...\n"
# run the job (submit the job to PBS)
myjob2.run()
print "Job ID : %s" % (myjob2.jobid)
print "Job State : %s" % (myjob2.get_state())
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob2.wait()
print "Job State : %s" % (myjob2.get_state())
print "Exitcode : %s" % (myjob2.exitcode)
failed = False
why = ""
if myjob2.get_state() != saga.job.Job.Done:
failed = True
why = "Job 2 returned in state 'Failed'."
except saga.Exception, ex:
failed = True
why = str(ex)
