def main():
try:
ctx = saga.Context("x509")
ctx.user_proxy = '/Users/mark/proj/myproxy/xsede.x509'
session = saga.Session()
session.add_context(ctx)
# open home directory on a remote machine
#remote_dir = saga.filesystem.Directory('sftp://hotel.futuregrid.org/opt/',
#remote_dir = saga.filesystem.Directory('go://netbook/', session=session)
#remote_dir = saga.filesystem.Directory('go://marksant#netbook/~/tmp/go')
remote_dir = saga.filesystem.Directory('go://xsede#stampede/~/tmp/go/')
for entry in remote_dir.list():
if remote_dir.is_dir(entry):
print "d %12s %s" % (remote_dir.get_size(entry), entry)
elif remote_dir.is_link(entry):
print "l %12s %s" % (remote_dir.get_size(entry), entry)
elif remote_dir.is_file(entry):
print "- %12s %s" % (remote_dir.get_size(entry), entry)
else:
print 'Other taste ....: %s' % entry
return 0
except saga.SagaException as ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def process_dependencies(dependencies, machine_parameters):
i = 0
for x in dependencies:
ctx = saga.Context("ssh")
ctx.user_id = "vshah505"
session = saga.Session()
session.add_context(ctx)
flag = 1
for files in x:
if flag == 1:
service = machine_parameters[i][1]
service = service[(service.find('/') + 2):]
flag = 0
else:
#print("file://localhost" + os.getcwd() + "/" + files)
#print("file://"+service+machine_parameters[i][2])
f = saga.filesystem.File("file://localhost" + os.getcwd() +
"/" + files,
session=session)
f.copy("sftp://" + service + machine_parameters[i][2] + files)
print "Successful copy"
i += 1
def __init__(self, i_arch, i_project, i_user, i_serviceUrl, i_env,
i_maxJobs):
try:
# set architecture
self.m_architecture = i_arch
# set project
self.m_project = i_project
# set environment
self.m_environment = i_env
# set maximum number of jobs
self.m_maxJobs = i_maxJobs
# create context
self.m_context = saga.Context('ssh')
self.m_context.user_id = i_user
# create saga session
self.m_session = saga.Session()
self.m_session.add_context(self.m_context)
# create service
self.m_service = saga.job.Service(i_serviceUrl,
session=self.m_session)
# catch saga exceptions
except saga.SagaException, i_ex:
self.sagaEx(i_ex)
def start(self):
"""Start the process"""
#self.user.server.port = random_port()
cmd = []
#env = self.user_env(self.env)
cmd.extend(self.cmd)
cmd.extend(self.get_args())
self.log.info("Spawning %r", cmd)
ctx = saga.Context("ssh")
session = saga.Session()
session.add_context(ctx)
js = saga.job.Service("pbs+ssh://gordon.sdsc.edu", session=session)
jd = saga.job.Description()
jd.environment = {'MYOUTPUT': '"Hello from SAGA"'}
jd.executable = cmd[0]
jd.arguments = cmd[1:]
jd.output = "mysagajob.stdout"
jd.error = "mysagajob.stderr"
jd.wall_time_limit = 5
self.job = js.create_job(jd)
self.job.run()
def main():
try:
# Your ssh identity on the remote machine.
ctx = saga.Context("ssh")
ctx.user_id = getpass.getuser() # Change if necessary
session = saga.Session()
session.add_context(ctx)
# open home directory on a remote machine
remote_dir = saga.filesystem.Directory(
'sftp://hotel.futuregrid.org/opt/', session=session)
for entry in remote_dir.list():
if remote_dir.is_dir(entry):
print "d %12s %s" % (remote_dir.get_size(entry), entry)
else:
print "- %12s %s" % (remote_dir.get_size(entry), entry)
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def main():
try:
# Your ssh identity on the remote machine.
ctx = saga.Context("ssh")
# Change e.g., if you have a differnent username on the remote machine
#ctx.user_id = "your_ssh_username"
session = saga.Session()
session.add_context(ctx)
# open home directory on a remote machine
remote_dir = saga.filesystem.Directory(
'sftp://stampede.tacc.xsede.org/etc/', session=session)
# copy .bash_history to /tmp/ on the local machine
remote_dir.copy('hosts', 'file://localhost/tmp/')
# list 'h*' in local /tmp/ directory
local_dir = saga.filesystem.Directory('file://localhost/tmp/')
for entry in local_dir.list(pattern='h*'):
print entry
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def main():
try:
# Your ssh identity on the remote machine. Change if necessary.
ctx = saga.Context("ssh")
#ctx.user_id = "oweidner"
#ctx.user_key = "/Users/oweidner/.ssh/sagaproj_rsa"
session = saga.Session()
session.add_context(ctx)
# Create a job service object that represent the local machine.
# The keyword 'fork://' in the url scheme triggers the 'shell' adaptor
# which can execute jobs on the local machine as well as on a remote
# machine via "ssh://hostname". You can use 'localhost' or replace
# it with the name/address of a machien you have ssh access to.
js = saga.job.Service("ssh://localhost", session=session)
# describe our job
jd = saga.job.Description()
# Next, we describe the job we want to run. A complete set of job
# description attributes can be found in the API documentation.
jd.environment = {'MYOUTPUT': '"Hello from SAGA"'}
jd.executable = '/bin/echo'
jd.arguments = ['$MYOUTPUT']
jd.output = "mysagajob.stdout"
jd.error = "mysagajob.stderr"
# Create a new job from the job description. The initial state of
# the job is 'New'.
myjob = js.create_job(jd)
# Check our job's id and state
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...starting job...\n"
# Now we can start our job.
myjob.run()
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob.wait()
print "Job State : %s" % (myjob.state)
print "Exitcode : %s" % (myjob.exit_code)
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def _get_session():
global session
if session is None:
ctx = saga.Context("ssh")
ctx.user_id = USER
session = saga.Session(default=False)
session.add_context(ctx)
return session
def create_session():
""" Creates and returns a new SAGA session """
ctx = saga.Context("UserPass")
ctx.user_id = USER
ctx.user_pass = PASSWORD
session = saga.Session()
session.add_context(ctx)
return session
def connect(self, machine):
"""Stablish SSH connection with remote cluster"""
try:
ctx = saga.Context(self.config.get_server(machine).get_protocol())
ctx.user_id = self.config.get_server(machine).get_user()
session = saga.Session()
session.add_context(ctx)
except saga.SagaException, ex:
logging.error("Job: ", str(ex))
def transfer_file(src_files, target_dir, resource, tar):
# Remote transfering happens by using a saga session
# Here we create one. We are adding as connecting context
# gsissh
#ctx = rs.Context("gsissh")
session = rs.Session()
# session.add_context(ctx)
# Setting up the remote filesystem connection. `remote_dir` now is a
target_connection = resource + ':2222/'
remote_dir = rs.filesystem.Directory('gsisftp://' + target_connection,
session=session)
# Create the remote directory with all the parents
remote_dir.make_dir(target_dir, flags=rs.filesystem.CREATE_PARENTS)
tgt_dir = 'gsisftp://' + target_connection + target_dir
# Do the actual transfer.
start_time = time()
if tar:
SrcFiles = ['tartest.tar']
tar = tarfile.open(SrcFiles[0], "w")
for filename in src_files:
tar.add(filename)
tar.close()
else:
SrcFiles = src_files
for src_file in SrcFiles:
src_filename = src_file.split('/')[-1]
src_context = {
'pwd': os.path.dirname(os.path.abspath(src_file)),
'unit': tgt_dir,
'pilot': tgt_dir,
'resource': tgt_dir
}
tgt_context = {
'pwd': tgt_dir,
'unit': tgt_dir,
'pilot': tgt_dir,
'resource': tgt_dir
}
src = complete_url(src_filename, src_context,
ru.get_logger('transfer.files'))
tgt = complete_url(src_filename, tgt_context,
ru.get_logger('transfer.files'))
#target_file = target_dir + src_file.split('/')[-1]
remote_dir.copy(src, tgt, flags=rs.filesystem.CREATE_PARENTS)
end_time = time()
print(end_time - start_time)
def get_proxy():
logger = logging.getLogger('periodic_tasks_logger')
getRotatingFileHandler(logger, 'celery.get_proxy.log')
proxy_local = '/tmp/x509up_u%s' % os.geteuid()
proxy_user_id = settings.PROXY_USER_ID
proxy_password = settings.PROXY_PASSWORD
try:
ctx = saga.Context("UserPass")
ctx.user_id = proxy_user_id # remote login name
ctx.user_pass = proxy_password # password
if os.path.isfile(proxy_local):
old_proxy = os.stat(proxy_local).st_mtime
logger.info("Current proxy: %s" % time.ctime(old_proxy))
logger.info('connect to pandawms')
session = saga.Session()
session.add_context(ctx)
js = saga.job.Service("ssh://pandawms.jinr.ru", session=session)
jd = saga.job.Description()
jd.executable = "voms-proxy-init -voms vo.compass.cern.ch:/vo.compass.cern.ch/Role=production --valid 96:00 -q -old --out /home/virthead/x509up_u500 -pwstdin < proxy/gp"
jd.output = "/home/virthead/proxy/GetProxy.stdout" # full path to remote stdout
jd.error = "/home/virthead/proxy/GetProxy.stderr" # full path to remote srderr
myjob = js.create_job(jd)
myjob.run()
myjob.wait()
old_proxy = 0.0
outfilesource = 'sftp://pandawms.jinr.ru/home/virthead/x509up_u500' # path to proxy
outfiletarget = 'file://localhost%s' % proxy_local
logger.info('start loading proxy')
load = True
while load:
out = saga.filesystem.File(outfilesource, session=session)
out.copy(outfiletarget)
new_proxy = os.stat(proxy_local).st_mtime
if new_proxy > old_proxy:
load = False
logger.info('proxy loaded')
new_proxy = os.stat(proxy_local).st_mtime
logger.info("New proxy: %s" % time.ctime(new_proxy))
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
logger.exception("An exception occured: (%s) %s " % (ex.type,
(str(ex))))
# Trace back the exception. That can be helpful for debugging.
logger.exception(" \n*** Backtrace:\n %s" % ex.traceback)
return -1
def __init__(self, platform_config):
'''
This is the base constructor for the job deployment class.
This will be called by overridden constructors in the platform-specific
subclass implementations.
'''
self.platform_config = platform_config
self.host = self.platform_config.platform_service_host
self.port = self.platform_config.platform_service_port
self.session = saga.Session(default=False)
def get_osg_task_status(task_id):
# Your ssh identity on the remote machine.
ctx = saga.Context("ssh")
ctx.user_id = getpass.getuser() # Change if necessary
session = saga.Session()
session.add_context(ctx)
js = saga.job.Service(CONDOR_URL, session=session)
sleebjob_clone = js.get_job(task_id)
return sleebjob_clone.state
def test_directory_get_session(self):
""" Testing if the correct session is being used
"""
try:
tc = testing.get_test_config()
session = tc.session or saga.Session()
d = saga.filesystem.Directory(tc.filesystem_url, session=session)
assert d.get_session() == session, 'Failed setting the session'
except saga.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def test_ptyshell_file_stage():
""" Test pty_shell file staging """
conf = sutc.TestConfig()
shell = sups.PTYShell(saga.Url("fork://localhost"), saga.Session())
txt = "______1______2_____3_____"
shell.stage_to_remote(txt, "/tmp/saga-test-staging-$$")
out = shell.read_from_remote("/tmp/saga-test-staging-$$")
assert (txt == out)
ret, out, _ = shell.run_sync("rm /tmp/saga-test-staging-111")
assert (ret == 0)
assert (out == "")
def perf(n_jobs, tuples):
try:
s = saga.Session()
targets = ""
urls = []
threads = []
services = []
n_services = 0
for (n, url) in tuples:
for i in range(0, n):
urls.append(url)
n_services += n
targets += "%d*%s " % (n, url)
for url in urls:
thread = threading.Thread(target=create_service,
args=[url, services])
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
threads = []
start = time.time()
for service in services:
thread = threading.Thread(target=run_jobs,
args=[service, n_jobs // n_services])
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
stop = time.time()
seconds = stop - start
rate = n_jobs / (seconds)
print "%10s %5s %5.2f %7.2f %s" % (n_services, n_jobs, seconds,
rate, targets)
except saga.exceptions.SagaException as e:
print "Exception: ==========\n%s" % e.get_message()
print "%s=====================" % e.get_traceback()
def test_file_get_session(self):
""" Testing if the correct session is being used
"""
try:
tc = testing.get_test_config()
session = tc.session or saga.Session()
filename = deepcopy(saga.Url(tc.filesystem_url))
filename.path += "/%s" % self.uniquefilename1
f = saga.filesystem.File(filename,
saga.filesystem.CREATE,
session=session)
assert f.get_session() == session, 'Failed setting the session'
f2 = saga.filesystem.File(f.url, session=session)
assert f2.get_session() == session, 'Failed setting the session'
except saga.SagaException as ex:
assert False, "Unexpected exception: %s" % ex
def start_run_osg_task(task_id):
# Your ssh identity on the remote machine.
ctx = saga.Context("ssh")
ctx.user_id = getpass.getuser() # Change if necessary
session = saga.Session()
session.add_context(ctx)
js = saga.job.Service(CONDOR_URL, session=session)
jd = saga.job.Description()
jd.name = 'testjob'
jd.project = 'TG-MCB090174'
jd.environment = {'RUNTIME': '/etc/passwd'}
jd.wall_time_limit = 2 # minutes
jd.executable = '/bin/cat'
jd.arguments = ["$RUNTIME"]
jd.output = "saga_condorjob.stdout"
jd.error = "saga_condorjob.stderr"
# jd.candidate_hosts = ["FNAL_FERMIGRID", "cinvestav", "SPRACE",
# "NYSGRID_CORNELL_NYS1", "Purdue-Steele",
# "MIT_CMS_CE2", "SWT2_CPB", "AGLT2_CE_2",
# "UTA_SWT2", "GridUNESP_CENTRAL",
# "USCMS-FNAL-WC1-CE3"]
# create the job (state: New)
sleepjob = js.create_job(jd)
# check our job's id and state
print("Job ID : %s" % (sleepjob.id))
print("Job State : %s" % (sleepjob.state))
print("\n...starting job...\n")
sleepjob.run()
print("Job ID : %s" % (sleepjob.id))
print("Job State : %s" % (sleepjob.state))
def test_job_service_get_session():
""" Test if the job service session is set correctly
"""
js = None
session = None
try:
tc = testing.get_test_config()
session = tc.session or saga.Session()
js = saga.job.Service(tc.job_service_url, session)
assert js.get_session() == session, "Setting service session failed."
assert js.session == session, "Setting service session failed."
assert js._adaptor.get_session(
) == session, "Setting service session failed."
assert js._adaptor.session == session, "Setting service session failed."
except saga.SagaException as ex:
assert False, "unexpected exception %s" % ex
finally:
_silent_close_js(js)
def main():
try:
c = saga.Context ('ssh')
c.user_id = 'tg12736'
c.user_cert = '/home/user/ssh/id_rsa_xsede' # private key derived from cert
s = saga.Session (default=False) # create session with no contexts
s.add_context (c)
js = saga.job.Service ('ssh://login1.stampede.tacc.utexas.edu', session=s)
js.run_job ("/bin/true")
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def main():
try:
ctx = saga.Context("x509")
ctx.user_proxy = '/Users/mark/proj/myproxy/xsede.x509'
session = saga.Session()
session.add_context(ctx)
source = "go://marksant#netbook/Users/mark/tmp/go/"
#destination = "go://xsede#stampede/~/tmp/"
#destination = "go://gridftp.stampede.tacc.xsede.org/~/tmp/"
destination = "go://oasis-dm.sdsc.xsede.org/~/tmp/"
#destination = "go://ncsa#BlueWaters/~/tmp/"
filename = "my_file"
# open home directory on a remote machine
source_dir = saga.filesystem.Directory(source)
# copy .bash_history to /tmp/ on the local machine
source_dir.copy(filename, destination)
# list 'm*' in local /tmp/ directory
dest_dir = saga.filesystem.Directory(destination)
for entry in dest_dir.list(pattern='%s*' % filename[0]):
print entry
dest_file = saga.filesystem.File(os.path.join(destination, filename))
assert dest_file.is_file() == True
assert dest_file.is_link() == False
assert dest_file.is_dir() == False
print 'Size: %d' % dest_file.get_size()
return 0
except saga.SagaException as ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def __init__(self, cfg_file):
# initialize configuration. We only use the 'saga.tests' category from
# the config file.
rut.TestConfig.__init__(self, cfg_file, 'saga.tests')
# setup a saga session for the tests
# don't populate session with default contexts...
self.session = saga.Session(default=False)
# attempt to create a context from the test config
if self.context_type:
c = saga.Context(self.context_type)
c.user_id = self.context_user_id
c.user_pass = self.context_user_pass
c.user_cert = self.context_user_cert
c.user_proxy = self.context_user_proxy
# add it to the session
self.session.add_context(c)
def main():
try:
# Your ssh identity on the remote machine
ctx = saga.Context("ssh")
ctx.user_id = "oweidner"
session = saga.Session()
# session.add_context(ctx)
# Create a job service object that represent the local machine.
# The keyword 'fork://' in the url scheme triggers the 'shell' adaptor
# which can execute jobs on the local machine as well as on a remote
# machine via "ssh://hostname".
js = saga.job.Service("ssh://%s" % REMOTE_HOST, session=session)
# describe our job
jd = saga.job.Description()
# Next, we describe the job we want to run. A complete set of job
# description attributes can be found in the API documentation.
jd.environment = {'MYOUTPUT':'"Hello from SAGA"'}
jd.executable = '/bin/echo'
jd.arguments = ['$MYOUTPUT']
jd.output = "/tmp/mysagajob-%s.stdout" % getpass.getuser()
jd.error = "/tmp/mysagajob-%s.stderr" % getpass.getuser()
# Create a new job from the job description. The initial state of
# the job is 'New'.
myjob = js.create_job(jd)
# Check our job's id and state
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...starting job...\n"
# Now we can start our job.
myjob.run()
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob.wait()
print "Job State : %s" % (myjob.state)
print "Exitcode : %s" % (myjob.exit_code)
outfilesource = 'sftp://%s/tmp/mysagajob-%s.stdout' % (REMOTE_HOST, getpass.getuser())
outfiletarget = "file://%s/" % os.getcwd()
out = saga.filesystem.File(outfilesource, session=session)
out.copy(outfiletarget)
print "Staged out %s to %s (size: %s bytes)" % (outfilesource, outfiletarget, out.get_size())
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def main():
try:
# Your ssh identity on the remote machine.
ctx = saga.Context("ssh")
# Change e.g., if you have a differnent username on the remote machine
#ctx.user_id = "your_ssh_username"
session = saga.Session()
session.add_context(ctx)
# Create a job service object that represent a remote pbs cluster.
# The keyword 'pbs' in the url scheme triggers the PBS adaptors
# and '+ssh' enables PBS remote access via SSH.
js = saga.job.Service("lsf://localhost", session=session)
# Next, we describe the job we want to run. A complete set of job
# description attributes can be found in the API documentation.
jd = saga.job.Description()
jd.environment = {'FILENAME': 'testfile'}
jd.wall_time_limit = 1 # minutes
jd.executable = '/bin/touch'
jd.arguments = ['$FILENAME']
jd.total_cpu_count = 16
jd.queue = "regular"
jd.project = "URTG0003"
jd.working_directory = "$HOME/A/B/C"
jd.output = "examplejob.out"
jd.error = "examplejob.err"
# Create a new job from the job description. The initial state of
# the job is 'New'.
touchjob = js.create_job(jd)
# Register our callback. We want it to 'fire' on job state change
touchjob.add_callback(saga.STATE, job_state_change_cb)
# Check our job's id and state
print "Job ID : %s" % (touchjob.id)
print "Job State : %s" % (touchjob.state)
# Now we can start our job.
print "\n...starting job...\n"
touchjob.run()
print "Job ID : %s" % (touchjob.id)
# List all jobs that are known by the adaptor.
# This should show our job as well.
#print "\nListing active jobs: "
#for job in js.list():
# print " * %s" % job
# wait for our job to complete
print "\n...waiting for job...\n"
touchjob.wait()
print "Job State : %s" % (touchjob.state)
print "Exitcode : %s" % (touchjob.exit_code)
print "Exec. hosts : %s" % (touchjob.execution_hosts)
print "Create time : %s" % (touchjob.created)
print "Start time : %s" % (touchjob.started)
print "End time : %s" % (touchjob.finished)
js.close()
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occurred: (%s) %s " % (ex.type, (str(ex)))
# Get the whole traceback in case of an exception -
# this can be helpful for debugging the problem
print " \n*** Backtrace:\n %s" % ex.traceback
return -1
def start_cr():
"""
We use SAGA to start a VM (called Compute Resource (cr) in this context)
"""
# In order to connect to EC2, we need an EC2 ID and KEY. We read those
# from the environment.
ec2_ctx = rs.Context('EC2')
ec2_ctx.user_id = os.environ['EC2_ACCESS_KEY']
ec2_ctx.user_key = os.environ['EC2_SECRET_KEY']
# The SSH keypair we want to use the access the EC2 VM. If the keypair is
# not yet registered on EC2 saga will register it automatically. This
# context specifies the key for VM startup, ie. the VM will be configured to
# accept this key
ec2keypair_ctx = rs.Context('EC2_KEYPAIR')
ec2keypair_ctx.token = os.environ['EC2_KEYPAIR_ID']
ec2keypair_ctx.user_key = os.environ['EC2_KEYPAIR']
ec2keypair_ctx.user_id = 'admin' # the user id on the target VM
# We specify the *same* ssh key for ssh access to the VM. That now should
# work if the VM go configured correctly per the 'EC2_KEYPAIR' context
# above.
ssh_ctx = rs.Context('SSH')
ssh_ctx.user_id = 'admin'
ssh_ctx.user_key = os.environ['EC2_KEYPAIR']
session = rs.Session(False) # FALSE: don't use other (default) contexts
session.contexts.append(ec2_ctx)
session.contexts.append(ec2keypair_ctx)
session.contexts.append(ssh_ctx)
cr = None # compute resource handle
rid = None # compute resource ID
try:
# ----------------------------------------------------------------------
#
# reconnect to VM (ID given in ARGV[1])
#
if len(sys.argv) > 1:
rid = sys.argv[1]
# reconnect to the given resource
print 'reconnecting to %s' % rid
cr = rs.resource.Compute(id=rid, session=session)
print 'reconnected to %s' % rid
print " state : %s (%s)" % (cr.state, cr.state_detail)
# ----------------------------------------------------------------------
#
# start a new VM
#
else:
# start a VM if needed
# in our session, connect to the EC2 resource manager
rm = rs.resource.Manager("ec2://aws.amazon.com/", session=session)
# Create a resource description with an image and an OS template,.
# We pick a small VM and a plain Ubuntu image...
cd = rs.resource.ComputeDescription()
cd.image = 'ami-e6eeaa8e' # plain debain wheezy
cd.template = 'Small Instance'
# Create a VM instance from that description.
cr = rm.acquire(cd)
print "\nWaiting for VM to become active..."
# ----------------------------------------------------------------------
#
# use the VM
#
# Wait for the VM to 'boot up', i.e., become 'ACTIVE'
cr.wait(rs.resource.ACTIVE)
# Query some information about the newly created VM
print "Created VM: %s" % cr.id
print " state : %s (%s)" % (cr.state, cr.state_detail)
print " access : %s" % cr.access
# give the VM some time to start up comlpetely, otherwise the subsequent
# job submission might end up failing...
time.sleep(60)
return cr
except Exception as e:
# Catch all other exceptions
print "An Exception occured: %s " % e
raise
def _create_master_entry(self, url, session, prompt, logger, posix,
interactive):
# FIXME: cache 'which' results, etc
# FIXME: check 'which' results
with self.rlock:
info = {'posix': posix}
# get and evaluate session config
if not session:
session = saga.Session(default=True)
session_cfg = session.get_config('saga.utils.pty')
info['ssh_copy_mode'] = session_cfg['ssh_copy_mode'].get_value()
info['ssh_share_mode'] = session_cfg['ssh_share_mode'].get_value()
info['ssh_timeout'] = session_cfg['ssh_timeout'].get_value()
logger.info("ssh copy mode set to '%s'" % info['ssh_copy_mode'])
logger.info("ssh share mode set to '%s'" % info['ssh_share_mode'])
logger.info("ssh timeout set to '%s'" % info['ssh_timeout'])
# fill the info dict with details for this master channel, and all
# related future slave channels
info['schema'] = url.schema.lower()
info['host_str'] = url.host
info['prompt'] = prompt
info['logger'] = logger
info['url'] = url
info['pass'] = ""
info['key_pass'] = {}
info['scripts'] = _SCRIPTS
if not info['schema']:
info['schema'] = 'local'
# find out what type of shell we have to deal with
if info['schema'] in _SCHEMAS_SSH:
info['shell_type'] = "ssh"
info['copy_mode'] = info['ssh_copy_mode']
info['share_mode'] = info['ssh_share_mode']
info['ssh_exe'] = self._which("ssh")
info['scp_exe'] = self._which("scp")
info['sftp_exe'] = self._which("sftp")
elif info['schema'] in _SCHEMAS_GSI:
info['shell_type'] = "ssh"
info['copy_mode'] = info['ssh_copy_mode']
info['share_mode'] = info['ssh_share_mode']
info['ssh_exe'] = self._which("gsissh")
info['scp_exe'] = self._which("gsiscp")
info['sftp_exe'] = self._which("gsisftp")
elif info['schema'] in _SCHEMAS_SH:
info['shell_type'] = "sh"
info['copy_mode'] = "sh"
info['share_mode'] = "auto"
info[
'sh_env'] = "/usr/bin/env TERM=vt100 PROMPT_COMMAND='' PS1='PROMPT-$?->'"
info[
'cp_env'] = "/usr/bin/env TERM=vt100 PROMPT_COMMAND='' PS1='PROMPT-$?->'"
info['scp_root'] = "/"
if interactive: info['sh_args'] = "-i"
else: info['sh_args'] = ""
if "SHELL" in os.environ:
info['sh_exe'] = self._which(os.environ["SHELL"])
info['cp_exe'] = self._which("cp")
else:
info['sh_exe'] = self._which("sh")
info['cp_exe'] = self._which("cp")
else:
raise se.BadParameter._log (self.logger, \
"cannot handle schema '%s://'" % url.schema)
# If an SSH timeout has been specified set up the ConnectTimeout
# string
if info['ssh_timeout']:
info['ssh_connect_timeout'] = ('-o ConnectTimeout=%s' %
int(float(info['ssh_timeout'])))
else:
info['ssh_connect_timeout'] = ''
# depending on type, create command line (args, env etc)
#
# We always set term=vt100 to avoid ansi-escape sequences in the prompt
# and elsewhere. Also, we have to make sure that the shell is an
# interactive login shell, so that it interprets the users startup
# files, and reacts on commands.
try:
info['latency'] = sumisc.get_host_latency(url)
# FIXME: note that get_host_latency is considered broken (see
# saga/utils/misc.py line 73), and will return a constant 250ms.
except Exception as e:
info['latency'] = 1.0 # generic value assuming slow link
info['logger'].warning("Could not contact host '%s': %s" %
(url, e))
if info['shell_type'] == "sh":
info[
'sh_env'] = "/usr/bin/env TERM=vt100 " # avoid ansi escapes
if not sumisc.host_is_local(url.host):
raise se.BadParameter._log (self.logger, \
"expect local host for '%s://', not '%s'" % (url.schema, url.host))
if 'user' in info and info['user']:
pass
else:
info['user'] = getpass.getuser()
else:
info[
'ssh_env'] = "/usr/bin/env TERM=vt100 " # avoid ansi escapes
info[
'scp_env'] = "/usr/bin/env TERM=vt100 " # avoid ansi escapes
info[
'sftp_env'] = "/usr/bin/env TERM=vt100 " # avoid ansi escapes
info['ssh_args'] = "-t " # force pty
info['scp_args'] = _SCP_FLAGS
info['sftp_args'] = _SFTP_FLAGS
if session:
for context in session.contexts:
# ssh can also handle UserPass contexts, and ssh type contexts.
# gsissh can handle the same, but also X509 contexts.
if context.type.lower() == "ssh":
if info['schema'] in _SCHEMAS_SSH + _SCHEMAS_GSI:
if context.attribute_exists(
"user_id") and context.user_id:
info['user'] = context.user_id
if context.attribute_exists(
"user_key") and context.user_key:
info[
'ssh_args'] += "-o IdentityFile=%s " % context.user_key
info[
'scp_args'] += "-o IdentityFile=%s " % context.user_key
info[
'sftp_args'] += "-o IdentityFile=%s " % context.user_key
if context.attribute_exists(
"user_pass") and context.user_pass:
info['key_pass'][
context.
user_key] = context.user_pass
if context.type.lower() == "userpass":
if info['schema'] in _SCHEMAS_SSH + _SCHEMAS_GSI:
if context.attribute_exists(
"user_id") and context.user_id:
info['user'] = context.user_id
if context.attribute_exists(
"user_pass") and context.user_pass:
info['pass'] = context.user_pass
if context.type.lower() == "x509":
if info['schema'] in _SCHEMAS_GSI:
if context.attribute_exists(
"user_proxy") and context.user_proxy:
info[
'ssh_env'] += "X509_USER_PROXY='%s' " % context.user_proxy
info[
'scp_env'] += "X509_USER_PROXY='%s' " % context.user_proxy
info[
'sftp_env'] += "X509_USER_PROXY='%s' " % context.user_proxy
if context.attribute_exists(
"user_cert") and context.user_cert:
info[
'ssh_env'] += "X509_USER_CERT='%s' " % context.user_cert
info[
'scp_env'] += "X509_USER_CERT='%s' " % context.user_cert
info[
'sftp_env'] += "X509_USER_CERT='%s' " % context.user_cert
if context.attribute_exists(
"user_key") and context.user_key:
info[
'ssh_env'] += "X509_USER_key='%s' " % context.user_key
info[
'scp_env'] += "X509_USER_key='%s' " % context.user_key
info[
'sftp_env'] += "X509_USER_key='%s' " % context.user_key
if context.attribute_exists(
"cert_repository"
) and context.cert_repository:
info[
'ssh_env'] += "X509_CERT_DIR='%s' " % context.cert_repository
info[
'scp_env'] += "X509_CERT_DIR='%s' " % context.cert_repository
info[
'sftp_env'] += "X509_CERT_DIR='%s' " % context.cert_repository
if url.port and url.port != -1:
info['ssh_args'] += "-o Port=%d " % int(url.port)
info['scp_args'] += "-o Port=%d " % int(url.port)
info['sftp_args'] += "-o Port=%d " % int(url.port)
# all ssh based shells allow for user_id and user_pass from contexts
# -- but the data given in the URL take precedence
if url.username: info['user'] = url.username
if url.password: info['pass'] = url.password
ctrl_user = pwd.getpwuid(os.getuid()).pw_name
ctrl_base = "/tmp/saga_ssh_%s" % ctrl_user
if 'user' in info and info['user']:
info['host_str'] = "%s@%s" % (info['user'],
info['host_str'])
info['ctrl'] = "%s_%%h_%%p.%s.ctrl" % (ctrl_base,
info['user'])
else:
info['user'] = getpass.getuser()
info['ctrl'] = "%s_%%h_%%p.ctrl" % (ctrl_base)
info['m_flags'] = _SSH_FLAGS_MASTER % (
{
'share_mode': info['share_mode'],
'ctrl': info['ctrl'],
'connect_timeout': info['ssh_connect_timeout']
})
info['s_flags'] = _SSH_FLAGS_SLAVE % (
{
'share_mode': info['share_mode'],
'ctrl': info['ctrl'],
'connect_timeout': info['ssh_connect_timeout']
})
logger.debug('SSH Connection M_FLAGS: %s' % info['m_flags'])
logger.debug('SSH Connection S_FLAGS: %s' % info['s_flags'])
# we want the userauth and hostname parts of the URL, to get the
# scp-scope fs root.
info['scp_root'] = ""
has_auth = False
if url.username:
info['scp_root'] += url.username
has_auth = True
if url.password:
info['scp_root'] += ":"
info['scp_root'] += url.password
has_auth = True
if has_auth:
info['scp_root'] += "@"
info['scp_root'] += "%s:" % url.host
# FIXME: port needs to be handled as parameter
# if url.port :
# info['scp_root'] += ":%d" % url.port
# keep all collected info in the master dict, and return it for
# registration
return info
import time
import saga
def my_cb(a, b, c):
print " ----- callback: [%s, %s, %s]" % (a, b, c)
return True
try:
c = saga.Context('ssh')
c.user_key = '/home/merzky/.ssh/id_rsa_test'
c.user_id = 'tester'
c.user_pass = '******'
s = saga.Session(default=True)
# s.add_context (c)
# js = saga.job.Service ('gsissh://gsissh.kraken.nics.xsede.org', session=s)
# js = saga.job.Service ('ssh://localhost/', session=s)
js = saga.job.Service('ssh://india.futuregrid.org/', session=s)
jd = saga.job.Description()
jd.executable = '/bin/echo'
jd.arguments = ['hello world; date ; sleep 3']
# jd.output = "/tmp/out"
# jd.error = "/tmp/err"
j = js.create_job(jd)
# j.add_callback ('State', my_cb)
print j.created
def __init__(self, addr):
self.addr = addr
self.ctx = saga.Context("ssh")
self.ctx.user_id = "dilawar"
self.session = saga.Session()
self.session.add_context(self.ctx)
def main():
try:
# Your ssh identity on the remote machine
ctx = saga.Context("ssh")
ctx.user_id = "your_username"
session = saga.Session()
session.add_context(ctx)
# Create a job service object that represent a remote loadleveler cluster.
# The keyword 'loadl' in the url scheme triggers the LoadLeveler adaptors
# and '+ssh' enables LoadLeveler remote access via SSH.
# and 'cluster' URL query specify loadleveler cluster name. "llq -X cluster"
js = saga.job.Service("loadl+ssh://%s?cluster=your_cluster_name" %
REMOTE_HOST,
session=session)
# describe our job
jd = saga.job.Description()
# Next, we describe the job we want to run. A complete set of job
# description attributes can be found in the API documentation.
jd.environment = {'MYOUTPUT': '"Hello LoadLevler Adaptor from SAGA"'}
jd.executable = '/bin/echo'
jd.arguments = ['$MYOUTPUT']
jd.output = "/tmp/mysagajob.stdout"
jd.error = "/tmp/mysagajob.stderr"
# Create a new job from the job description. The initial state of
# the job is 'New'.
myjob = js.create_job(jd)
# Check our job's id and state
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...starting job...\n"
# Now we can start our job.
myjob.run()
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob.wait()
print "Job ID : %s" % (myjob.id)
print "Job State : %s" % (myjob.state)
print "\n...waiting for job...\n"
# wait for the job to either finish or fail
myjob.wait()
print "Job State : %s" % (myjob.state)
print "Exitcode : %s" % (myjob.exit_code)
outfilesource = 'sftp://%s/tmp/mysagajob.stdout' % REMOTE_HOST
outfiletarget = 'file://localhost/tmp/'
out = saga.filesystem.File(outfilesource, session=session)
out.copy(outfiletarget)
print "Staged out %s to %s (size: %s bytes)\n" % (
outfilesource, outfiletarget, out.get_size())
return 0
except saga.SagaException, ex:
# Catch all saga exceptions
print "An exception occured: (%s) %s " % (ex.type, (str(ex)))
# Trace back the exception. That can be helpful for debugging.
print " \n*** Backtrace:\n %s" % ex.traceback
return -1