def test_pass_issue_57():
for i in [16, 32, 64]:
session = rp.Session(database_url=db_url)
try:
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.stampede"]["user_id"]
session.add_context(c)
pmgr = rp.PilotManager(session=session)
umgr = rp.UnitManager(session=session,
scheduler=rp.SCHED_ROUND_ROBIN)
pdesc = rp.ComputePilotDescription()
pdesc.resource = "xsede.stampede"
pdesc.project = CONFIG["xsede.stampede"]["project"]
pdesc.cores = i
pdesc.runtime = 20
pdesc.cleanup = False
pilots = pmgr.submit_pilots(pdesc)
umgr.add_pilots(pilots)
unit_descrs = []
for k in range(0, i * 2):
cu = rp.ComputeUnitDescription()
cu.cores = 1
cu.executable = "/bin/date"
unit_descrs.append(cu)
units = umgr.submit_units(unit_descrs)
try:
umgr.wait_units()
for unit in units:
unit.wait()
except:
pass
pmgr.cancel_pilots()
pmgr.wait_pilots()
except Exception as e:
print "TEST FAILED"
raise
finally:
session.close()
def test_fail_session_ctx():
s1 = None
s2 = None
try:
s1 = rp.Session(database_url=db_url, database_name='rp-testing')
print "Session 1: %s (%d)" % (s1.uid, len(s1.list_contexts()))
c1 = rp.Context('ssh')
c1.user_id = "tg802352"
print 'context 1: %s' % c1
s1.add_context(c1)
c2 = rp.Context('ssh')
c2.user_id = "abcedesds"
print 'context 2: %s' % c2
s1.add_context(c2)
for c in s1.list_contexts():
print c
s2 = rp.Session(uid=s1.uid)
print "Session 2: %s (%d)" % (s2.uid, len(s2.list_contexts()))
for c in s2.list_contexts():
print c
assert (len(s1.list_contexts()) == len(s2.list_contexts()))
except Exception as e:
raise
finally:
if s1: s1.close()
if s2: s2.close()
def setup_gordon(request):
session1 = rp.Session()
print "session id gordon: {0}".format(session1.uid)
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.gordon"]["user_id"]
session1.add_context(c)
try:
pmgr1 = rp.PilotManager(session=session1)
print "pm id gordon: {0}".format(pmgr1.uid)
umgr1 = rp.UnitManager (session=session1,
scheduler=rp.SCHEDULER_DIRECT_SUBMISSION)
pdesc1 = rp.ComputePilotDescription()
pdesc1.resource = "xsede.gordon"
pdesc1.project = CONFIG["xsede.gordon"]["project"]
pdesc1.runtime = 30
pdesc1.cores = 16
pdesc1.cleanup = False
pilot1 = pmgr1.submit_pilots(pdesc1)
pilot1.register_callback(pilot_state_cb)
umgr1.add_pilots(pilot1)
except Exception as e:
print 'test failed'
raise
def fin():
print "finalizing..."
pmgr1.cancel_pilots()
pmgr1.wait_pilots()
print 'closing session'
session1.close()
time.sleep(5)
request.addfinalizer(fin)
return session1, pilot1, pmgr1, umgr1, "xsede.gordon"
def setup_stampede_two(request):
session3 = rp.Session()
print "session id stampede: {0}".format(session3.uid)
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.stampede"]["user_id"]
session3.add_context(c)
try:
pmgr3 = rp.PilotManager(session=session3)
print "pm id stampede: {0}".format(pmgr3.uid)
umgr3 = rp.UnitManager(session=session3,
scheduler=rp.SCHED_DIRECT_SUBMISSION)
pdesc3 = rp.ComputePilotDescription()
pdesc3.resource = "xsede.stampede"
pdesc3.project = CONFIG["xsede.stampede"]["project"]
pdesc3.runtime = 20
pdesc3.cores = int(CONFIG["xsede.stampede"]["cores"]) * 2
pdesc3.cleanup = False
pilot3 = pmgr3.submit_pilots(pdesc3)
pilot3.register_callback(pilot_state_cb)
umgr3.add_pilots(pilot3)
except Exception as e:
print 'test failed'
raise
def fin():
print "finalizing..."
pmgr3.cancel_pilots()
pmgr3.wait_pilots()
print 'closing session'
session3.close()
request.addfinalizer(fin)
return session3, pilot3, pmgr3, umgr3, "xsede.stampede"
def setup_comet(request):
session2 = rp.Session()
print "session id comet: {0}".format(session2.uid)
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.comet"]["user_id"]
session2.add_context(c)
try:
pmgr2 = rp.PilotManager(session=session2)
print "pm id gordon: {0}".format(pmgr2.uid)
umgr2 = rp.UnitManager(session=session2,
scheduler=rp.SCHED_DIRECT_SUBMISSION)
pdesc2 = rp.ComputePilotDescription()
pdesc2.resource = "xsede.comet"
pdesc2.project = CONFIG["xsede.comet"]["project"]
pdesc2.runtime = 30
pdesc2.cores = 24
pdesc2.cleanup = False
pilot2 = pmgr2.submit_pilots(pdesc2)
pilot2.register_callback(pilot_state_cb)
umgr2.add_pilots(pilot2)
except Exception as e:
print 'test failed'
raise
def fin():
print "finalizing..."
pmgr2.cancel_pilots()
pmgr2.wait_pilots()
print 'closing session'
session2.close()
request.addfinalizer(fin)
return session2, pilot2, pmgr2, umgr2, "xsede.comet"
def setup_stampede_683(request):
session = rp.Session()
print "session id stampede: {0}".format(session.uid)
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.stampede"]["user_id"]
session.add_context(c)
try:
pmgr = rp.PilotManager(session=session)
umgr = rp.UnitManager(session=session,
scheduler=rp.SCHEDULER_BACKFILLING)
pdesc = rp.ComputePilotDescription()
pdesc.resource = "xsede.stampede"
pdesc.project = CONFIG["xsede.stampede"]["project"]
pdesc.runtime = 40
pdesc.cores = 683
pdesc.cleanup = False
pilot = pmgr.submit_pilots(pdesc)
pilot.register_callback(pilot_state_cb)
umgr.add_pilots(pilot)
except Exception as e:
print 'test failed'
raise
def fin():
print "finalizing..."
pmgr.cancel_pilots()
pmgr.wait_pilots()
print 'closing session'
session.close()
request.addfinalizer(fin)
return session, pilot, pmgr, umgr, "xsede.stampede"
BLOCK_SIZE = int(sys.argv[2])
cores = int(sys.argv[3])
session_name = sys.argv[4]
MANIFEST_NAME = "manifest.json"
try:
PROJECT = os.environ['RADICAL_PILOT_PROJECT']
if not PROJECT:
raise ValueError
except (KeyError, ValueError):
raise RuntimeError(
"Set RADICAL_PILOT_PROJECT to your XSEDE allocation")
try:
session = rp.Session(name=session_name)
c = rp.Context('ssh')
session.add_context(c)
print "initialize pilot manager ..."
pmgr = rp.PilotManager(session=session)
#pmgr.register_callback (pilot_state_cb)
pdesc = rp.ComputePilotDescription()
pdesc.resource = "xsede.stampede" # xsede.stampede or xsede.comet or ... check docs!
pdesc.runtime = 20 # minutes
pdesc.cores = cores
pdesc.project = PROJECT #Project allocation, pass through env var PROJECT
pdesc.cleanup = False
pdesc.access_schema = 'ssh'
# submit the pilot.
session = rp.Session(name=session_name)
print "session id: %s" % session.uid
# all other pilot code is now tried/excepted. If an exception is caught, we
# can rely on the session object to exist and be valid, and we can thus tear
# the whole RP stack down via a 'session.close()' call in the 'finally'
# clause...
try:
# ----- CHANGE THIS -- CHANGE THIS -- CHANGE THIS -- CHANGE THIS ------
#
# Change the user name below if you are using a remote resource
# and your username on that resource is different from the username
# on your local machine.
#
c = rp.Context('userpass')
#c.user_id = "tutorial_X"
#c.user_pass = "******"
session.add_context(c)
# Add a Pilot Manager. Pilot managers manage one or more ComputePilots.
print "Initializing Pilot Manager ..."
pmgr = rp.PilotManager(session=session)
# Register our callback with the PilotManager. This callback will get
# called every time any of the pilots managed by the PilotManager
# change their state.
pmgr.register_callback(pilot_state_cb)
# ----- CHANGE THIS -- CHANGE THIS -- CHANGE THIS -- CHANGE THIS ------
#
def test_pass_issue_359():
session = rp.Session()
try:
c = rp.Context('ssh')
c.user_id = CONFIG["xsede.stampede"]["user_id"]
session.add_context(c)
pmgr = rp.PilotManager(session=session)
pmgr.register_callback(pilot_state_cb)
core_configs = [8, 16, 17, 32, 33]
umgr_list = []
for cores in core_configs:
umgr = rp.UnitManager(session=session,
scheduler=rp.SCHED_DIRECT_SUBMISSION)
umgr.register_callback(unit_state_cb)
pdesc = rp.ComputePilotDescription()
pdesc.resource = "xsede.stampede"
pdesc.project = CONFIG["xsede.stampede"]["project"]
pdesc.runtime = 10
pdesc.cores = cores
pilot = pmgr.submit_pilots(pdesc)
umgr.add_pilots(pilot)
umgr_list.append(umgr)
unit_list = []
for umgr in umgr_list:
test_task = rp.ComputeUnitDescription()
test_task.pre_exec = CONFIG["xsede.stampede"]["pre_exec"]
test_task.input_staging = ["../helloworld_mpi.py"]
test_task.executable = "python"
test_task.arguments = ["helloworld_mpi.py"]
test_task.mpi = True
test_task.cores = 8
unit = umgr.submit_units(test_task)
unit_list.append(unit)
for umgr in umgr_list:
umgr.wait_units()
for unit in unit_list:
print "* Task %s - state: %s, exit code: %s, started: %s, finished: %s, stdout: %s" \
% (unit.uid, unit.state, unit.exit_code, unit.start_time, unit.stop_time, unit.stdout)
assert (unit.state == rp.DONE)
except Exception as e:
print 'test failed'
raise
finally:
pmgr.cancel_pilots()
pmgr.wait_pilots()
session.close()
#!/usr/bin/env python
import radical.pilot as rp
s1 = None
s2 = None
try:
s1 = rp.Session()
print "Session 1: %s (%d)" % (s1.uid, len(s1.list_contexts()))
# Add an ssh identity to the session.
c1 = rp.Context('ssh')
c1.user_id = "tg802352"
print 'context 1: %s' % c1
s1.add_context(c1)
# Add an ssh identity to the session.
c2 = rp.Context('ssh')
c2.user_id = "abcedesds"
print 'context 2: %s' % c2
s1.add_context(c2)
for c in s1.list_contexts():
print c
print "stderr: %s" % unit.stderr
os.system('clear')
print "Seeing if I can come up with a Radical Script on my own"
print "NEED TO PRINT ALL OBJECT IDS"
# if database_url parameter is not set, whatever the environment variable
# RADICAL_PILOT_DBURL is set to will be used
my_session = rp.Session(
database_url="mongodb://ec2-54-221-194-147.compute-1.amazonaws.com:24242/")
#my_session = rp.Session()
print " Now we create a Context object"
context = rp.Context(
'ssh') # We communicate with the machine doing the work via ssh
context.user_id = "nrs76"
my_session.add_context(context)
print "Let's create a Pilot Manager"
pm = rp.PilotManager(session=my_session)
print " Now we create the description for a Compute Pilot and submit it to the Pilot Manager"
pd = rp.ComputePilotDescription()
pd.resource = 'futuregrid.india'
pd.cores = 1
pd.runtime = 5 # minutes
print " submitting the Pilot Description to the Pilot Manager to create the Pilot"