def setUp(self):
super(TestSQLitePoolWithTaxis, self).setUp()
with self.test_pool:
self.taxi_one = taxi.Taxi(time_limit=4000,
cores=8,
nodes=1,
name="one")
self.taxi_two = taxi.Taxi(time_limit=9999,
cores=1,
nodes=1,
name="two")
self.test_pool.register_taxi(self.taxi_one)
self.test_pool.register_taxi(self.taxi_two)
def test_taxi_relaunch(self):
# Set up single copy task pool, registered to test1
test_task = taxi.tasks.Copy(self.src_files[0], self.dst_files[0], for_taxi=self.my_taxi.name)
with self.my_disp:
self.my_disp.initialize_new_task_pool([test_task])
# Add a second taxi to the pool and queue it
taxi_two = taxi.Taxi(name='test2', time_limit=120, nodes=1, cores=1)
with self.my_pool:
self.my_pool.register_taxi(taxi_two)
self.my_disp.register_taxi(taxi_two, self.my_pool)
self.my_pool.submit_taxi_to_queue(taxi_two, self.my_queue)
with self.my_pool:
pool_stat = self.my_pool.get_all_taxis_in_pool()
# Run queue with the second, taskless taxi
# During this step: taxi_two should detect that there is a task it cannot
# run, and launch my_taxi to run it.
with self.squeue:
self.squeue.run_next_job()
# After one step, the second taxi should be held, the first still idle
with self.my_pool:
pool_stat = self.my_pool.get_all_taxis_in_pool()
self.assertEqual(pool_stat[0].status, 'Q')
self.assertEqual(pool_stat[1].status, 'H')
# The first taxi should now be queued into the 'batch' system
qstat = self.my_queue.report_taxi_status(self.my_taxi)
self.assertEqual(qstat['status'], 'Q')
self.assertEqual(qstat['job_number'], 2)
# And the copy task should still be unfinished
with self.my_disp:
task_blob = self.my_disp.get_all_tasks(self.my_taxi, include_complete=False)
self.assertEqual(len(task_blob), 1)
# Step into the queue one more time
with self.squeue:
self.squeue.run_next_job()
# Now both taxis should be held
with self.my_pool:
pool_stat = self.my_pool.get_all_taxis_in_pool()
self.assertEqual(pool_stat[0].status, 'H')
self.assertEqual(pool_stat[1].status, 'H')
# The task should be finished
with self.my_disp:
task_blob = self.my_disp.get_all_tasks(self.my_taxi, include_complete=True)
self.assertEqual(len(task_blob), 1)
self.assertEqual(task_blob[1].status, 'complete')
# And the destination file should exist
files = os.listdir('./test_run')
self.assertTrue(self.dst_files[0].split('/')[-1] in files)
def main():
""" Code to use Taxi class """
my_taxi = taxi.Taxi('Prius', 100)
my_taxi.drive(40)
print(my_taxi)
my_taxi.start_fare()
my_taxi.drive(100)
print(my_taxi)
my_taxi.add_fuel(65)
my_taxi.start_fare()
my_taxi.drive(45)
print(my_taxi)
def _create_taxi_object(self, db_taxi):
"""Interface to translate Taxi representation in the DB to a Taxi object.
"""
db_taxi = dict(db_taxi)
new_taxi = taxi.Taxi()
new_taxi.rebuild_from_dict(db_taxi)
new_taxi.pool_path = self.db_path
new_taxi.log_dir = self.log_dir # Tell taxi where log_dir for this pool is
# Pass on pool-level attributes to taxi for convenience
new_taxi.allocation = self.allocation
new_taxi.queue = self.queue
return new_taxi
def test_read_write_taxi(self):
with self.test_pool:
taxi_one = taxi.Taxi(time_limit=4000, cores=8, nodes=1, name="one")
self.test_pool.register_taxi(taxi_one)
taxi_list = self.test_pool.get_all_taxis_in_pool()
self.assertEqual(len(taxi_list), 1)
taxi_one_return = taxi_list[0]
self.assertEqual(taxi_one_return.name, "one")
self.assertEqual(taxi_one_return.time_limit, taxi_one.time_limit)
self.assertEqual(taxi_one_return.cores, taxi_one.cores)
self.assertEqual(taxi_one_return.time_last_submitted, None)
self.assertEqual(taxi_one_return.status, 'I')
def setUp(self):
super(TestSQLitePoolQueueInteraction, self).setUp()
# Three taxis in pool; taxi_one and taxi_two start with status='I' (default)
# taxi_three starts with status='H' (hold)
with self.test_pool:
self.taxi_one = taxi.Taxi(time_limit=4000.,
cores=8,
nodes=1,
name="one")
self.taxi_two = taxi.Taxi(time_limit=9999.,
cores=1,
nodes=1,
name="two")
self.taxi_three = taxi.Taxi(time_limit=4040.,
cores=8,
nodes=1,
name="three")
self.test_pool.register_taxi(self.taxi_one)
self.test_pool.register_taxi(self.taxi_two)
self.test_pool.register_taxi(self.taxi_three)
self.test_pool.update_taxi_status(self.taxi_three, 'H')
def setUp(self):
if not os.path.exists('./test_run'):
ensure_path_exists('./test_run')
## Source/destination for copy test tasks
self.src_files = ['./test_run/test_ab', './test_run/test_cd', './test_run/test_ef']
self.dst_files = ['./test_run/test_uv', './test_run/test_wx', './test_run/test_yz']
# Use abspaths
# TODO: Change test to use relative paths, becomes an implicit test of working dir functionality
self.src_files = map(taxi.expand_path, self.src_files)
self.dst_files = map(taxi.expand_path, self.dst_files)
for i in range(len(self.src_files)):
with open(self.src_files[i], 'w') as test_file:
test_file.write(self.src_files[i])
for dst_fn in self.dst_files:
if os.path.exists(dst_fn):
os.remove(dst_fn)
## Set up queue system
self.my_queue = local_queue.LocalQueue()
self.squeue = qrun.SQLiteSerialQueue()
## Set up test pool
self.my_taxi = taxi.Taxi(name='test1', time_limit=120, nodes=1, cores=1)
self.pool_path = './test_run/test-pool.sqlite'
self.pool_wd = './test_run/'
self.pool_ld = './test_run/log'
self.my_pool = taxi.pool.SQLitePool(self.pool_path, 'test_pool', self.pool_wd, self.pool_ld)
## Set up test dispatch
self.disp_path = './test_run/test-disp.sqlite'
self.my_disp = taxi.dispatcher.SQLiteDispatcher(self.disp_path)
with self.my_disp:
pass
## Registration
with self.my_pool:
self.my_pool.register_taxi(self.my_taxi)
self.my_disp.register_taxi(self.my_taxi, self.my_pool)
pool_name=pool_name,
work_dir=(base_path + "/work/"),
log_dir=(base_path + "/log/"),
allocation='multirep'
)
## Setup dispatcher
my_disp = taxi.dispatcher.SQLiteDispatcher(db_path=dispatch_db_path)
## Initialize task pool with the dispatch
with my_disp:
my_disp.initialize_new_task_pool(task_pool)
# Create taxi(s) to run the job
taxi_list = []
for i in range(2):
taxi_list.append(taxi.Taxi(
time_limit=10*60,
cores=32,
nodes=1
))
## Register taxis and launch!
with my_pool:
for my_taxi in taxi_list:
my_pool.register_taxi(my_taxi)
my_disp.register_taxi(my_taxi, my_pool)
my_pool.spawn_idle_taxis(dispatcher=my_disp)
def setUp(self):
print 'In setUp()'
self.t1 = taxi.Taxi(100, 'normal')
self.t2 = taxi.Taxi(100, 'special')
self.t3 = taxi.Taxi(100, 'somthingelse')
task_pool,
out_dir=base_path + '/data/',
gauge_dir=base_path + '/gauge/')
## Set up pool and feed it taxis
pool_name = "spec"
my_pool = taxi.pool.SQLitePool(db_path=pool_db_path,
pool_name=pool_name,
work_dir=(base_path + "/work/"),
log_dir=(base_path + "/log/"))
## Setup dispatcher
my_disp = taxi.dispatcher.SQLiteDispatcher(db_path=dispatch_db_path)
## Initialize task pool with the dispatch
with my_disp:
my_disp.initialize_new_task_pool(task_pool)
# Create taxi(s) to run the tasks
taxi_list = []
for i in range(20):
taxi_list.append(taxi.Taxi(time_limit=10 * 3600, cores=8, nodes=1))
## Register taxis and launch!
with my_pool:
for my_taxi in taxi_list:
my_pool.register_taxi(my_taxi)
my_disp.register_taxi(my_taxi, my_pool)
my_pool.spawn_idle_taxis(dispatcher=my_disp)
my_pool = taxi.pool.SQLitePool(
db_path=parg.pool_path,
pool_name=parg.pool_name,
)
my_queue = local_queue.LocalQueue()
## "Who am I?" -- Get information about this taxi from pool
with my_pool:
taxi_obj = my_pool.get_taxi(parg.name) # Get taxi object from pool
if taxi_obj is None:
# Taxi object with our name was not found in pool
# Construct taxi object...
taxi_obj = taxi.Taxi(name=parg.name,
pool_name=parg.pool_name,
time_limit=parg.time_limit,
cores=parg.cores,
nodes=parg.nodes)
taxi_obj.pool_path = parg.pool_path
taxi_obj.dispatch_path = parg.dispatch_path
# ...and add it to the pool
with my_pool:
my_pool.register_taxi(taxi_obj)
my_dispatch.register_taxi(taxi_obj, my_pool)
## Record starting time
taxi_obj.start_time = _start_time
print "Start time:", datetime.datetime.fromtimestamp(
_start_time).isoformat(' ')
## Diagnostic outputs: where are we running?
