def test_start(self):
dbi = FakeDataBaseInterface(10)
class FakeAction(sch.Action):
def run_remote_task(self):
dbi.files[self.obs] = self.task
def all_done():
for f in dbi.files:
if dbi.get_obs_status(f) != 'COMPLETE':
return False
return True
task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT)
s = sch.Scheduler(task_clients,
self.wf,
nstills=1,
actions_per_still=1,
blocksize=10)
# myscheduler = StillScheduler(task_clients, wf,
# actions_per_still=ACTIONS_PER_STILL, blocksize=BLOCK_SIZE,
# nstills=len(STILLS)) # Init scheduler daemon
t = threading.Thread(target=s.start,
args=(dbi, FakeAction),
kwargs={'sleeptime': 0})
t.start()
tstart = time.time()
while not all_done() and time.time() - tstart < 1:
time.sleep(.1)
s.quit()
for f in dbi.files:
self.assertEqual(dbi.get_obs_status(f), 'COMPLETE')
def test_faulty(self):
for i in xrange(1):
dbi = FakeDataBaseInterface(10)
class FakeAction(sch.Action):
def __init__(self, f, task, neighbors, still, wf):
sch.Action.__init__(self, f, task, neighbors, still, wf, timeout=.01)
def run_remote_task(self):
if random.random() > .5:
dbi.files[self.obs] = self.task
def all_done():
for f in dbi.files:
if dbi.get_obs_status(f) != 'COMPLETE':
return False
return True
task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT)
s = sch.Scheduler(task_clients, self.wf, nstills=1, actions_per_still=1, blocksize=10)
t = threading.Thread(target=s.start, args=(dbi, FakeAction), kwargs={'sleeptime': 0})
t.start()
tstart = time.time()
while not all_done() and time.time() - tstart < 10:
# print s.launched_actions[0][0].obs, s.launched_actions[0][0].task
# print [(a.obs, a.task) for a in s.action_queue]
time.sleep(.1)
s.quit()
# for f in dbi.files:
# print f, dbi.files[f]
for f in dbi.files:
self.assertEqual(dbi.get_obs_status(f), 'COMPLETE')
def start_client(sg, wf, args):
#
# Instantiate a still client instance
#
if args.init is True: # See if we were told to initiate the database
sg.dbi.createdb()
print("Database has been initialized")
sys.exit(0)
try:
# Testing the database to make sure we made a connection, its fun..
if sg.dbi.test_db() is False:
print("Incorrect number of tables read from the database")
sys.exit(1)
except:
print("We could not run a test on the database and are aborting. Please check the DB config settings")
sys.exit(1)
print("My Log Path : %s") % sg.log_path
sg.logger = setup_logger("Scheduler", "DEBUG", sg.log_path)
task_clients = [TaskClient(sg.dbi, s, wf, sg.port, sg) for s in sg.hosts]
# Screw it going to just break a bunch of the unittest stuff and simplify
# the calling of the scheduler to take SpawnerClass
myscheduler = StillScheduler(task_clients, wf, sg) # Init scheduler daemon
myscheduler.start(dbi=sg.dbi, ActionClass=Action)
return 0
def setUp(self):
self.ntimes = 10
self.npols = 4
self.dbi = PopulatedDataBaseInterface(self.ntimes, self.npols, test=True)
self.files = self.dbi.list_observations()
self.sg = SpawnerClass()
self.sg.config_file = "still_test_paper.cfg"
self.wf = WorkFlow()
process_client_config_file(self.sg, self.wf)
self.task_clients = [TaskClient(self.dbi, 'localhost', self.wf, port=TEST_PORT)]
def setUp(self):
self.nfiles = 10
dbi = FakeDataBaseInterface(self.nfiles)
self.dbi = dbi
self.sg = SpawnerClass()
self.sg.config_file = "still_test_paper.cfg"
self.wf = WorkFlow()
process_client_config_file(self.sg, self.wf)
class FakeAction(sch.Action):
def run_remote_task(self):
dbi.files[self.filename] = self.task
self.FakeAction = FakeAction
self.task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT)
def find_all_taskmanagers(self):
###
# find_all_taskmanagers : Check the database for all available stills with status OK
# Should also remove stills that have gone offline.
###
logger.debug("looking for TaskManagers...")
self.stills = self.dbi.get_available_stills()
while len(self.stills) < 1:
logger.debug("Can't find any TaskManagers! Waiting for 10sec and trying again")
time.sleep(10)
self.stills = self.dbi.get_available_stills()
for still in self.stills:
if still.hostname not in self.task_clients:
logger.debug("Discovery of new TaskManager : %s" % still.hostname)
self.task_clients[still.hostname] = TaskClient(self.dbi, still.hostname, self.wf, still.port, self.sg)
self.launched_actions[still.hostname] = []
return