def test_start(self): dbi = FakeDataBaseInterface(10) class FakeAction(sch.Action): def run_remote_task(self): dbi.files[self.obs] = self.task def all_done(): for f in dbi.files: if dbi.get_obs_status(f) != 'COMPLETE': return False return True task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT) s = sch.Scheduler(task_clients, self.wf, nstills=1, actions_per_still=1, blocksize=10) # myscheduler = StillScheduler(task_clients, wf, # actions_per_still=ACTIONS_PER_STILL, blocksize=BLOCK_SIZE, # nstills=len(STILLS)) # Init scheduler daemon t = threading.Thread(target=s.start, args=(dbi, FakeAction), kwargs={'sleeptime': 0}) t.start() tstart = time.time() while not all_done() and time.time() - tstart < 1: time.sleep(.1) s.quit() for f in dbi.files: self.assertEqual(dbi.get_obs_status(f), 'COMPLETE')
def test_faulty(self): for i in xrange(1): dbi = FakeDataBaseInterface(10) class FakeAction(sch.Action): def __init__(self, f, task, neighbors, still, wf): sch.Action.__init__(self, f, task, neighbors, still, wf, timeout=.01) def run_remote_task(self): if random.random() > .5: dbi.files[self.obs] = self.task def all_done(): for f in dbi.files: if dbi.get_obs_status(f) != 'COMPLETE': return False return True task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT) s = sch.Scheduler(task_clients, self.wf, nstills=1, actions_per_still=1, blocksize=10) t = threading.Thread(target=s.start, args=(dbi, FakeAction), kwargs={'sleeptime': 0}) t.start() tstart = time.time() while not all_done() and time.time() - tstart < 10: # print s.launched_actions[0][0].obs, s.launched_actions[0][0].task # print [(a.obs, a.task) for a in s.action_queue] time.sleep(.1) s.quit() # for f in dbi.files: # print f, dbi.files[f] for f in dbi.files: self.assertEqual(dbi.get_obs_status(f), 'COMPLETE')
def start_client(sg, wf, args): # # Instantiate a still client instance # if args.init is True: # See if we were told to initiate the database sg.dbi.createdb() print("Database has been initialized") sys.exit(0) try: # Testing the database to make sure we made a connection, its fun.. if sg.dbi.test_db() is False: print("Incorrect number of tables read from the database") sys.exit(1) except: print("We could not run a test on the database and are aborting. Please check the DB config settings") sys.exit(1) print("My Log Path : %s") % sg.log_path sg.logger = setup_logger("Scheduler", "DEBUG", sg.log_path) task_clients = [TaskClient(sg.dbi, s, wf, sg.port, sg) for s in sg.hosts] # Screw it going to just break a bunch of the unittest stuff and simplify # the calling of the scheduler to take SpawnerClass myscheduler = StillScheduler(task_clients, wf, sg) # Init scheduler daemon myscheduler.start(dbi=sg.dbi, ActionClass=Action) return 0
def setUp(self): self.ntimes = 10 self.npols = 4 self.dbi = PopulatedDataBaseInterface(self.ntimes, self.npols, test=True) self.files = self.dbi.list_observations() self.sg = SpawnerClass() self.sg.config_file = "still_test_paper.cfg" self.wf = WorkFlow() process_client_config_file(self.sg, self.wf) self.task_clients = [TaskClient(self.dbi, 'localhost', self.wf, port=TEST_PORT)]
def setUp(self): self.nfiles = 10 dbi = FakeDataBaseInterface(self.nfiles) self.dbi = dbi self.sg = SpawnerClass() self.sg.config_file = "still_test_paper.cfg" self.wf = WorkFlow() process_client_config_file(self.sg, self.wf) class FakeAction(sch.Action): def run_remote_task(self): dbi.files[self.filename] = self.task self.FakeAction = FakeAction self.task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT)
def find_all_taskmanagers(self): ### # find_all_taskmanagers : Check the database for all available stills with status OK # Should also remove stills that have gone offline. ### logger.debug("looking for TaskManagers...") self.stills = self.dbi.get_available_stills() while len(self.stills) < 1: logger.debug("Can't find any TaskManagers! Waiting for 10sec and trying again") time.sleep(10) self.stills = self.dbi.get_available_stills() for still in self.stills: if still.hostname not in self.task_clients: logger.debug("Discovery of new TaskManager : %s" % still.hostname) self.task_clients[still.hostname] = TaskClient(self.dbi, still.hostname, self.wf, still.port, self.sg) self.launched_actions[still.hostname] = [] return