def test_process_tasks_of_one_base_class(self): scheduler = Scheduler() task1 = TestTask1() task2 = TestTask2() task3 = TestTask3() task4_1 = TestTask4() task4_2 = TestTask4() task2.add_preceding_task(task1) task4_1.add_preceding_tasks([task1, task2]) task3.add_preceding_task(task1) task4_2.add_preceding_tasks([task1, task3]) scheduler.add_task_to_queue(task1) scheduler.add_task_to_queue(task2) scheduler.add_task_to_queue(task3) scheduler.add_task_to_queue(task4_1) scheduler.add_task_to_queue(task4_2) scheduler.run_tasks() self.assertIn(task1, scheduler._result_map) self.assertIn(task2, scheduler._result_map) self.assertIn(task3, scheduler._result_map) self.assertIn(task4_1, scheduler._result_map) self.assertIn(task4_2, scheduler._result_map) self.assertGreater(scheduler._result_map[task2], scheduler._result_map[task1]) self.assertGreater(scheduler._result_map[task3], scheduler._result_map[task1]) self.assertGreater(scheduler._result_map[task4_1], scheduler._result_map[task2]) self.assertGreater(scheduler._result_map[task4_2], scheduler._result_map[task3])
async def coro_biphase( self, signals: SignalPairs, fs: float, f0: float, fr: Tuple[float, float], on_progress: Callable[[int, int], None], ) -> List[Tuple]: """ Calculates biphase and biamplitude. Used in "wavelet bispectrum analysis". :param signals: the signal pairs :param fs: the sampling frequency :param f0: the resolution :param fr: 'x' and 'y' frequencies :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler( progress_callback=on_progress, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) args = [(s1, s2, fs, f0, fr, s1.output_data.opt) for s1, s2 in signals.get_pairs()] return await self.scheduler.map(target=_biphase, args=args, process_type=mp.Process, queue_type=mp.Queue)
async def coro_phase_coherence( self, signals: SignalPairs, params: PCParams, on_progress: Callable[[int, int], None], ) -> List[tuple]: """ Performs wavelet phase coherence between signal pairs. Used in "wavelet phase coherence". :param signals: the pairs of signals :param params: the parameters which are used in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) for i in range(signals.pair_count()): pair = signals.get_pair_by_index(i) self.scheduler.add( target=_phase_coherence, args=(pair, params), subtasks=params.surr_count, process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def do_calculations(self): """ Does the calculations using a scheduler, and shows the output in the label. """ self.scheduler = Scheduler(progress_callback=self.on_progress) num_processes = 16 args = [] for _ in range(num_processes): sleep_time = random.randint(1, 8) # Add to list of arguments. Must be tuple. args.append((sleep_time, )) # Run all processes and `await` the results: an ordered list containing one int from each process. output: List[int] = await self.scheduler.map( target=long_calculation, args=args, ) # (If the scheduler was terminated before completion, we don't want the results). if not self.scheduler.terminated: text = ", ".join([str(i) for i in output]) self.label.setText(f"Output: {text}") self.button.setText("Start")
async def coro_transform( self, params: TFParams, on_progress: Callable[[int, int], None]) -> List[tuple]: """ Performs a wavelet transform or windowed Fourier transform of signals. Used in "time-frequency analysis". :param params: the parameters which are used in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) signals: Signals = params.signals params.remove_signals( ) # Don't want to pass large unneeded object to other process. for time_series in signals: self.scheduler.add( target=_time_frequency, args=(time_series, params), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_bandpass_filter( self, signals: Signals, intervals: tuple, on_progress: Callable[[int, int], None], ) -> List[tuple]: """ Performs bandpass filter on signals. Used in "ridge extraction and filtering". :param signals: the signals :param intervals: the intervals to calculate bandpass filter on :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) for s in signals: fs = s.frequency for i in range(len(intervals)): fmin, fmax = intervals[i] self.scheduler.add( target=_bandpass_filter, args=(s, fmin, fmax, fs), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_ridge_extraction( self, params: REParams, on_progress: Callable[[int, int], None]) -> List[tuple]: """ Performs ridge extraction on wavelet transforms. Used in "ridge extraction and filtering". :param params: the parameters which are used in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) signals = params.signals num_transforms = len(signals) intervals = params.intervals for i in range(num_transforms): for j in range(len(intervals)): fmin, fmax = intervals[j] params.set_item(_fmin, fmin) params.set_item(_fmax, fmax) self.scheduler.add( target=_ridge_extraction, args=(signals[i], params), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_preprocess(self, signals: Union[TimeSeries, List[TimeSeries]], fmin: float, fmax: float) -> List[ndarray]: """ Performs preprocessing on a single signal. :param signals: the signal or signals to perform pre-processing on :param fmin: the minimum frequency :param fmax: the maximum frequency :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler( run_in_thread=True, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) if isinstance(signals, TimeSeries): signals = [signals] args = [(s.signal, s.frequency, fmin, fmax) for s in signals] return await self.scheduler.map( target=pymodalib.preprocess, args=args, process_type=mp.Process, queue_type=mp.Queue, )
async def coro_bayesian( self, signals: SignalPairs, paramsets: List[ParamSet], on_progress: Callable[[int, int], None], ) -> List[tuple]: """ Performs Bayesian inference on signal pairs. Used in "dynamical Bayesian inference". :param signals: the signals :param paramsets: the parameter sets to use in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) for params in paramsets: for pair in signals.get_pairs(): self.scheduler.add( target=_dynamic_bayesian_inference, args=(*pair, params), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_phase_coherence( self, signals: SignalPairs, params: PCParams, on_progress: Callable[[int, int], None], ) -> List[Tuple]: """ Performs wavelet phase coherence between signal pairs. Used in "wavelet phase coherence". :param signals: the pairs of signals :param params: the parameters which are used in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler( progress_callback=on_progress, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) return await self.scheduler.map( target=_phase_coherence, args=[(pair, params) for pair in signals.get_pairs()], subtasks=params.surr_count, process_type=mp.Process, queue_type=mp.Queue, )
async def coro_get_plugins(self): self.scheduler = Scheduler() self.scheduler.add(target=online.find_online_plugins) self.tbl1: QTableView = self.tbl_trusted self.tbl2: QTableView = self.tbl_community self.btn_apply.clicked.connect(self.on_apply_clicked) self.model1 = QStandardItemModel() self.model2 = QStandardItemModel() for m in (self.model1, self.model2): m.itemChanged.connect(self.on_item_changed) m.setHorizontalHeaderLabels(["Install status", "Game", "Author"]) for t, m in zip([self.tbl1, self.tbl2], [self.model1, self.model2]): t.verticalHeader().setVisible(False) t.setModel(m) trusted, community = (await self.scheduler.run())[0] self.add_items(self.model1, trusted) self.add_items(self.model2, community) self.add_dev_items(self.model2, self.installed) self.tbl1.resizeColumnsToContents() self.tbl2.resizeColumnsToContents()
async def coro_bispectrum_analysis( self, signals: SignalPairs, params: BAParams, on_progress: Callable[[int, int], None], ) -> List[tuple]: """ Performs wavelet bispectrum analysis on signal pairs. Used in "wavelet bispectrum analysis". :param signals: the signal pairs :param params: the parameters to use in the algorithm :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) for pair in signals.get_pairs(): self.scheduler.add( target=_bispectrum_analysis, args=(*pair, params), subtasks=4, process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_biphase( self, signals: SignalPairs, fs: float, f0: float, fr: Tuple[float, float], on_progress: Callable[[int, int], None], ) -> List[tuple]: """ Calculates biphase and biamplitude. Used in "wavelet bispectrum analysis". :param signals: the signal pairs :param fs: the sampling frequency :param f0: the resolution :param fr: 'x' and 'y' frequencies :param on_progress: progress callback :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler(progress_callback=on_progress) for pair in signals.get_pairs(): opt = pair[0].output_data.opt self.scheduler.add( target=_biphase, args=(*pair, fs, f0, fr, opt), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
async def coro_group_coherence(self, sig1a: ndarray, sig1b: ndarray, fs: float, percentile: Optional[float], on_progress: Callable[[int, int], None], *args, **kwargs) -> List[Tuple]: """ Calculates group coherence. Parameters ---------- sig1a : ndarray The set of signals A for group 1. sig1b : ndarray The set of signals B for group 1. fs : float The sampling frequency of the signals. percentile : Optional[float] The percentile at which the surrogates will be subtracted. on_progress : Callable Function called to report progress. args Arguments to pass to the wavelet transform. kwargs Keyword arguments to pass to the wavelet transform. Returns ------- freq : ndarray [1D array] The frequencies. coh1 : ndarray [2D array] The residual coherence for group 1. surr1 : ndarray [3D array] The surrogates for group 1. """ self.stop() self.scheduler = Scheduler( progress_callback=on_progress, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) return await self.scheduler.map( target=functools.partial( pymodalib.group_coherence, sig1a, sig1b, fs, percentile, True, *args, **kwargs, ), args=[ tuple(), ], )
async def coro_statistical_test( self, freq: ndarray, coh1: ndarray, coh2: ndarray, bands: List[Tuple[float, float]], on_progress: Callable[[int, int], None], ) -> Dict[Tuple[float, float], float]: """ Performs a statistical test on the results of group phase coherence, to check for significance. Parameters ---------- freq : ndarray [1D array] The frequencies from group coherence. coh1 : ndarray [2D array] The coherence of the first group. coh2 : ndarray [2D array] The coherence of the second group. bands : List[Tuple[float,float]] List containing the frequency bands which will be tested for significance. on_progress : Callable Function called to report progress. Returns ------- pvalues : Dict[Tuple[float, float], float] A list containing the p-values for each frequency band. """ self.stop() self.scheduler = Scheduler( run_in_thread=self.should_run_in_thread, progress_callback=on_progress, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) from pymodalib.algorithms.group_coherence import statistical_test results = (await self.scheduler.map( target=statistical_test, args=[( freq, coh1, coh2, bands, )], ))[0] return dict(zip(bands, results))
def test_add_task(): scheduler = Scheduler() scheduler.start() assert scheduler.tasks.qsize() == 0 scheduler.add(generate_task()) assert scheduler.tasks.qsize() == 1 scheduler.stop()
async def test_async_task(self): logging.config.fileConfig(fname=os.path.join(os.path.dirname( __file__), '..', '..', 'src', 'config', 'logging.conf')) logging.info('Starting') dpm = DependencyManager() t1 = LongLastingTask('t1') t2 = LongLastingTask('t2') t3 = LongLastingTask('t3') dp1: Dependency = await dpm.create_dependency('dep_1', t1) dp2: Dependency = await dpm.create_dependency('dep_2', t2) dp3: Dependency = await dpm.create_dependency('dep_3', t3) assert len(await dpm.get_tasks_in_dependency('dep_1')) == 0 t11 = LongLastingTask('t11') t12 = LongLastingTask('t12') t13 = LongLastingTask('t13') await dpm.add_task_to_dependency('dep_1', t11) await dpm.add_task_to_dependency('dep_1', t12) await dpm.add_task_to_dependency('dep_1', t13) assert len(await dpm.get_tasks_in_dependency('dep_1')) == 3 t21 = LongLastingTask('t21') t22 = LongLastingTask('t22') t23 = LongLastingTask('t23') await dpm.add_task_to_dependency('dep_2', t21) await dpm.add_task_to_dependency('dep_2', t11) await dpm.add_task_to_dependency('dep_2', t12) await dpm.add_task_to_dependency('dep_2', t22) await dpm.add_task_to_dependency('dep_2', t23) t31 = LongLastingTask('t31') t32 = LongLastingTask('t32') await dpm.add_task_to_dependency('dep_3', t31) await dpm.add_task_to_dependency('dep_3', t32) t4 = LongLastingTask('t4') dp4: Dependency = await dpm.create_dependency('dep_4', t4) await dpm.add_task_to_dependency('dep_4', t31) await dpm.add_task_to_dependency('dep_4', t1) deps = await t1.resolve_dependencies() assert len(deps) == 3, f'acquired {len(deps)} instead of 3' deps = await dpm.get_tasks_in_dependency('dep_4') assert len(deps) == 5, f'acquired {len(deps)} instead of 5' sch = Scheduler(dpm) await sch.add_task(t1) await sch.add_task(t2) await sch.add_task(t4) await sch.start() results = await sch.collect_results_for_task(t1) logging.info(f'Results: {results}')
def test_scheduler_clear_queue(self): scheduler = Scheduler() task1 = TestTask1() scheduler.add_task_to_queue(task1) scheduler.clear_queue() self.assertListEqual(scheduler.get_queue(), list())
def fifo(self): ''' @return: a FIFO kernel ''' self.imanager = InterruptorManager() self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu) self.queue = OwnQueue(self.condition) self.scheduler = Scheduler(self.cpu, self.queue, -1, self.condition) return self.__build(self.queue, self.scheduler)
def main(spark): scheduler = Scheduler() get_recipes_task = GetRecipes(spark) find_meat_task = FindMeatRecipes(spark) find_meat_task.add_preceding_task(get_recipes_task) scheduler.add_task_to_queue(get_recipes_task) scheduler.add_task_to_queue(find_meat_task) scheduler.run_tasks()
def withPriority(self): ''' @return: a Priority kernel ''' self.imanager = InterruptorManager() self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu) self.func = lambda pcb1, pcb2: pcb1.getPriority() >= pcb2.getPriority() self.queue = OwnHeap(self.condition, self.func) self.scheduler = Scheduler(self.cpu, self.queue, -1, self.condition) return self.__build(self.queue, self.scheduler)
def roundRobin(self, quantum): ''' @return: given a <quantum>, returns a Round Robbin configured kernel, with quantum <quantum> ''' self.imanager = InterruptorManager() self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu) self.queue = OwnQueue(self.condition) self.scheduler = Scheduler(self.cpu, self.queue, quantum, self.condition) return self.__build(self.queue, self.scheduler)
async def get_latest_version(self): self.scheduler = Scheduler() releases = (await self.scheduler.map(target=online.get_switcher_releases, args=[()]))[0] self.prefs.last_update_check = time.time() self.prefs.commit() return releases[0]
def main(): # print('problem generation...') # prob = Problem.Random(4, 25) # print('saving problem data...') # prob.Save() print("loading problem data...") prob = Problem.Load() print("initializing scheduler...") sched = Scheduler(20, prob) # print('generating random solution...') # sched.solution = sched.RandomSolution() print("solving...") sched.solution = sched.Solve(iterations=5000) print("Cmax:") print(sched.Fitness(sched.solution)) print("saving...") sched.SaveGraphData() print("calling gui...") main_data.main()
async def coro_preprocess(self, signal: TimeSeries, fmin: float, fmax: float) -> List[Tuple]: """ Performs preprocessing on a single signal. :param signal: the signal as a 1D array :param fmin: the minimum frequency :param fmax: the maximum frequency :return: list containing the output from each process """ self.stop() self.scheduler = Scheduler() self.scheduler.add( target=_preprocess, args=(signal.signal, signal.frequency, fmin, fmax), process_type=mp.Process, queue_type=mp.Queue, ) return await self.scheduler.run()
def test_runner(): scheduler = Scheduler(interval=1) runner = scheduler.run scheduler.run = Mock() scheduler.start() sleep(2) scheduler.stop() assert 1 <= scheduler.run.call_count <= 2 scheduler.run = runner
def roundRobin_withPriority(self, quantum): ''' @return: given a <quantum> it returns a Priority-RR kernel with quantum <quantum> ''' self.imanager = InterruptorManager() self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu) self.func = lambda pcb1, pcb2: pcb1.getPriority() >= pcb2.getPriority() self.queue = OwnHeap(self.condition, self.func) self.scheduler = Scheduler(self.cpu, self.queue, quantum, self.condition) return self.__build(self.queue, self.scheduler)
def solve_problem(self): ready_to_solve = QtGui.QMessageBox.Yes if not self.new: ready_to_solve = QtGui.QMessageBox.question(self, 'Solver', 'You are going to generate solution for the previously generated problem, are you sure?', QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.No) if ready_to_solve == QtGui.QMessageBox.Yes : mutop_operator = self.mutop_combo.currentIndex() xop_operator = self.xop_combo.currentIndex() self.statusBar.setText('Loading problem data...') prob = Problem.Load() pop_size = int(self.population_edit.displayText()) self.statusBar.setText('Initializing scheduler...') sched = Scheduler(pop_size, prob) self.statusBar.setText('Solving...') iter_num = int(self.iter_edit.displayText()) test_q = int(self.iter_edit.displayText()) cross_pr = float(self.crossp_edit.displayText()) parent_f = float(self.parents_edit.displayText()) sched.SetOperators(mutationOp = mutop_operator, crossingOp = xop_operator) sched.setSelectionParams(parentsInNewPop = parent_f, mutationProb = cross_pr) sched.Trials = int(self.test_edit.displayText()) sched.solution = sched.Solve(iterations = iter_num) self.statusBar.setText('Saving...') sched.SaveGraphData() self.statusBar.setText('Calling graph gui...') self.DataAnalysisDialog = DataAnalysis() self.DataAnalysisDialog.create() self.DataAnalysisDialog.show() self.statusBar.setText('Solved!')
def main(): # print('problem generation...') # prob = Problem.Random(4, 25) # print('saving problem data...') # prob.Save() print('loading problem data...') prob = Problem.Load() print('initializing scheduler...') popSize = 20 sched = Scheduler(popSize, prob) # print('generating random solution...') # sched.solution = sched.RandomSolution() sched.setSelectionParams() n = 10 print('solving...') for u in range(2,10): averIters = 0 averCmax = 0 sched.setSelectionParams(u) print('---------------------------------------') print('ParentsInNewPop = {0}'.format(u)) for i in range(n): sched.solution = sched.Solve(iterations = 5000) cmax = sched.Fitness(sched.solution) averCmax += cmax/n iters = sched.lastIters averIters += iters/n print('Try {0}: Algorithm took {1} iterations'.format(i, iters) + ' with Cmax: {0}'.format(cmax)) print('In {0} tries, algorithm took on average {1} iterations'.format(n, round(averIters))) print('with average Cmax: {0}'.format(averCmax)) print('saving...') # sched.SaveGraphData() print('calling gui...')
def test_process_one_task(self): scheduler = Scheduler() task1 = TestTask1() scheduler.add_task_to_queue(task1) scheduler.run_tasks() self.assertIn(task1, scheduler._result_map) self.assertIsNotNone(scheduler._result_map[task1])
def test_custom_validator(mock_print): def test_validator(job): job() return SchedulerPolicies.RETRY scheduler = Scheduler(validator=test_validator, interval=1) print_task = Task(mock_print, Priorities.MEDIUM) scheduler.add(print_task) scheduler.start() sleep(3) scheduler.stop() assert 2 <= mock_print.call_count <= 3
def test_process_task_with_not_implemented_process(self): scheduler = Scheduler() task5 = TestTask5() scheduler.add_task_to_queue(task5) with self.assertRaises(Exception) as exception_context: scheduler.run_tasks() self.assertEqual(str(exception_context.exception), 'Method process of task {} did not implemented' .format(task5.get_name()))
async def coro_harmonics( self, signals: Signals, params: DHParams, preprocess: bool, on_progress: Callable[[int, int], None], ) -> List[Tuple]: """ Detects harmonics in signals. :param signals: the signals :param params: the parameters to pass to the harmonic finder :param preprocess: whether to perform pre-processing on the signals :param on_progress: the progress callback :return: list containing the output from each process """ # Whether to parallelize the algorithm for each calculation. parallel = len(signals) < Scheduler.optimal_process_count() self.stop() self.scheduler = Scheduler( progress_callback=on_progress, raise_exceptions=True, capture_stdout=True, only_threads=self.only_threads, ) args = [( preprocess, sig.signal, params, *params.args(), parallel, params.crop, ) for sig in signals] return await self.scheduler.map(target=harmonic_wrapper, args=args)
def __init__(self, cpu, politica): Scheduler.__init__(self, cpu, politica) self.queue = Queue.PriorityQueue()
def __init__(self,queuelist): Scheduler.__init__(self,queuelist)
def test_initial_returns_population(self): sched = Scheduler(10) pop = sched.initial() self.assertHasPopulationType(pop)