def test_scheduler_clear_queue(self):
scheduler = Scheduler()
task1 = TestTask1()
scheduler.add_task_to_queue(task1)
scheduler.clear_queue()
self.assertListEqual(scheduler.get_queue(), list())
async def coro_transform(
self, params: TFParams,
on_progress: Callable[[int, int], None]) -> List[tuple]:
"""
Performs a wavelet transform or windowed Fourier transform of signals.
Used in "time-frequency analysis".
:param params: the parameters which are used in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
signals: Signals = params.signals
params.remove_signals(
) # Don't want to pass large unneeded object to other process.
for time_series in signals:
self.scheduler.add(
target=_time_frequency,
args=(time_series, params),
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
def test_process_tasks_of_one_base_class(self):
scheduler = Scheduler()
task1 = TestTask1()
task2 = TestTask2()
task3 = TestTask3()
task4_1 = TestTask4()
task4_2 = TestTask4()
task2.add_preceding_task(task1)
task4_1.add_preceding_tasks([task1, task2])
task3.add_preceding_task(task1)
task4_2.add_preceding_tasks([task1, task3])
scheduler.add_task_to_queue(task1)
scheduler.add_task_to_queue(task2)
scheduler.add_task_to_queue(task3)
scheduler.add_task_to_queue(task4_1)
scheduler.add_task_to_queue(task4_2)
scheduler.run_tasks()
self.assertIn(task1, scheduler._result_map)
self.assertIn(task2, scheduler._result_map)
self.assertIn(task3, scheduler._result_map)
self.assertIn(task4_1, scheduler._result_map)
self.assertIn(task4_2, scheduler._result_map)
self.assertGreater(scheduler._result_map[task2], scheduler._result_map[task1])
self.assertGreater(scheduler._result_map[task3], scheduler._result_map[task1])
self.assertGreater(scheduler._result_map[task4_1], scheduler._result_map[task2])
self.assertGreater(scheduler._result_map[task4_2], scheduler._result_map[task3])
async def coro_phase_coherence(
self,
signals: SignalPairs,
params: PCParams,
on_progress: Callable[[int, int], None],
) -> List[Tuple]:
"""
Performs wavelet phase coherence between signal pairs. Used in "wavelet phase coherence".
:param signals: the pairs of signals
:param params: the parameters which are used in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(
progress_callback=on_progress,
raise_exceptions=True,
capture_stdout=True,
only_threads=self.only_threads,
)
return await self.scheduler.map(
target=_phase_coherence,
args=[(pair, params) for pair in signals.get_pairs()],
subtasks=params.surr_count,
process_type=mp.Process,
queue_type=mp.Queue,
)
async def do_calculations(self):
"""
Does the calculations using a scheduler, and shows the output in the label.
"""
self.scheduler = Scheduler(progress_callback=self.on_progress)
num_processes = 16
args = []
for _ in range(num_processes):
sleep_time = random.randint(1, 8)
# Add to list of arguments. Must be tuple.
args.append((sleep_time, ))
# Run all processes and `await` the results: an ordered list containing one int from each process.
output: List[int] = await self.scheduler.map(
target=long_calculation,
args=args,
)
# (If the scheduler was terminated before completion, we don't want the results).
if not self.scheduler.terminated:
text = ", ".join([str(i) for i in output])
self.label.setText(f"Output: {text}")
self.button.setText("Start")
async def coro_phase_coherence(
self,
signals: SignalPairs,
params: PCParams,
on_progress: Callable[[int, int], None],
) -> List[tuple]:
"""
Performs wavelet phase coherence between signal pairs. Used in "wavelet phase coherence".
:param signals: the pairs of signals
:param params: the parameters which are used in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
for i in range(signals.pair_count()):
pair = signals.get_pair_by_index(i)
self.scheduler.add(
target=_phase_coherence,
args=(pair, params),
subtasks=params.surr_count,
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_bandpass_filter(
self,
signals: Signals,
intervals: tuple,
on_progress: Callable[[int, int], None],
) -> List[tuple]:
"""
Performs bandpass filter on signals. Used in "ridge extraction and filtering".
:param signals: the signals
:param intervals: the intervals to calculate bandpass filter on
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
for s in signals:
fs = s.frequency
for i in range(len(intervals)):
fmin, fmax = intervals[i]
self.scheduler.add(
target=_bandpass_filter,
args=(s, fmin, fmax, fs),
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_ridge_extraction(
self, params: REParams,
on_progress: Callable[[int, int], None]) -> List[tuple]:
"""
Performs ridge extraction on wavelet transforms. Used in "ridge extraction and filtering".
:param params: the parameters which are used in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
signals = params.signals
num_transforms = len(signals)
intervals = params.intervals
for i in range(num_transforms):
for j in range(len(intervals)):
fmin, fmax = intervals[j]
params.set_item(_fmin, fmin)
params.set_item(_fmax, fmax)
self.scheduler.add(
target=_ridge_extraction,
args=(signals[i], params),
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_bispectrum_analysis(
self,
signals: SignalPairs,
params: BAParams,
on_progress: Callable[[int, int], None],
) -> List[tuple]:
"""
Performs wavelet bispectrum analysis on signal pairs.
Used in "wavelet bispectrum analysis".
:param signals: the signal pairs
:param params: the parameters to use in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
for pair in signals.get_pairs():
self.scheduler.add(
target=_bispectrum_analysis,
args=(*pair, params),
subtasks=4,
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_bayesian(
self,
signals: SignalPairs,
paramsets: List[ParamSet],
on_progress: Callable[[int, int], None],
) -> List[tuple]:
"""
Performs Bayesian inference on signal pairs. Used in "dynamical Bayesian inference".
:param signals: the signals
:param paramsets: the parameter sets to use in the algorithm
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
for params in paramsets:
for pair in signals.get_pairs():
self.scheduler.add(
target=_dynamic_bayesian_inference,
args=(*pair, params),
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_biphase(
self,
signals: SignalPairs,
fs: float,
f0: float,
fr: Tuple[float, float],
on_progress: Callable[[int, int], None],
) -> List[tuple]:
"""
Calculates biphase and biamplitude. Used in "wavelet bispectrum analysis".
:param signals: the signal pairs
:param fs: the sampling frequency
:param f0: the resolution
:param fr: 'x' and 'y' frequencies
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(progress_callback=on_progress)
for pair in signals.get_pairs():
opt = pair[0].output_data.opt
self.scheduler.add(
target=_biphase,
args=(*pair, fs, f0, fr, opt),
process_type=mp.Process,
queue_type=mp.Queue,
)
return await self.scheduler.run()
async def coro_preprocess(self, signals: Union[TimeSeries,
List[TimeSeries]],
fmin: float, fmax: float) -> List[ndarray]:
"""
Performs preprocessing on a single signal.
:param signals: the signal or signals to perform pre-processing on
:param fmin: the minimum frequency
:param fmax: the maximum frequency
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(
run_in_thread=True,
raise_exceptions=True,
capture_stdout=True,
only_threads=self.only_threads,
)
if isinstance(signals, TimeSeries):
signals = [signals]
args = [(s.signal, s.frequency, fmin, fmax) for s in signals]
return await self.scheduler.map(
target=pymodalib.preprocess,
args=args,
process_type=mp.Process,
queue_type=mp.Queue,
)
async def coro_biphase(
self,
signals: SignalPairs,
fs: float,
f0: float,
fr: Tuple[float, float],
on_progress: Callable[[int, int], None],
) -> List[Tuple]:
"""
Calculates biphase and biamplitude. Used in "wavelet bispectrum analysis".
:param signals: the signal pairs
:param fs: the sampling frequency
:param f0: the resolution
:param fr: 'x' and 'y' frequencies
:param on_progress: progress callback
:return: list containing the output from each process
"""
self.stop()
self.scheduler = Scheduler(
progress_callback=on_progress,
raise_exceptions=True,
capture_stdout=True,
only_threads=self.only_threads,
)
args = [(s1, s2, fs, f0, fr, s1.output_data.opt)
for s1, s2 in signals.get_pairs()]
return await self.scheduler.map(target=_biphase,
args=args,
process_type=mp.Process,
queue_type=mp.Queue)
async def coro_get_plugins(self):
self.scheduler = Scheduler()
self.scheduler.add(target=online.find_online_plugins)
self.tbl1: QTableView = self.tbl_trusted
self.tbl2: QTableView = self.tbl_community
self.btn_apply.clicked.connect(self.on_apply_clicked)
self.model1 = QStandardItemModel()
self.model2 = QStandardItemModel()
for m in (self.model1, self.model2):
m.itemChanged.connect(self.on_item_changed)
m.setHorizontalHeaderLabels(["Install status", "Game", "Author"])
for t, m in zip([self.tbl1, self.tbl2], [self.model1, self.model2]):
t.verticalHeader().setVisible(False)
t.setModel(m)
trusted, community = (await self.scheduler.run())[0]
self.add_items(self.model1, trusted)
self.add_items(self.model2, community)
self.add_dev_items(self.model2, self.installed)
self.tbl1.resizeColumnsToContents()
self.tbl2.resizeColumnsToContents()
async def test_async_task(self):
logging.config.fileConfig(fname=os.path.join(os.path.dirname(
__file__), '..', '..', 'src', 'config', 'logging.conf'))
logging.info('Starting')
dpm = DependencyManager()
t1 = LongLastingTask('t1')
t2 = LongLastingTask('t2')
t3 = LongLastingTask('t3')
dp1: Dependency = await dpm.create_dependency('dep_1', t1)
dp2: Dependency = await dpm.create_dependency('dep_2', t2)
dp3: Dependency = await dpm.create_dependency('dep_3', t3)
assert len(await dpm.get_tasks_in_dependency('dep_1')) == 0
t11 = LongLastingTask('t11')
t12 = LongLastingTask('t12')
t13 = LongLastingTask('t13')
await dpm.add_task_to_dependency('dep_1', t11)
await dpm.add_task_to_dependency('dep_1', t12)
await dpm.add_task_to_dependency('dep_1', t13)
assert len(await dpm.get_tasks_in_dependency('dep_1')) == 3
t21 = LongLastingTask('t21')
t22 = LongLastingTask('t22')
t23 = LongLastingTask('t23')
await dpm.add_task_to_dependency('dep_2', t21)
await dpm.add_task_to_dependency('dep_2', t11)
await dpm.add_task_to_dependency('dep_2', t12)
await dpm.add_task_to_dependency('dep_2', t22)
await dpm.add_task_to_dependency('dep_2', t23)
t31 = LongLastingTask('t31')
t32 = LongLastingTask('t32')
await dpm.add_task_to_dependency('dep_3', t31)
await dpm.add_task_to_dependency('dep_3', t32)
t4 = LongLastingTask('t4')
dp4: Dependency = await dpm.create_dependency('dep_4', t4)
await dpm.add_task_to_dependency('dep_4', t31)
await dpm.add_task_to_dependency('dep_4', t1)
deps = await t1.resolve_dependencies()
assert len(deps) == 3, f'acquired {len(deps)} instead of 3'
deps = await dpm.get_tasks_in_dependency('dep_4')
assert len(deps) == 5, f'acquired {len(deps)} instead of 5'
sch = Scheduler(dpm)
await sch.add_task(t1)
await sch.add_task(t2)
await sch.add_task(t4)
await sch.start()
results = await sch.collect_results_for_task(t1)
logging.info(f'Results: {results}')
def test_process_one_task(self):
scheduler = Scheduler()
task1 = TestTask1()
scheduler.add_task_to_queue(task1)
scheduler.run_tasks()
self.assertIn(task1, scheduler._result_map)
self.assertIsNotNone(scheduler._result_map[task1])
async def coro_group_coherence(self, sig1a: ndarray, sig1b: ndarray,
fs: float, percentile: Optional[float],
on_progress: Callable[[int, int], None],
*args, **kwargs) -> List[Tuple]:
"""
Calculates group coherence.
Parameters
----------
sig1a : ndarray
The set of signals A for group 1.
sig1b : ndarray
The set of signals B for group 1.
fs : float
The sampling frequency of the signals.
percentile : Optional[float]
The percentile at which the surrogates will be subtracted.
on_progress : Callable
Function called to report progress.
args
Arguments to pass to the wavelet transform.
kwargs
Keyword arguments to pass to the wavelet transform.
Returns
-------
freq : ndarray
[1D array] The frequencies.
coh1 : ndarray
[2D array] The residual coherence for group 1.
surr1 : ndarray
[3D array] The surrogates for group 1.
"""
self.stop()
self.scheduler = Scheduler(
progress_callback=on_progress,
raise_exceptions=True,
capture_stdout=True,
only_threads=self.only_threads,
)
return await self.scheduler.map(
target=functools.partial(
pymodalib.group_coherence,
sig1a,
sig1b,
fs,
percentile,
True,
*args,
**kwargs,
),
args=[
tuple(),
],
)
def test_add_task():
scheduler = Scheduler()
scheduler.start()
assert scheduler.tasks.qsize() == 0
scheduler.add(generate_task())
assert scheduler.tasks.qsize() == 1
scheduler.stop()
def test_process_task_with_not_implemented_process(self):
scheduler = Scheduler()
task5 = TestTask5()
scheduler.add_task_to_queue(task5)
with self.assertRaises(Exception) as exception_context:
scheduler.run_tasks()
self.assertEqual(str(exception_context.exception), 'Method process of task {} did not implemented'
.format(task5.get_name()))
def fifo(self):
'''
@return: a FIFO kernel
'''
self.imanager = InterruptorManager()
self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu)
self.queue = OwnQueue(self.condition)
self.scheduler = Scheduler(self.cpu, self.queue, -1, self.condition)
return self.__build(self.queue, self.scheduler)
async def get_latest_version(self):
self.scheduler = Scheduler()
releases = (await
self.scheduler.map(target=online.get_switcher_releases,
args=[()]))[0]
self.prefs.last_update_check = time.time()
self.prefs.commit()
return releases[0]
def withPriority(self):
'''
@return: a Priority kernel
'''
self.imanager = InterruptorManager()
self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu)
self.func = lambda pcb1, pcb2: pcb1.getPriority() >= pcb2.getPriority()
self.queue = OwnHeap(self.condition, self.func)
self.scheduler = Scheduler(self.cpu, self.queue, -1, self.condition)
return self.__build(self.queue, self.scheduler)
def roundRobin(self, quantum):
'''
@return: given a <quantum>, returns a Round Robbin configured kernel, with quantum <quantum>
'''
self.imanager = InterruptorManager()
self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu)
self.queue = OwnQueue(self.condition)
self.scheduler = Scheduler(self.cpu, self.queue, quantum,
self.condition)
return self.__build(self.queue, self.scheduler)
def test_process_task_with_no_preceding_started(self):
scheduler = Scheduler()
task1 = TestTask1()
task2 = TestTask2()
task2.add_preceding_task(task1)
scheduler.add_task_to_queue(task2)
with self.assertRaises(Exception) as exception_context:
scheduler.run_tasks()
self.assertEqual(str(exception_context.exception),
'Some tasks were not executed due to incorrect execution graph')
def roundRobin_withPriority(self, quantum):
'''
@return: given a <quantum> it returns a Priority-RR kernel with quantum <quantum>
'''
self.imanager = InterruptorManager()
self.cpu = CPU(self.ram, self.imanager, self.condition, self.mmu)
self.func = lambda pcb1, pcb2: pcb1.getPriority() >= pcb2.getPriority()
self.queue = OwnHeap(self.condition, self.func)
self.scheduler = Scheduler(self.cpu, self.queue, quantum,
self.condition)
return self.__build(self.queue, self.scheduler)
def main(spark):
scheduler = Scheduler()
get_recipes_task = GetRecipes(spark)
find_meat_task = FindMeatRecipes(spark)
find_meat_task.add_preceding_task(get_recipes_task)
scheduler.add_task_to_queue(get_recipes_task)
scheduler.add_task_to_queue(find_meat_task)
scheduler.run_tasks()
def test_runner():
scheduler = Scheduler(interval=1)
runner = scheduler.run
scheduler.run = Mock()
scheduler.start()
sleep(2)
scheduler.stop()
assert 1 <= scheduler.run.call_count <= 2
scheduler.run = runner
async def coro_statistical_test(
self,
freq: ndarray,
coh1: ndarray,
coh2: ndarray,
bands: List[Tuple[float, float]],
on_progress: Callable[[int, int], None],
) -> Dict[Tuple[float, float], float]:
"""
Performs a statistical test on the results of group phase coherence, to check for significance.
Parameters
----------
freq : ndarray
[1D array] The frequencies from group coherence.
coh1 : ndarray
[2D array] The coherence of the first group.
coh2 : ndarray
[2D array] The coherence of the second group.
bands : List[Tuple[float,float]]
List containing the frequency bands which will be tested for significance.
on_progress : Callable
Function called to report progress.
Returns
-------
pvalues : Dict[Tuple[float, float], float]
A list containing the p-values for each frequency band.
"""
self.stop()
self.scheduler = Scheduler(
run_in_thread=self.should_run_in_thread,
progress_callback=on_progress,
raise_exceptions=True,
capture_stdout=True,
only_threads=self.only_threads,
)
from pymodalib.algorithms.group_coherence import statistical_test
results = (await self.scheduler.map(
target=statistical_test,
args=[(
freq,
coh1,
coh2,
bands,
)],
))[0]
return dict(zip(bands, results))
def test_custom_validator(mock_print):
def test_validator(job):
job()
return SchedulerPolicies.RETRY
scheduler = Scheduler(validator=test_validator, interval=1)
print_task = Task(mock_print, Priorities.MEDIUM)
scheduler.add(print_task)
scheduler.start()
sleep(3)
scheduler.stop()
assert 2 <= mock_print.call_count <= 3
def test_process_two_tasks(self):
scheduler = Scheduler()
task1 = TestTask1()
task2 = TestTask2()
scheduler.clear_queue()
task1.clear_preceding_tasks()
task2.clear_preceding_tasks()
task2.add_preceding_task(task1)
scheduler.add_task_to_queue(task1)
scheduler.add_task_to_queue(task2)
scheduler.run_tasks()
self.assertIn(task1, scheduler._result_map)
self.assertIn(task2, scheduler._result_map)
self.assertGreater(scheduler._result_map[task2], scheduler._result_map[task1])
