def __init__(
self,
edit_speed_profile_presenter: EditSpeedProfilePresenter,
preferences_presenter: PreferencesPresenter,
get_status_interactor: GetStatusInteractor,
set_speed_profile_interactor: SetSpeedProfileInteractor,
settings_interactor: SettingsInteractor,
check_new_version_interactor: CheckNewVersionInteractor,
speed_profile_changed_subject: SpeedProfileChangedSubject,
speed_step_changed_subject: SpeedStepChangedSubject,
composite_disposable: CompositeDisposable,
) -> None:
LOG.debug("init MainPresenter ")
self.main_view: MainViewInterface = MainViewInterface()
self._edit_speed_profile_presenter = edit_speed_profile_presenter
self._preferences_presenter = preferences_presenter
self._scheduler: SchedulerBase = ThreadPoolScheduler(
multiprocessing.cpu_count())
self._get_status_interactor: GetStatusInteractor = get_status_interactor
self._set_speed_profile_interactor: SetSpeedProfileInteractor = set_speed_profile_interactor
self._settings_interactor = settings_interactor
self._check_new_version_interactor = check_new_version_interactor
self._speed_profile_changed_subject = speed_profile_changed_subject
self._speed_step_changed_subject = speed_step_changed_subject
self._composite_disposable: CompositeDisposable = composite_disposable
self._profile_selected: Dict[str, SpeedProfile] = {}
self._should_update_fan_speed: bool = False
self.application_quit: Callable = lambda *args: None # will be set by the Application
def main(config):
stream = config['stream']
prob_thresh = config['prob_thresh']
table = config['table']
iterator_type = config['iterator_type']
shard_id = kinesis.get_shard_id(stream)
source = kinesis.KinesisSource(stream, shard_id, iterator_type)
detector = tf.TfDetector(prob_thresh=prob_thresh)
db = dynamo.Dynamo(table)
Record = dynamo.DynamoRecord
optimal_thread_count = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
try:
# FIXME: this will currently block on detect_objects
# need buffered/pausable implementation:
# this will ultimately cause oom error:
# .flat_map(lambda it: rx.Observable.start(lambda: detect_objects(detector, it))) \
rx.Observable.from_iterable(get_frames(source)) \
.flat_map(lambda it: it) \
.map(lambda it: detect_objects(detector, it)) \
.filter(lambda frame_detection_pair: len(frame_detection_pair[1]) > 0) \
.flat_map(lambda frame_detection_pair, _: rx.Observable.start(lambda: insert_db(Record, db, frame_detection_pair))) \
.subscribe(
on_next,
on_err,
lambda: print('Complete'))
except Exception as e:
logger.error("exception: %s" % e)
raise e
def set_grpc_observable(self, new_observable: Observable, **kw):
new_observable = observe_on(ThreadPoolScheduler(1))(new_observable)
method_name = self._get_method_name(**kw)
if method_name not in self._observable_to_be_subscribed:
_LOGGER.exception('Method name {} not found, kw: {}'.format(method_name, kw))
raise KeyError
self.observables[method_name] = new_observable
self._observable_to_be_subscribed[method_name] = new_observable
def __init__(self):
self._cap = cv.VideoCapture(0)
self._stop_webcam = False
# set up observable using multithreading
producer_thread = ThreadPoolScheduler(1)
self.observable = Observable.create(self._observer_fn) \
.subscribe_on(producer_thread) \
.publish() \
.auto_connect()
def class_thirtyfive():
from rx.concurrency import ThreadPoolScheduler
from threading import current_thread
import multiprocessing, time, random, hashlib
def intense_calculation(obs):
for n in range(100000):
_ = hashlib.sha256(str(n).encode()).hexdigest()
THREAD_COUNT = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(THREAD_COUNT)
def __init__(self, host: str, port: int):
"""
Arguments:
host {str} -- Host name
port {int} -- Port number
"""
# domain
self.__domain = '{host}:{port}'.format(host=host, port=port)
# ThreadPool
self._pool_scheduler = ThreadPoolScheduler(1)
def __init__(self, protobuf_module, max_workers: int):
self.observers = {} # type: typing.Dict[str, typing.List[Observer]]
self.observables = {} # type: typing.Dict[str, Observable]
self._observable_to_be_subscribed = {} # type: typing.Dict[str, Observable]
pb_descriptor = protobuf_module.DESCRIPTOR
for service in pb_descriptor.services_by_name.values():
for method in service.methods:
full_method_name = self._get_method_name(method=method)
self._add_observer(full_method_name)
self._observable_to_be_subscribed[full_method_name] = observe_on(ThreadPoolScheduler(max_workers))(
self.get_grpc_observable(method_name=full_method_name)
)
def main(address: str = '0.0.0.0', port: int = 0,
server_factory=TCPServer,
keep_connection: bool = False,
panic: bool = False,
delimiter: bytes = b'\r\n',
workers: int = 2,
initial_packages: List[str] = [],
initial_values: Dict[str, object] = dict(),
encoding: Tuple[callable, callable] = parse_encoding('utf-8'),
package_parser: Tuple[callable, callable] = parse_parser('json')):
encode_func, decode_func = encoding
to_pack, from_pack = package_parser
manager = LocalSharedResourcesManager({
**dict((package, locate(package)) for package in initial_packages),
**initial_values
})
shared_server = ReactiveSharedResourcesServer(manager)
def accept_client(client):
print('accept client', client)
try:
observable_packages = client.as_observable(pool_scheduler) \
.map(map_debug) \
.flat_map(BuffAndSplit(delimiter, b'')) \
.map(map_debug)
if not keep_connection:
observable_packages = observable_packages.first()
result = observable_packages \
.safe_map(decode_func, print) \
.map(map_debug) \
.safe_map(from_pack, print) \
.safe_map(default_command_mapper, print) \
.flat_map(shared_server) \
.safe_map(to_pack, print) \
.safe_map(encode_func, print) \
.safe_map(lambda e: e + delimiter, print) \
.map(map_debug)
if not panic:
result = result.retry()
result.subscribe(client)
except Exception as ex:
print(ex)
pool_scheduler = ThreadPoolScheduler(workers)
server = server_factory()
server.connect(*(address, port,))
print(address, server.port)
server.as_observable(pool_scheduler) \
.subscribe(accept_client)
return server
def test_threadpool_executor(self):
with open(os.path.join(self.temp_dir, self._files[1]), 'wb') as qoox:
qoox.write(bytearray(self._fake_mkv_magic_bytes))
scanner = FileSystemMediaScanner(media_scan_path=self.temp_dir,
job_context=self.job_context,
skip_filetype_checks=False)
scanner_observer = rx.Observable.create(scanner)
processed_items = []
subject = rx.subjects.Subject()
subject.subscribe(on_next=lambda item: '')
subject.subscribe(on_next=lambda item: '')
subject.subscribe(on_next=lambda item: processed_items.append(item))
scanner_observer.subscribe_on(ThreadPoolScheduler(3)).observe_on(
ThreadPoolScheduler(3)).subscribe(subject)
time.sleep(1)
self.assertEqual(1, len(processed_items))
self.assertTrue(one(_.filename == 'B Quux 720p.mkv', processed_items))
def run(args):
""" Sets up and runs bot """
driver = init_chrome_driver(args)
prepare_database(args)
driver.get(SITE_PATH)
connection = connect_database(args)
scrape_ui(driver, connection)
print("Scraping completed")
connection = connect_database(args)
optimal_thread_count = multiprocessing.cpu_count() + 1
scheduler = ThreadPoolScheduler(optimal_thread_count)
update_descriptions(connection, scheduler)
scheduler.executor.shutdown()
print("Comlete!")
def class_thirtyfour():
#schedulers
#ImmediateScheduler
#NewThreadScheduler
#ThreadPoolScheduler <--
#TimeoutScheduler
from rx.concurrency import ThreadPoolScheduler
import time
from threading import current_thread
import multiprocessing
import hashlib
from rx.internal import extensionmethod
thread_count = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(thread_count)
def intense_calculation(obs):
for n in range(500000):
_ = hashlib.sha256(str(n).encode()).hexdigest()
#time.sleep(4)
return obs
@extensionmethod(Observable)
def apply_pool_scheduler(obs, boolean):
if boolean == True:
return obs.subscribe_on(pool_scheduler)
else:
return obs
def do_example(schedule):
start = time.perf_counter()
letters = Observable.from_(
["alpha", "Beta", "Gamma", "Delta", "Epsilon"])
letters.map(intense_calculation)\
.apply_pool_scheduler(schedule)\
.subscribe(on_next=lambda s: print("process 1 {}".format(s))
,on_error=lambda s: print("error {}".format(s))
,on_completed=lambda: print("process 1 done, total time {}".format(time.perf_counter() - start)))
letters.map(intense_calculation)\
.apply_pool_scheduler(schedule)\
.subscribe(on_next=lambda s: print("process 2 {}".format(s))
,on_error=lambda s: print("error {}".format(s))
,on_completed=lambda: print("process 2 done, total time {}".format(time.perf_counter() - start)))
print("==== no schedule")
do_example(False)
print("==== with schedule")
do_example(True)
input("")
def bus_context(conf):
blinker_ticks = Observable.interval(conf.blink.interval * 1000)
montage_ticks = Observable.interval(conf.montage.interval * 1000)
event_loop = EventLoopScheduler()
buttons = [
Button(Shoot, conf.event.shoot, conf.bounce_time, event_loop),
Button(Quit, conf.event.quit, conf.bounce_time, event_loop),
Button(Quit, conf.event.reboot, conf.bounce_time, event_loop),
Button(Quit, conf.event.shutdown, conf.bounce_time, event_loop)
]
commands = (Observable.merge([button.events for button in buttons]).merge(
ThreadPoolScheduler(max_workers=conf.workers),
blinker_ticks.map(const(Blink())),
montage_ticks.map(const(ShowRandomMontage()))))
with commands.subscribe(on_next=inject(handle_command, conf)):
yield
def __init__(self, client, fit_fun):
"""
Конструктор -__- класса ExecParallel
Аргумены:
client - ссылка на ipyparallel.Client()
fit_fun - ссылка на фитнесс-функцию. Функция принимает 1 аргумент (хромосому), возвращать она должна
кортеж (хромосома, фитнесс-функция)
"""
try:
self.client = client
self.lbv = client.load_balanced_view()
self.fit_fun = fit_fun
self._state = ExState.paused
self.scheduler = ThreadPoolScheduler()
self.async_res = None
self.stream = None
except:
self._state = ExState.error
def __init__(self, fun, n_workers):
"""
:param pauser:
:param fun: wchr -> lbv.apply_async()
"""
self.fun = fun
self.curr_tr_sched = CurrentThreadScheduler()
self.thr_pool_sched = ThreadPoolScheduler()
self.sub_new_jobs = Subject()
self.sub_done_jobs = Subject()
self.sub_freeworkers = Subject()
self.n_workers = n_workers
self.sub_new_jobs \
.map(self.on_next_njob_map) \
.flat_map(lambda ar: Observable.just(ar).observe_on(self.thr_pool_sched).map(lambda ar2: ar2.result())) \
.observe_on(self.curr_tr_sched) \
.subscribe(on_next=self.job_done)
def stepup_slsqp(self, gen0):
self.calc_gen(gen0)
lst = reduce_pop(gen0.pop_list, self.opts['n_op_direct'])
opti_lst = [
OptiSLSQP(op, self.executor.sub_in, self.executor.sub_compl)
for op in lst
]
tp = ThreadPoolScheduler()
calc_stream = Observable \
.from_(opti_lst) \
.flat_map(lambda op: Observable.just(op).observe_on(tp).map(lambda op: op.run()))
calc_stream.to_blocking().last_or_default(0)
for op in lst:
op.remove_exept_best()
for o_sls in opti_lst:
o_sls.dispose_conn()
gen1 = Generation(self.chr_contr, gen0.num_g + 1, lst)
self.sub_gen_solved.on_next(gen1)
gen1.fill_pop(self.opts['pop_count'])
return gen1
def index():
if request.method == 'GET':
return render_template('index.html')
if request.method == 'POST':
data_inicio = request.form['data_inicio']
data_fim = request.form['data_fim']
email = request.form['email']
host_elastic = request.form['host_elastic']
if not data_inicio:
data_inicio = 'now-1d/d'
if not data_fim:
data_fim = 'now/d'
if not host_elastic:
host_elastic = 'localhost'
if not email:
return render_template('index.html',
sucesso=False,
msg='Favor informar um email valido')
filename = 'cartao_%s_%s.txt' % (data_inicio, data_fim)
#Observer
optimal_thread_count = multiprocessing.cpu_count()
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
thread_data = {
'data_inicio': data_inicio,
'data_fim': data_fim,
'email': email,
'filename': filename,
'host_elastic': host_elastic
}
Observable.from_([thread_data])\
.map(lambda s: efetua_tarefa(s))\
.subscribe_on(pool_scheduler) \
.subscribe(on_next=lambda s: print("PROCESS 1: {0} {1}".format(current_thread().name, s)),
on_error=lambda e: print(e),
on_completed=lambda: print("PROCESS 1 done!"))
#end
return render_template('index.html', sucesso=True)
def main():
arg_endpoint = sys.argv[1] if len(sys.argv) > 1 else None
arg_sensor_type = sys.argv[2] if len(sys.argv) > 2 else TYPE_REAL
arg_sensor_arg = sys.argv[3] if len(sys.argv) > 3 else None
is_debug = "--debug" in sys.argv
if(not arg_endpoint):
print("An error occurred! Expected the 'endpoint' argument.")
print_usage()
sys.exit(1)
client = MonitorClient(arg_endpoint)
scheduler = ThreadPoolScheduler(4)
print("Starting '{0}' sensor...".format(arg_sensor_type))
sensor = create_sensor(arg_sensor_type, arg_sensor_arg)
print("Starting scanner...")
scanner = FrameScanner()
print("Starting heartbeat...")
start_heartbeat(client)
frames = sensor.get_frames(scheduler = scheduler)
events = scanner.scan(frames)
if is_debug:
print("subscribing to frames...")
frames.subscribe(
on_next = lambda x: client.post_frame(x),
on_completed = lambda: log("completed frames"),
on_error = log_error
)
print("subscribing to events...")
last_event = events.map(lambda x: client.post_event(x['type'], x['delta'])).to_blocking().last()
print("completed.")
def __init__(self, fun, kpp_fun=None):
"""
:param fun: должна возвращать AsyncResult объект
что-то типа:
def fun(wchr):
return lbv.apply_async(task_fun , wchr)
"""
self.fun = fun
self.kpp_fun = kpp_fun
self.sub_in = Subject()
self.sub_compl = Subject()
tp = ThreadPoolScheduler()
def f2(ar):
if self.kpp_fun:
self.kpp_fun()
return ar.result()
self.sub_in \
.map(lambda wchr: self.fun(wchr)) \
.flat_map(lambda ar: Observable.just(ar).observe_on(tp).map(f2)) \
.subscribe(self.sub_compl)
.subscribe(self.on_event)
def on_event(self, event):
value = -1 * event.value if self.reverse else event.value
self.controls.run(value)
try:
GPIO.setmode(GPIO.BCM)
servo = ServoControls(18, 100)
servo.setup()
main = DCControls(pin_a=13, pin_b=6, pwm_pin=5)
main.setup()
scheduler = ThreadPoolScheduler(2)
stream = x_stream(xpad_event_stream(deadzone=0.3)).\
publish()
xbox_servo = XboxServo(Key.X1, servo, reverse=False)
xbox_servo.bind(stream)
xbox_dc = XboxDC(Key.Y2, main, reverse=True)
xbox_dc.bind(stream)
stream.sample(1000).subscribe(print)
stream.connect()
finally:
GPIO.cleanup()
from colorama import Fore
import asyncio
import multiprocessing
from threading import current_thread
from rx import Observable
from rx.concurrency import ThreadPoolScheduler
optimal_thread_count = multiprocessing.cpu_count()
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
numbers01 = Observable.from_(range(10)).subscribe_on(pool_scheduler)
numbers02 = Observable.from_(range(10)).subscribe_on(pool_scheduler)
async def main():
numbers01.subscribe(lambda x: print(Fore.RED + '01 - {0}'.format(x)))
numbers02.subscribe(lambda x: print(Fore.GREEN + '02 - {0}'.format(x)))
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
from rx import Observable
from rx.concurrency import ThreadPoolScheduler
from threading import current_thread
import multiprocessing
import time
import random
def cálculo_intenso(valor):
time.sleep(random.choice([0.3,0.5,1]))
return valor
num_threads = multiprocessing.cpu_count()
calendarizador_pool = ThreadPoolScheduler(num_threads)
print("Número de threads: {}".format(num_threads))
Observable.from_(["A","B","C","D","E"])\
.map(lambda s: cálculo_intenso(s))\
.subscribe_on(calendarizador_pool)\
.subscribe(on_next=lambda p: print("Proceso 1: {} {}".format(current_thread().name, p)),
on_error=lambda e: print(e),
on_completed=lambda: print("Proceso 1: Terminado"))
Observable.from_(range(10))\
.map(lambda s: cálculo_intenso(s))\
.subscribe_on(calendarizador_pool)\
.subscribe(on_next=lambda p: print("Proceso 2: {} {}".format(current_thread().name, p)),
def handle_audio_end(pid):
known_processes.remove(pid)
print("KONIEC", pid, len(known_processes), current_thread().name)
if not known_processes:
try:
global app_proc
app_proc.kill()
app_proc = None
except SystemError:
print('Error during killing chroma app')
# access global variables safely
processing_scheduler = ThreadPoolScheduler(1)
scanner = Observable.create(lambda subscriber: scan_processes(subscriber)) \
.distinct() \
.subscribe_on(NewThreadScheduler()) \
.publish()
# start chroma app when valid process shows up
scanner.observe_on(processing_scheduler) \
.subscribe(handle_audio_start)
# tear down chroma app when all valid processes are terminated
scanner.flat_map(lambda pid: Observable.from_callable(wait_for_end(pid), NewThreadScheduler())) \
.observe_on(processing_scheduler) \
.subscribe(handle_audio_end)
logger.debug(
f"> Available INPUT_CONTENXT_MANAGERS: {INPUT_CONTEXT_MANAGERS.keys()}"
)
logger.debug(
f"> Available OUTPUT_CONTEXT_MANAGERS: {OUTPUT_CONTEXT_MANAGERS.keys()}"
)
external_predictor_class = find_predictor_class(args.module_name,
args.predictor_class_name)
external_predictor = external_predictor_class()
ctxmgr_settings = collect_ctxmgr_settings(args.input_ctx_manager,
args.output_ctx_manager)
for setting in ctxmgr_settings:
logger.info(f"{setting}: {ctxmgr_settings[setting]}")
scheduler = ThreadPoolScheduler(max_workers=1)
checkers_observable = rx.empty()
if settings.INSTANCE_ON_AWS:
logger.info(f"instance_type: {get_instance_type()}"
) # Assumes being run on AWS EC2 instance
if args.is_spot_instance or settings.AWS_ENABLE_SPOTINSTANCE_STATE_LOGGING:
logger.info(f"Start spot_instance_observable monitoring...")
spot_instance_observable = spot_instance_check_observable()
checkers_observable = checkers_observable.pipe(
ops.merge(spot_instance_observable))
elif args.is_spot_instance or settings.AWS_ENABLE_SPOTINSTANCE_STATE_LOGGING:
logger.warning(
f'"--spot-instance" flag or AWS_ENABLE_SPOTINSTANCE_STATE_LOGGING envar given, '
f"but INSTANCE_ON_AWS == False, logging NOT performed!")
from cyclotron import Component
from cyclotron.asyncio.runner import run
import cyclotron_std.sys.argv as argv
import cyclotron_std.argparse as argparse
import cyclotron_aiohttp.httpd as httpd
import cyclotron_std.io.file as file
import audio_encode_server.encoder as encoder
import audio_encode_server.s3 as s3
Drivers = namedtuple('Drivers', ['encoder', 'httpd', 's3', 'file', 'argv'])
Source = namedtuple('Source', ['encoder', 'httpd', 's3', 'file', 'argv'])
Sink = namedtuple('Sink', ['encoder', 'httpd', 's3', 'file'])
s3_scheduler = ThreadPoolScheduler(max_workers=1)
encode_scheduler = ThreadPoolScheduler(max_workers=4)
asyncio.get_event_loop().set_debug(True)
def parse_config(file_data):
return file_data.pipe(
ops.map(lambda i: json.loads(
i, object_hook=lambda d: namedtuple('x', d.keys())(*d.values()))),
ops.share(),
)
def create_arg_parser():
parser = argparse.ArgumentParser("audio encode server")
parser.add_argument('--config',
def run2():
s = requests.Session()
r = s.post(f"{FORUM_URL}/account.php",
data={
"action": "login",
"next": "/",
"username": USERNAME,
"password": PASSWORD
})
if "TEMPORARY LOCKOUT!" in r.text:
print("Too many wrong logins...")
exit(0)
elif """<div id="notregistered">""" in r.text:
print(r.text)
print("Login failed")
exit(0)
def on_complete():
print(f"Done! {current_thread().name}")
def on_error(e):
print(f"Error! {e} {current_thread().name}")
def on_next(_):
pass
def get_page_urls_from_forum(s_, a):
year, f_id = a
url = f"{FORUM_URL}/forumdisplay.php?forumid={f_id}"
if year == 2018:
resp = s_.post(url,
data={
"rem": "Remove",
"ac_year": year,
"ac_day": "",
"ac_month": ""
})
else:
resp = s_.post(url,
data={
"set": "GO",
"ac_year": year,
"ac_day": "",
"ac_month": ""
})
d = pq(resp.text)
if """<div id="notregistered">""" in resp.text:
print("Login failed")
exit(0)
s_.post(url,
data={
"rem": "Remove",
"ac_year": year,
"ac_day": "",
"ac_month": ""
})
pages = d.find(".pages a[title=\"Last page\"]").text()
pages = re.match("(\d+)", pages)
i = int(pages.group(1)) if pages else 1
if i == 1:
with open("log.html", "wb") as f:
f.write(resp.content)
return list(
map(lambda z: (s_, f"{url}&pagenumber={z}", f_id, year),
range(1, i)))
def get_threads_from_page(s_, a):
s_, url, f_id, year = a
resp = s.post(url, data={"ac_year": year})
d2 = pq(resp.text)
threads = []
for h in d2.find(".thread_title"):
t_id = int(re.search("threadid=(\d+)", h.get("href")).group(1))
threads += [(f_id, t_id, year)]
return threads
def get_page_urls_from_thread(s_, x):
f_id, t_id, year = x
u = f"{FORUM_URL}/showthread.php?threadid={t_id}"
resp_ = s_.get(u)
d = pq(resp_.text)
pages = d.find(".pages a[title=\"Last page\"]").text()
pages = re.match("(\d+)", pages)
i = int(pages.group(1)) if pages else 1
return list(
map(lambda z: (f"{u}&pagenumber={z}", f_id, t_id, year),
range(1, i)))
def get_posts(s_, g):
url, forum_id, t_id, year = g
if year == 2018:
resp = s_.post(url,
data={
"rem": "Remove",
"ac_year": year,
"ac_day": "",
"ac_month": ""
})
else:
resp = s_.post(url,
data={
"set": "GO",
"ac_year": year,
"ac_day": "",
"ac_month": ""
})
d = pq(resp.text)
found_posts = []
z = d.find(".post")
for h in z:
post_id = int(re.match("post(\d+)", d(h).attr("id")).group(1))
user_id = int(
re.search("userid-(\d+)",
d(h).find(".userinfo").attr("class")).group(1))
post_body = d(h).find(".postbody").html()
fail = [
"<!-- google_ad_section_start -->",
"<!-- google_ad_section_end -->"
]
for f in fail:
post_body = post_body.replace(f, "")
post_date_str = d(h).find(".postdate").text().replace(
"#", "").replace("?", "").strip()
post_date = datetime.strptime(post_date_str, '%b %d, %Y %H:%M')
p = Post.insert(post_id, user_id, post_body, t_id, post_date)
found_posts += [p]
return found_posts
scheduler = ThreadPoolScheduler(multiprocessing.cpu_count())
scheduler2 = ThreadPoolScheduler(multiprocessing.cpu_count())
scheduler3 = ThreadPoolScheduler(multiprocessing.cpu_count())
scheduler4 = ThreadPoolScheduler(multiprocessing.cpu_count())
forumids = Observable.from_([1, 25, 26, 154, 273, 21])
years = Observable.from_(range(2017, 2019))
forumids \
.flat_map(lambda g: years.map(lambda h: (h, g))) \
.flat_map(lambda z: Observable.just(z, scheduler).flat_map(lambda a: get_page_urls_from_forum(s, a))) \
.flat_map(lambda z: Observable.just(z, scheduler2).flat_map(lambda a: get_threads_from_page(s, a))) \
.flat_map(lambda z: Observable.just(z, scheduler3).flat_map(lambda a: get_page_urls_from_thread(s, a))) \
.flat_map(lambda z: Observable.just(z, scheduler4).flat_map(lambda a: get_posts(s, a))) \
.subscribe(on_next=on_next,
on_error=on_error,
on_completed=on_complete)
scheduler.executor.shutdown()
scheduler2.executor.shutdown()
scheduler3.executor.shutdown()
scheduler4.executor.shutdown()
# test_merge()
# test_marbles()
print(f'main_thread:={threading.current_thread().name}')
stop = False
def subscribe(observer):
cnt = 0
while not stop and cnt < 10000:
observer.on_next(cnt)
cnt += 1
def stop_fn():
print('being disposed')
stop = True
return stop_fn
sub = Observable.create(subscribe) \
.subscribe_on(ThreadPoolScheduler(2)) \
.buffer_with_count(10) \
.map(lambda l:l[-1]) \
.subscribe(on_next=print, on_error=print)
print('...zzzZZZZ')
time.sleep(0.5)
sub.dispose()
print('disposed()')
time.sleep(2)
print('done')
def test_threadpool_now_units(self):
scheduler = ThreadPoolScheduler()
diff = scheduler.now
sleep(0.1)
diff = scheduler.now - diff
assert timedelta(milliseconds=80) < diff < timedelta(milliseconds=180)
def test_threadpool_now(self):
scheduler = ThreadPoolScheduler()
diff = scheduler.now - default_now()
assert abs(diff) < timedelta(milliseconds=1)
import unittest
import threading
from datetime import timedelta
from time import sleep
from rx.concurrency import ThreadPoolScheduler
from rx.internal.basic import default_now
thread_pool_scheduler = ThreadPoolScheduler()
class TestThreadPoolScheduler(unittest.TestCase):
def test_threadpool_now(self):
scheduler = ThreadPoolScheduler()
diff = scheduler.now - default_now()
assert abs(diff) < timedelta(milliseconds=1)
def test_threadpool_now_units(self):
scheduler = ThreadPoolScheduler()
diff = scheduler.now
sleep(0.1)
diff = scheduler.now - diff
assert timedelta(milliseconds=80) < diff < timedelta(milliseconds=180)
def test_schedule_action(self):
ident = threading.current_thread().ident
evt = threading.Event()
nt = thread_pool_scheduler
class Schedulers(object):
computation = ThreadPoolScheduler(multiprocessing.cpu_count() * 2)