def scan(self):
self.is_armed = not self.bluetooth.is_there_friendly_devices_nearby()
self.blueStream.on_next(self.is_armed)
# TODO: decouple sms alert
if self.is_armed:
Observable.just(True).subscribe(SmsObserver('system armed'))
else:
Observable.just(True).subscribe(SmsObserver('system disarmed'))
def test_paused_with_immediate_unpause(self):
subscription = [None]
scheduler = TestScheduler()
results = scheduler.create_observer()
xs = scheduler.create_hot_observable(
on_next(150, 1),
on_next(210, 2),
on_completed(500)
)
controller = Observable.just(True)
pausable_buffered = xs.pausable_buffered(controller)
def action1(scheduler, state):
subscription[0] = pausable_buffered.subscribe(results)
scheduler.schedule_absolute(200, action1)
scheduler.start()
results.messages.assert_equal(
on_next(210, 2),
on_completed(500)
)
def main():
# Specify image file (image #2 cooperates better)
FILE = 'res/box-2.jpg'
# Stream the image as a whole
originalObservable = Observable.just(cv2.imread(FILE))
# Manipulate the image to bring out the box
imageObservable = (
originalObservable
.map(lambda img: cv2.bilateralFilter(img, 9, 75, 75)) # bilateral blur (denoise)
.map(lambda img: cv2.Canny(img, 100, 100, apertureSize=3)) # detect the edges
)
# Detect any lines and represent them with [startPoint, endPoint] in list [lines]
lines = (
imageObservable
# transform the stream into multiple numpy vectors (each line represented by [[x1, y1, x2, y2]])
.flat_map(lambda img: cv2.HoughLinesP(img, 1, pi/180, 50, minLineLength=70, maxLineGap=40))
.map(lambda wrapped_line: wrapped_line[0]) # flatten line array into [x1, y1, x2, y2]
.map(lambda lv, _: [Line((lv[0], lv[1]), (lv[2], lv[3]))]) # pack line vector into [((x1, y2), (x2, y2))]
.reduce(lambda accumulated, line: accumulated + line, []) # reduce all lines into one [line, line, ..]
)
# For debugging/visualization purposes only
# http://reactivex.io/documentation/operators/zip.html
lines.zip(originalObservable, lambda lines, image: (image, lines)).subscribe(ImageObserver())
def watch(self):
# TODO: simplify this with reactive programming
while True:
if self.is_armed:
time.sleep(parameters.main_interval_between_scan_while_armed)
else:
time.sleep(parameters.main_interval_between_scan_while_disarmed)
will_arm = not self.bluetooth.is_there_friendly_devices_nearby()
if not self.is_armed:
if will_arm:
Observable.just(True).subscribe(SmsObserver('system armed'))
self.is_armed = True
self.blueStream.on_next(True)
else:
if not will_arm:
Observable.just(True).subscribe(SmsObserver('system disarmed'))
self.is_armed = False
self.blueStream.on_next(False)
def message():
data = parse_qs(request.get_data(False, True, False))
# For some reason these values are lists first
data = {x: data[x][0] for x in data}
token = data['token']
print("Credentials are - \n" + str(credentials))
print("Token received is " + token)
print("Stored token is " + credentials['slack_token'])
# Verify it came from slack
if token != credentials['slack_token']:
return Response(json.dumps({'text': 'Invalid'}), status=403, mimetype='application/json')
else:
message_data = data['text']
Observable.just(message_data, new_thread_scheduler)\
.map(lambda s: process_command(s))\
.subscribe(BotObserver())\
resp = Response(None, status=200, mimetype='application/json')
return resp
f = urlopen(link)
return Observable.from_(f) \
.map(lambda s: s.decode("gbk").strip())
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"))
codes = codes = [
"usr_aapl", "usr_fb", "usr_goog", "usr_baba", "usr_ge", "usr_tsla",
"usr_atvi", "usr_hpq"
]
source = Observable.from_(codes)\
.map(lambda s: "http://hq.sinajs.cn/list={0}".format(s))\
.flat_map(lambda s:
Observable.just(s).subscribe_on(pool_scheduler).flat_map(lambda t: read_request(t))
)\
.map(lambda s: s.split('"')[1]) \
.filter(lambda l: l != "") \
.map(lambda s: s.split(","))\
.map(lambda s: "股票：{0} 价格：{1}".format(s[0], s[1]))
source.subscribe(
on_next=lambda i: print("{0} {1}".format(current_thread().name, i)),
on_error=lambda e: print(e),
on_completed=lambda: print(
"PROCESS 1 Done!",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"))
def deepspeech_server(sources):
argv = sources.argv.argv
stt = sources.httpd.route
stt_response = sources.deepspeech.text.share()
ds_logs = sources.deepspeech.log
config_data = sources.file.response
http_ds_error, route_ds_error = make_error_router()
args = argparse.argparse(
argv=argv.skip(1).subscribe_on(aio_scheduler),
parser=Observable.just(
argparse.Parser(description="deepspeech server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
]))
config_file = (args.filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value)))
config = parse_config(config_data).subscribe_on(aio_scheduler)
logs_config = (config.flat_map(lambda i: Observable.from_(i.log.level).map(
lambda i: logging.SetLevel(logger=i.logger, level=i.level)).concat(
Observable.just(logging.SetLevelDone()))))
logs = Observable.merge(logs_config, ds_logs)
log_ready = sources.logging.response.take(1)
ds_stt = (stt.flat_map(lambda i: i.request).map(
lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)))
ds_arg = (
# config is hot, the combine operator allows to keep its last value
# until logging is initialized
log_ready.combine_latest(
config, lambda _, i: i).map(lambda i: deepspeech.Initialize(
model=i.deepspeech.model,
alphabet=i.deepspeech.alphabet,
lm=i.deepspeech.lm,
trie=i.deepspeech.trie,
features=deepspeech.FeaturesParameters(
n_features=i.deepspeech.features.n_features,
n_context=i.deepspeech.features.n_context,
beam_width=i.deepspeech.features.beam_width,
lm_alpha=i.deepspeech.features.lm_alpha,
lm_beta=i.deepspeech.features.lm_beta,
) if i.deepspeech.features is not None else None)))
ds = ds_stt.merge(ds_arg)
http_init = (config.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=i.server.http.request_max_size),
httpd.AddRoute(
methods=['POST'],
path='/api/stt',
id='stt',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http_response = (stt_response.let(
route_ds_error,
error_map=lambda e: httpd.Response(
data="Speech to text error".encode('utf-8'),
context=e.args[0].context,
status=500)).map(lambda i: httpd.Response(
data=i.text.encode('utf-8'),
context=i.context,
)))
http = Observable.merge(http_init, http_response, http_ds_error)
return DeepspeechSink(file=file.Sink(request=config_file),
logging=logging.Sink(request=logs),
deepspeech=deepspeech.Sink(speech=ds),
httpd=httpd.Sink(control=http))
from rx import Observable
number = Observable.just(1)
number.subscribe(on_next=lambda i: print("item: {}".format(i)),
on_error=lambda e: print("error: {}".format(e)),
on_completed=lambda: print("completed"))
import asyncio
from rx import Observable
stream = Observable.just("Hello, world!")
async def hello_world():
n = await stream
print(n)
loop = asyncio.get_event_loop()
# Blocking call which returns when the hello_world() coroutine is done
loop.run_until_complete(hello_world())
loop.close()
from rx import Observable, Observer
# Using Observable.range()
letters = Observable.range(1, 10)
letters.subscribe(lambda value: print(value))
# Using Observable.just()
greeting = Observable.just("Hello World!")
greeting.subscribe(lambda value: print(value))
def getIntervalOb(time):
return Observable.interval(time).switch_map(
lambda i: Observable.just(i).subscribe_on(pool_scheduler))
from dumper import Dumper
from rx import Observable, Observer
from rx.internal import extensionmethod
from rx.subjects import Subject
in_ = '23432/XX428/X21X71'
Observable.from_(in_) \
.flat_map(lambda q: Observable.range(1, 2) if q == 'X' else Observable.just(q)) \
.buffer_with_count(2) \
.map(lambda x, i: i) \
.take(10) \
.subscribe(Dumper('s'))
def read_last_line_from_file(filename):
with open(filename) as file:
lines = file.readlines()
last_line = lines[-1]
return Observable.just(last_line)
def extract_features(sources):
aio_scheduler = AsyncIOScheduler()
file_response = sources.file.response.share()
config_sink = config.read_configuration(
config.Source(file_response=file_response,
argv=sources.argv.argv.subscribe_on(aio_scheduler)))
configuration = config_sink.configuration.share()
walk_adapter = walk.adapter(sources.walk.response)
#file_adapter = file.adapter(sources.media_file.response)
#write_feature_request, write_feature_file = router.make_crossroad_router(file_response)
media_file_request, feature_file_request, process_path = path_processor.make_path_processor(
sources.media_file.response, sources.feature_file.response)
random_cross_request, cross_random = router.make_crossroad_router(
sources.random.response)
features = (
configuration.flat_map(
lambda configuration: walk_adapter.api.walk(configuration.dataset.
voxceleb2_path)
# extract features from files
.let(
process_path,
configuration=configuration,
#file_adapter=file_adapter,
)
# create sets
.reduce(lambda acc, i: acc + [{
'file': i,
'label': label_from_path(i),
'set': set_from_path(i),
}],
seed=[])
# todo: shuffle
.map(train_test_split).flat_map(
lambda dataset: Observable.just(dataset['test']).map(pair_set)
# shuffle apn pairs
.map(lambda i: random.Shuffle(id='dev_test_set', data=i)).let(
cross_random).filter(lambda i: i.id == 'dev_test_set').
map(lambda i: i.data).map(lambda i: test_dev_split(
i, configuration.dataset.dev_set_utterance_count,
configuration.dataset.test_set_utterance_count)).map(
lambda i: {
'train': dataset['train'],
'dev': i['dev'],
'test': i['test'],
}))).share())
# save dataset json file
write_dataset_request = (features.map(json.dumps).with_latest_from(
configuration, lambda dataset, configuration: file.Write(
id='write_dataset',
path=configuration.dataset.path,
data=dataset,
mode='w')).share())
# random
random_request = Observable.concat(
configuration.map(lambda i: random.SetSeed(value=i.random_seed)),
random_cross_request)
logs = features
exit = sources.dataset_file.response.ignore_elements()
return Sink(file=file.Sink(request=config_sink.file_request),
dataset_file=file.Sink(request=write_dataset_request),
media_file=file.Sink(request=media_file_request),
feature_file=file.Sink(request=feature_file_request),
logging=logging.Sink(request=logs),
walk=walk.Sink(request=walk_adapter.sink),
stop=stop.Sink(control=exit),
random=random.Sink(request=random_request))
def to_file(filename):
f = open(filename)
return Observable.using(lambda: Disposable(lambda: f.close()),
lambda d: Observable.just(f))
def _do_work(self, task):
self.logger.info('Starting {}...'.format(task))
return Observable.just(task).map(
lambda task: self.post_process_imp.putTaskToPostgreSQL(task)
).catch_exception(lambda error: self._logErrorAndResume(error, task))
def publish_tasks(self, time_arr, task_arr):
if self.task_producer is None:
self.task_producer = Observable.from_(time_arr).flat_map(lambda i: Observable.timer(i * 1000).switch_map(lambda i: Observable.just(i).subscribe_on(pool_scheduler))).zip(Observable.from_(task_arr), lambda x, y: y).publish().ref_count()
return self.task_producer
def process(path, configuration):
''' compute features for all files in path
This lettable operator processes all files present in the path observable.
It reads each file, process them, and writes the result. The processing
is multithreaded via a thread pool. The resulting observable is still
scheduled in the asyncio event loop.
'''
aio_ts_scheduler = AsyncIOScheduler(threadsafe=True)
aio_scheduler = AsyncIOScheduler()
thread_scheduler = ThreadPoolScheduler(
max_workers=configuration.data_preparation.cpu_core_count)
# prepare file routing : read media and write features
media_file_response = file_response
feature_write_request, write_feature_file = router.make_crossroad_router(
feature_response)
media_read_request, read_media_file = router.make_crossroad_router(
media_file_response)
media_read_request \
.flat_map(lambda i: Observable.just(i, scheduler=thread_scheduler)) \
.subscribe(sink_request)
feature_write_request.subscribe(feature_request)
# feature engineering
return (
path
#.do_action(TraceObserver(prefix='write1', trace_next_payload=False))
.flat_map(
lambda i: i.files
.map(lambda path: file.Read(
id='read_media',
path=path,
mode='rb',
))
.let(read_media_file)
.filter(lambda i: i.id == 'read_media')
.flat_map(lambda media: media.data
#.do_action(lambda i: print('write20-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write20', trace_next_payload=False))
.let(process_audio, configuration=configuration.features)
.map(lambda i: data_to_feature(i, media.path, configuration))
#.do_action(TraceObserver(prefix='write21-{}'.format(threading.get_ident()), trace_next_payload=False))
# .do_action(lambda i: print(i.path))
)
#)
#.do_action(lambda i: print('write2-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write2', trace_next_payload=False))
)
# write feature file to disk
.map(lambda i: file.Write(
id='write_feature',
path=i.path, data=i.data,
mode='wb', mkdirs=True))
.let(write_feature_file)
.filter(lambda i: i.id == 'write_feature')
.map(lambda i: i.path)
#.do_action(lambda i: print('write2-2-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write2-2', trace_next_payload=False))
.observe_on(aio_ts_scheduler)
#.map(lambda i: "/foo/bar/i/4.mp3")
#.do_action(lambda i: print('write3-{}'.format(threading.get_ident())))
#.do_action(TraceObserver(prefix='write3', trace_next_payload=False))
)
from modules.door_listener import DoorListener
from modules.light import LightOnObserver
from modules.light import LightOffObserver
from modules.siren_client import SirenClientObserver
from modules.sms import SmsObserver
from modules.pirs import Pirs
from modules.blue import Scanner
import parameters
door_listener = DoorListener()
pirs = Pirs()
scanner = Scanner()
# check that everything is ok
Observable.just(True).subscribe(SmsObserver('alarm started'))
Observable.just(True).subscribe(LightOnObserver())
Observable.timer(2000).subscribe(LightOffObserver())
# rise error: Camera component couldn't be enabled: Out of resources (other than memory)
# Observable.just(True).subscribe(CameraObserver()) # TODO take an initial photo on startup
scanner.scan()
def main():
# start bluetooth scanning in the background
new_bluetooth_thread = threading.Thread(target=scanner.watch)
new_bluetooth_thread.daemon = True # stop if the program exits
new_bluetooth_thread.start()
# start listening door tag
new_doortag_thread = threading.Thread(target=door_listener.listen)
import asyncio
import aiohttp
from rx import Observable
from rx.concurrency import AsyncIOScheduler
from rx import Observable
loop = asyncio.get_event_loop()
loop.set_debug(True)
scheduler = AsyncIOScheduler(loop)
def request(method, url, **kw):
future = asyncio.ensure_future(aiohttp.request(method, url, **kw))
return Observable.from_future(future)
ob = Observable.just(1, scheduler=scheduler)
ob10 = ob.\
map(lambda x: request('GET', 'https://httpbin.org/')).\
flat_map(lambda x: x)
ob10.subscribe(print)
ob.subscribe(print)
loop.run_forever()
def catch_error(error):
exe_context.errors.append(error)
return Observable.just(None)
def test_for_each_argument_checking(self):
some = Observable.just(42).to_blocking()
self.assertRaises(TypeError, lambda: Observable(None).to_blocking().for_each(lambda x: x))
self.assertRaises(TypeError, lambda: some.for_each(lambda: None))
def to_file(filename):
f = open(filename)
return Observable.using(
lambda: Disposable(lambda: f.close()),
lambda d: Observable.just(f)
)
import multiprocessing
import random
import time
from threading import current_thread
from rx import Observable
from rx.concurrency import ThreadPoolScheduler
def intense_calculation(value):
# sleep for a random short duration between 0.5 to 2.0 seconds to simulate a long-running calculation
time.sleep(random.randint(5, 20) * .1)
return value
# calculate number of CPU's, then create a ThreadPoolScheduler with that number of threads
optimal_thread_count = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
Observable.of("Alpha", "Beta", "Gamma", "Delta", "Epsilon") \
.flat_map(lambda s: Observable.just(s).subscribe_on(pool_scheduler).map(lambda s: intense_calculation(s))) \
.subscribe(on_next=lambda s: print("PROCESS 1: {0} {1}".format(current_thread().name, s)),
on_error=lambda e: print(e),
on_completed=lambda: print("PROCESS 1 done!"))
def class_five():
Observable.just("Hello world").subscribe(on_next=lambda s: print(s))
def commentObservable(obs):
return Observable.just("/*").concat(obs.map(lambda line: " * " + line)).concat(Observable.just(" */"))
def get_weather(location):
with requests.session() as session:
endpoint = '{}/{},{}'.format(URI, location[0], location[1])
return Observable.just(session, scheduler=Scheduler.current_thread) \
.map(lambda x: x.get(endpoint)) \
.interval(1000)
def audio_encoder(sources):
# Parse configuration
read_config_file = (
sources.argv.argv.skip(1)
.let(argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(
name='--config', help="Path of the server configuration file")
]))
.filter(lambda i: i.key == 'config')
.map(lambda i: file.Read(id='config', path=i.value))
)
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (
config
.map(lambda i: encoder.Initialize(storage_path=i.encode.storage_path))
)
encode_request = (
sources.httpd.route
.filter(lambda i: i.id == 'flac_transcode')
.flat_map(lambda i: i.request)
.do_action(lambda i: print("[{}]http req: {}".format(datetime.datetime.now(), threading.get_ident())))
#.observe_on(encode_scheduler)
.flat_map(lambda i: Observable.just(i, encode_scheduler))
.do_action(lambda i: print("[{}]encode req: {}".format(datetime.datetime.now(), threading.get_ident())))
.map(lambda i: encoder.EncodeMp3(
id=i.context,
data=i.data,
key=i.match_info['key']))
)
encoder_request = Observable.merge(encode_init, encode_request)
# store encoded file
store_requests = (
sources.encoder.response
.do_action(lambda i: print("[{}]encode res: {}".format(datetime.datetime.now(), threading.get_ident())))
.observe_on(s3_scheduler)
.do_action(lambda i: print("[{}]s3 req: {}".format(datetime.datetime.now(), threading.get_ident())))
.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
))
)
# acknowledge http request
http_response = (
sources.s3.response
.do_action(lambda i: print("[{}]s3 res: {}".format(datetime.datetime.now(), threading.get_ident())))
.do_action(lambda i: print("httpd res: {}".format(threading.get_ident())))
.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
))
)
# http server
http_init = (
config
.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(
host=i.server.http.host,
port=i.server.http.port),
]))
)
http = Observable.merge(http_init, http_response)
# s3 database
s3_init = (
config
.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
))
)
# merge sink requests
file_requests = read_config_file
s3_requests = Observable.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
from __future__ import print_function
from rx import Observable
from rx.concurrency import ThreadPoolScheduler
from threading import current_thread
import multiprocessing, time, random
def intense_calculation(value):
# sleep for a random short duration between 0.5 to 2.0 seconds to simulate a long-running calculation
time.sleep(random.randint(5,20) * .1)
return value
# calculate number of CPU's and add 1, then create a ThreadPoolScheduler with that number of threads
optimal_thread_count = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
# Create Process 1
Observable.from_(["Alpha","Beta","Gamma","Delta","Epsilon", "Alpha","Beta","Gamma","Delta","Epsilon"]) \
.flat_map(lambda s: Observable.just(s).subscribe_on(pool_scheduler).map(lambda s: intense_calculation(s))) \
.subscribe(on_next=lambda s: print("PROCESS 1: {0} {1}".format(current_thread().name, s)),
on_error=lambda e: print(e),
on_completed=lambda: print("PROCESS 1 done!"))
input("Press any key to exit\n")
from rx import Observable scheduler = ThreadPoolScheduler() xs = Observable.range(1, 5).flat_map(lambda x: Observable.just(x, scheduler=scheduler), mapper)
from __future__ import print_function
from rx import Observable, Observer
from rx.concurrency import ThreadPoolScheduler
from threading import current_thread
import multiprocessing, time, random
optimal_thread_count = multiprocessing.cpu_count() + 1
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
# .switch_map(lambda i: Observable.from_(running_tasks).subscribe_on(pool_scheduler).map(lambda s: print_task_st(s))) \
a = Observable.interval(1000).switch_map(lambda i: Observable.just(
i).subscribe_on(pool_scheduler)).publish().ref_count()
a.subscribe(
lambda i: print("first observer {0}, {1}".format(i,
current_thread().name)))
time.sleep(5)
a.subscribe(
lambda i: print("second observer {0}, {1}".format(i,
current_thread().name)))
while True:
pass
def catch_error(error):
subscriber_exe_context.errors.append(error)
return Observable.just(None)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logger = logging.getLogger(__name__)
def file_source(file):
return open(file, 'r')
def watch(handle):
return select.select([handle], [], [], 5)
def print_f(value):
print value
def handle_select(value):
if ([], [], []) == value:
logger.info('Nothing happening...')
else:
return value[0][0].readline()
Observable.just(sys.stdin) \
.map(watch) \
.map(handle_select) \
.filter(bool) \
.repeat() \
.subscribe(on_next=logger.info)
