class Async_Optimize(object):
def __init__(self):
self.callback = lambda *x: None
self.runsignal = Event()
self.SENTINEL = "SENTINEL"
def async_callback_collect(self, q):
while self.runsignal.is_set():
try:
for ret in iter(lambda: q.get(timeout=1), self.SENTINEL):
self.callback(*ret)
self.runsignal.clear()
except Empty:
pass
def opt_async(self, f, df, x0, callback, update_rule=PolakRibiere,
messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
report_every=10, *args, **kwargs):
self.runsignal.set()
c = None
outqueue = None
if callback:
outqueue = Queue()
self.callback = callback
c = Thread(target=self.async_callback_collect, args=(outqueue,))
c.start()
p = _CGDAsync(f, df, x0, update_rule, self.runsignal, self.SENTINEL,
report_every=report_every, messages=messages, maxiter=maxiter,
max_f_eval=max_f_eval, gtol=gtol, outqueue=outqueue, *args, **kwargs)
p.start()
return p, c
def opt(self, f, df, x0, callback=None, update_rule=FletcherReeves,
messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
report_every=10, *args, **kwargs):
p, c = self.opt_async(f, df, x0, callback, update_rule, messages,
maxiter, max_f_eval, gtol,
report_every, *args, **kwargs)
while self.runsignal.is_set():
try:
p.join(1)
if c: c.join(1)
except KeyboardInterrupt:
# print "^C"
self.runsignal.clear()
p.join()
if c: c.join()
if c and c.is_alive():
# self.runsignal.set()
# while self.runsignal.is_set():
# try:
# c.join(.1)
# except KeyboardInterrupt:
# # print "^C"
# self.runsignal.clear()
# c.join()
print("WARNING: callback still running, optimisation done!")
return p.result
async def check_multiprocess_shutdown_event(
shutdown_event: EventType, sleep: Callable[[float], Awaitable[Any]]
) -> None:
while True:
if shutdown_event.is_set():
return
await sleep(0.1)
async def check_shutdown(
shutdown_event: EventType, sleep: Callable[[float], Awaitable[Any]]
) -> None:
while True:
if shutdown_event.is_set():
raise Shutdown()
await sleep(0.1)
class Generator():
def __init__(self, configurationPath, outputQueue):
self.processorQueue = outputQueue
self.logger = logging.getLogger("generator")
self.lirchandle = pylirc.init("pylirc", configurationPath, False)
self.stopFlag = Event()
def loop(self):
self.logger.info("Starting up.")
if(self.lirchandle):
inputLirc = [self.lirchandle]
timeout = 2
self.logger.info("Succesfully opened lirc, handle is " + str(self.lirchandle))
self.logger.info("Started.")
while (not self.stopFlag.is_set()):
inputready, outputready, exceptready = select.select(inputLirc,[],[], timeout)
s = pylirc.nextcode(1)
if(s):
for code in s:
repeat = code["repeat"]
currentCommand = code["config"]
self.logger.info("New event received: id: = %s, repeat = %s" % (currentCommand, repeat))
try:
self.processorQueue.put_nowait(GeneratorEvent(currentCommand, repeat))
except Queue.Full:
self.logger.error("Processor queue is overloaded.")
self.logger.info("Shutted down.")
def shutdown(self):
self.logger.debug("Shutting down.")
self.stopFlag.set()
pylirc.exit()
async def wslogs(self,
remote_id: 'str',
stop_event: Event,
current_line: int = 0):
""" websocket log stream from remote pea/pod
:param remote_id: the identity of that pea/pod
:param stop_event: the multiprocessing event which marks if stop event is set
:param current_line: the line number from which logs would be streamed
:return:
"""
with ImportExtensions(required=True):
import websockets
try:
# sleeping for few seconds to allow the logs to be written in remote
await asyncio.sleep(3)
async with websockets.connect(
f'{self.log_url}/{remote_id}?timeout=20') as websocket:
await websocket.send(json.dumps({'from': current_line}))
remote_loggers = {}
while True:
log_line = await websocket.recv()
if log_line:
try:
log_line = json.loads(log_line)
current_line = int(list(log_line.keys())[0])
log_line_dict = list(log_line.values())[0]
log_line_dict = json.loads(
log_line_dict.split('\t')[-1].strip())
name = log_line_dict['name']
if name not in remote_loggers:
remote_loggers[name] = JinaLogger(
context=f'🌏 {name}')
# TODO: change logging level, process name in local logger
remote_loggers[name].info(
f'{log_line_dict["message"].strip()}')
except json.decoder.JSONDecodeError:
continue
await websocket.send(json.dumps({}))
if stop_event.is_set():
for logger in remote_loggers.values():
logger.close()
raise RemotePodClosed
except websockets.exceptions.ConnectionClosedOK:
self.logger.debug(f'Client got disconnected from server')
return current_line
except websockets.exceptions.WebSocketException as e:
self.logger.error(
f'Got following error while streaming logs via websocket {repr(e)}'
)
return 0
def _worker(queue: JoinableQueue, engine: Engine,
discovery: Dict[str, DiscoveryState], objects: List[Artifact],
lock: LockT, stopswitch: EventT):
while not stopswitch.is_set():
try:
artifact = queue.get(block=True, timeout=0.5)
except QueueEmpty:
continue
try:
new_artifacts = engine.process(artifact, discovery, objects, lock)
for new_artifact in new_artifacts:
queue.put(new_artifact)
finally:
queue.task_done()
def log(self, remote_id: 'str', stop_event: Event, **kwargs) -> None:
""" Start the log stream from remote pea/pod, will use local logger for output
:param remote_id: the identity of that pea/pod
:return:
"""
with ImportExtensions(required=True):
import requests
try:
url = f'{self.log_url}/?{self.kind}_id={remote_id}'
r = requests.get(url=url, stream=True)
for log_line in r.iter_content():
if log_line:
self.logger.info(f'🌏 {log_line.strip()}')
if stop_event.is_set():
break
except requests.exceptions.RequestException as ex:
self.logger.error(
f'couldn\'t connect with remote jinad url {repr(ex)}')
finally:
self.logger.info(f'🌏 exiting from remote logger')
class Async_Optimize(object):
def __init__(self):
self.callback = lambda *x: None
self.runsignal = Event()
self.SENTINEL = "SENTINEL"
def async_callback_collect(self, q):
while self.runsignal.is_set():
try:
for ret in iter(lambda: q.get(timeout=1), self.SENTINEL):
self.callback(*ret)
self.runsignal.clear()
except Empty:
pass
def opt_async(self,
f,
df,
x0,
callback,
update_rule=PolakRibiere,
messages=0,
maxiter=5e3,
max_f_eval=15e3,
gtol=1e-6,
report_every=10,
*args,
**kwargs):
self.runsignal.set()
c = None
outqueue = None
if callback:
outqueue = Queue()
self.callback = callback
c = Thread(target=self.async_callback_collect, args=(outqueue, ))
c.start()
p = _CGDAsync(f,
df,
x0,
update_rule,
self.runsignal,
self.SENTINEL,
report_every=report_every,
messages=messages,
maxiter=maxiter,
max_f_eval=max_f_eval,
gtol=gtol,
outqueue=outqueue,
*args,
**kwargs)
p.start()
return p, c
def opt(self,
f,
df,
x0,
callback=None,
update_rule=FletcherReeves,
messages=0,
maxiter=5e3,
max_f_eval=15e3,
gtol=1e-6,
report_every=10,
*args,
**kwargs):
p, c = self.opt_async(f, df, x0, callback, update_rule, messages,
maxiter, max_f_eval, gtol, report_every, *args,
**kwargs)
while self.runsignal.is_set():
try:
p.join(1)
if c: c.join(1)
except KeyboardInterrupt:
# print "^C"
self.runsignal.clear()
p.join()
if c: c.join()
if c and c.is_alive():
# self.runsignal.set()
# while self.runsignal.is_set():
# try:
# c.join(.1)
# except KeyboardInterrupt:
# # print "^C"
# self.runsignal.clear()
# c.join()
print("WARNING: callback still running, optimisation done!")
return p.result
