def _wrap(
local_rank: int,
fn: Callable,
args: Dict[int, Tuple],
envs: Dict[int, Dict[str, str]],
stdout_redirects: Dict[
int, str], # redirect file for stdout (to console if None)
stderr_redirects: Dict[
int, str], # redirect file for stderr (to console if None)
ret_vals: Dict[int, mp.SimpleQueue],
queue_finished_reading_event: synchronize.Event,
) -> None:
# get the per-rank params up front so we fail fast if no mapping is found
args_ = args[local_rank]
env_ = envs[local_rank]
ret_val_ = ret_vals[local_rank]
stdout_rd = stdout_redirects[local_rank]
stderr_rd = stderr_redirects[local_rank]
stdout_cm = redirect_stdout(stdout_rd) if stdout_rd else _nullcontext()
stderr_cm = redirect_stderr(stderr_rd) if stderr_rd else _nullcontext()
for k, v in env_.items():
os.environ[k] = v
with stdout_cm, stderr_cm:
ret = record(fn)(*args_)
ret_val_.put(ret)
queue_finished_reading_event.wait()
class Generator():
def __init__(self, configurationPath, outputQueue):
self.processorQueue = outputQueue
self.logger = logging.getLogger("generator")
self.lirchandle = pylirc.init("pylirc", configurationPath, False)
self.stopFlag = Event()
def loop(self):
self.logger.info("Starting up.")
if(self.lirchandle):
inputLirc = [self.lirchandle]
timeout = 2
self.logger.info("Succesfully opened lirc, handle is " + str(self.lirchandle))
self.logger.info("Started.")
while (not self.stopFlag.is_set()):
inputready, outputready, exceptready = select.select(inputLirc,[],[], timeout)
s = pylirc.nextcode(1)
if(s):
for code in s:
repeat = code["repeat"]
currentCommand = code["config"]
self.logger.info("New event received: id: = %s, repeat = %s" % (currentCommand, repeat))
try:
self.processorQueue.put_nowait(GeneratorEvent(currentCommand, repeat))
except Queue.Full:
self.logger.error("Processor queue is overloaded.")
self.logger.info("Shutted down.")
def shutdown(self):
self.logger.debug("Shutting down.")
self.stopFlag.set()
pylirc.exit()
def __init__(self, future):
self.error = None
self.finished_event = Event()
self.future = future
self.future.add_callbacks(
callback=self.handle_page,
errback=self.handle_error)
self.res = set()
def _process_inner(function: Callable[..., Any], log_level: str,
log_format: str, start_event: synchronize.Event,
stop_event: synchronize.Event, **kwargs: Any) -> None:
basic_config(level=log_level, log_format=log_format)
start_event.set()
try:
function(**kwargs)
finally:
stop_event.set()
def watch_for_smtps(ready_flag: MP_Event, retq: MP.Queue):
has_smtps = False
ready_flag.set()
start = time.monotonic()
delay = AUTOSTOP_DELAY * 1.5
while (time.monotonic() - start) <= delay:
try:
with SMTP_SSL("localhost", 8025, timeout=0.1) as client:
client.ehlo("exemple.org")
has_smtps = True
break
except Exception:
time.sleep(0.05)
retq.put(has_smtps)
class ASensor(Thread):
'''
La classe ASensor rappresenta un sensore generico, in essa sono implementati:
gli attributi, le configurazioni iniziali comuni a qualunque sensore.
Ogni sensore e' uno Thread in modo tale che il suo flusso di controllo sia indipendente.
'''
def __init__(self, name,typeSensor,pin):
'''
il costruttuore accetta in ingresso:
un nome per identificare il sensore, il tipo del sensore
e il PIN al quale e' collegato sul Raspberry.
Viene creato un oggetto di tipo Subject che rappresenta lo Stream di valori in uscita dal sensore.
Necessario per implementare il paradigma reactive
'''
Thread.__init__(self, None, None, name, None, None, None)
self.name=name
self.typeSensor=typeSensor
self.pin=pin
self.flag = Event()
self.stream=Subject()
'''
Classi che verranno implementate dai singoli sensori
'''
def getValue(self):
pass
def getName(self):
return self.name
def getType(self):
return self.typeSensor
def getPin(self):
return self.pin
def getOutputStream(self):
return self.stream
'''
la funzione initialize configura il PIN per poterlo utilizzare.
Questa procedura e' comune a tutti i sensori (con l'utilizzo della libreria RPi.GPIO)
'''
def stop(self):
self.flag.set()
class Async_Optimize(object):
def __init__(self):
self.callback = lambda *x: None
self.runsignal = Event()
self.SENTINEL = "SENTINEL"
def async_callback_collect(self, q):
while self.runsignal.is_set():
try:
for ret in iter(lambda: q.get(timeout=1), self.SENTINEL):
self.callback(*ret)
self.runsignal.clear()
except Empty:
pass
def opt_async(self, f, df, x0, callback, update_rule=PolakRibiere,
messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
report_every=10, *args, **kwargs):
self.runsignal.set()
c = None
outqueue = None
if callback:
outqueue = Queue()
self.callback = callback
c = Thread(target=self.async_callback_collect, args=(outqueue,))
c.start()
p = _CGDAsync(f, df, x0, update_rule, self.runsignal, self.SENTINEL,
report_every=report_every, messages=messages, maxiter=maxiter,
max_f_eval=max_f_eval, gtol=gtol, outqueue=outqueue, *args, **kwargs)
p.start()
return p, c
def opt(self, f, df, x0, callback=None, update_rule=FletcherReeves,
messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
report_every=10, *args, **kwargs):
p, c = self.opt_async(f, df, x0, callback, update_rule, messages,
maxiter, max_f_eval, gtol,
report_every, *args, **kwargs)
while self.runsignal.is_set():
try:
p.join(1)
if c: c.join(1)
except KeyboardInterrupt:
# print "^C"
self.runsignal.clear()
p.join()
if c: c.join()
if c and c.is_alive():
# self.runsignal.set()
# while self.runsignal.is_set():
# try:
# c.join(.1)
# except KeyboardInterrupt:
# # print "^C"
# self.runsignal.clear()
# c.join()
print("WARNING: callback still running, optimisation done!")
return p.result
async def check_multiprocess_shutdown_event(
shutdown_event: EventType, sleep: Callable[[float], Awaitable[Any]]
) -> None:
while True:
if shutdown_event.is_set():
return
await sleep(0.1)
def execute_worker(
task_queue: Queue,
continue_execution_flag: Value,
i_am_done_event: EventType,
worker_wait_for_exit_event: EventType,
) -> None:
"""Define workers work loop."""
while True:
if not continue_execution_flag.value:
i_am_done_event.set()
worker_wait_for_exit_event.wait()
try:
_ = task_queue.get(block=False)
sleep(0.001)
except Empty:
continue
async def check_shutdown(
shutdown_event: EventType, sleep: Callable[[float], Awaitable[Any]]
) -> None:
while True:
if shutdown_event.is_set():
raise Shutdown()
await sleep(0.1)
class Solver(object):
UNSOLVED = -1
SOLVED = 0
SOLVING = 1
UNSOLVABLE = 2
r"""
@attention: Always reference the model thus: solver.model.xxx
"""
def __init__(self, model, count=1):
self._lock = RLock()
self._model = model
# Context:
self._abort = Event()
self._abort.clear()
self._solvers = Context(count, self._getModel, self._abort, self._lock)
self._isAborted = False
def _getModel(self):
with self.lock():
return self._model
model = property(_getModel)
def __str__(self):
pre="Unsolved"
with self.lock():
if len(list(self._model.unknowns()))==0:
pre="> SOLVED < "
return "\n".join(["%(PRE)sSolver:"%{"PRE":pre}, str(self._model)])
def lock(self):
return self._lock
def serialize(self):
# Retrieve the model ready for storing:
with self.lock():
return self._model.serialize()
def deserialize(self, data):
# Load the model ready for use:
with self.lock():
self._model = self.model.deserialize(data)
def clone(self):
with self.lock():
try:
self.abort()
except Aborted, _e:
pass
self._abort.set()
m = self.model.clone()
self._isAborted = False
return Solver(m)
def __init__(self, model, count=1):
self._lock = RLock()
self._model = model
# Context:
self._abort = Event()
self._abort.clear()
self._solvers = Context(count, self._getModel, self._abort, self._lock)
self._isAborted = False
class PipelineSolver(object):
def __init__(self, model, abort, lock):
self.model = model
self.abort = abort
self.lock = lock
self.event = Event()
self.event.clear()
self.args = []
self.unsolvable = False
self.solved = False
model.addParentListener(self)
self._strategies = StrategiesFactory(model, self._checkAbort, lambda: self.solved, lambda: self.unsolvable)
self._resetPipeline()
def notification(self, who, event=None, args=None):
if who==grid.NAME:
if event in [iModelNotifications.ALL(), iModelNotifications.SOLVED(), iModelNotifications.UNSOLVABLE]:
self.args = []
if event==iModelNotifications.SOLVED():
self.solved = True
if event==iModelNotifications.UNSOLVABLE():
self.unsolvable = True
if event==iModelNotifications.CUSTOM():
return
self.args.append((event, args))
def __call__(self):
try:
if self.unsolvable:
raise UnSolveable(self.model)
if self.solved or self.model.solved:
raise Solved(self.model)
eventCount = self._eventCount()
if len(self.pipeline)>0 or eventCount>0:
self._executePipeline(eventCount)
else:
print "out of rules!"
raise UnSolveable(self.model)
except (Solved, UnSolveable, AbortAck), e:
if isinstance(e, Solved):
self.solved = True
self.model.solved = True
elif isinstance(e, UnSolveable):
self.unsolvable = True
self.model.unsolvable = True
self.args = []
self.pipeline = []
raise
def spin(msg: str, done: synchronize.Event) -> None:
for char in itertools.cycle("|/-\\"):
status = f"\r{char} {msg}"
print(status, end="", flush=True)
if done.wait(0.1):
break
blanks = " " * len(status)
print(f"\r{blanks}\r", end="")
def spin(msg: str, done: synchronize.Event) -> None: # <1>
for char in itertools.cycle(r'\|/-'): # <2>
status = f'\r{char} {msg}' # <3>
print(status, end='', flush=True)
if done.wait(.1): # <4>
break # <5>
blanks = ' ' * len(status)
print(f'\r{blanks}\r', end='') # <6>
def spin(msg: str, done: synchronize.Event) -> None:
for char in itertools.cycle(r'\|/-'):
status = f'\r{char} {msg}'
print(status, end='', flush=True)
if done.wait(.1):
break
blanks = ' ' * len(status)
print(f'\r{blanks}\r', end='')
def enqueue_worker(
workload_publisher_url: str,
task_queue: Queue,
continue_execution_flag: Value,
worker_wait_for_exit_event: EventType,
) -> None:
"""Fill the queue."""
context = Context()
sub_socket = context.socket(SUB)
sub_socket.connect(workload_publisher_url)
sub_socket.setsockopt_string(SUBSCRIBE, "")
while True:
published_data: Dict = sub_socket.recv_json()
if not continue_execution_flag.value:
worker_wait_for_exit_event.wait()
else:
handle_published_data(published_data, task_queue)
def _handle_lock(event_in: synchronize.Event, event_out: synchronize.Event,
path: str) -> None:
"""
Acquire a file lock on given path, then wait to release it. This worker is coordinated
using events to signal when the lock should be acquired and released.
:param multiprocessing.Event event_in: event object to signal when to release the lock
:param multiprocessing.Event event_out: event object to signal when the lock is acquired
:param path: the path to lock
"""
if os.path.isdir(path):
my_lock = lock.lock_dir(path)
else:
my_lock = lock.LockFile(path)
try:
event_out.set()
assert event_in.wait(
timeout=20), 'Timeout while waiting to release the lock.'
finally:
my_lock.release()
def watch_for_tls(ready_flag: MP_Event, retq: MP.Queue):
has_tls = False
req_tls = False
ready_flag.set()
start = time.monotonic()
delay = AUTOSTOP_DELAY * 4
while (time.monotonic() - start) <= delay:
try:
with SMTPClient("localhost", 8025, timeout=0.1) as client:
resp = client.docmd("HELP", "HELO")
if resp == S.S530_STARTTLS_FIRST:
req_tls = True
client.ehlo("exemple.org")
if "starttls" in client.esmtp_features:
has_tls = True
break
except Exception:
time.sleep(0.05)
retq.put(has_tls)
retq.put(req_tls)
def execute_queries(
worker_id: str,
task_queue: Queue,
connection_pool: pool,
continue_execution_flag: Value,
database_id: str,
i_am_done_event: EventType,
worker_wait_for_exit_event: EventType,
) -> None:
"""Define workers work loop."""
with PoolCursor(connection_pool) as cur:
with StorageCursor(STORAGE_HOST, STORAGE_PORT, STORAGE_USER,
STORAGE_PASSWORD, database_id) as log:
succesful_queries: List[Tuple[int, int, str, str, str]] = []
failed_queries: List[Tuple[int, str, str, str]] = []
last_batched = time_ns()
while True:
if not continue_execution_flag.value:
i_am_done_event.set()
worker_wait_for_exit_event.wait()
try:
task: Tuple[str, Tuple[Tuple[Union[str, int],
Optional[str]], ...], str,
str] = task_queue.get(block=False)
query, not_formatted_parameters, workload_type, query_type = task
query = query.replace("[STREAM_ID]", str(worker_id))
formatted_parameters = get_formatted_parameters(
not_formatted_parameters)
endts, latency = execute_task(cur, query,
formatted_parameters)
succesful_queries.append(
(endts, latency, workload_type, query_type, worker_id))
except Empty:
continue
except (ValueError, Error) as e:
failed_queries.append(
(time_ns(), worker_id, str(task), str(e)))
last_batched = log_results(log, last_batched,
succesful_queries, failed_queries)
def __init__(self, model, abort, lock):
self.model = model
self.abort = abort
self.lock = lock
self.event = Event()
self.event.clear()
self.args = []
self.unsolvable = False
self.solved = False
model.addParentListener(self)
self._strategies = StrategiesFactory(model, self._checkAbort, lambda: self.solved, lambda: self.unsolvable)
self._resetPipeline()
async def wslogs(self,
remote_id: 'str',
stop_event: Event,
current_line: int = 0):
""" websocket log stream from remote pea/pod
:param remote_id: the identity of that pea/pod
:param stop_event: the multiprocessing event which marks if stop event is set
:param current_line: the line number from which logs would be streamed
:return:
"""
with ImportExtensions(required=True):
import websockets
try:
# sleeping for few seconds to allow the logs to be written in remote
await asyncio.sleep(3)
async with websockets.connect(
f'{self.log_url}/{remote_id}?timeout=20') as websocket:
await websocket.send(json.dumps({'from': current_line}))
remote_loggers = {}
while True:
log_line = await websocket.recv()
if log_line:
try:
log_line = json.loads(log_line)
current_line = int(list(log_line.keys())[0])
log_line_dict = list(log_line.values())[0]
log_line_dict = json.loads(
log_line_dict.split('\t')[-1].strip())
name = log_line_dict['name']
if name not in remote_loggers:
remote_loggers[name] = JinaLogger(
context=f'🌏 {name}')
# TODO: change logging level, process name in local logger
remote_loggers[name].info(
f'{log_line_dict["message"].strip()}')
except json.decoder.JSONDecodeError:
continue
await websocket.send(json.dumps({}))
if stop_event.is_set():
for logger in remote_loggers.values():
logger.close()
raise RemotePodClosed
except websockets.exceptions.ConnectionClosedOK:
self.logger.debug(f'Client got disconnected from server')
return current_line
except websockets.exceptions.WebSocketException as e:
self.logger.error(
f'Got following error while streaming logs via websocket {repr(e)}'
)
return 0
def __init__(self, name,typeSensor,pin):
'''
il costruttuore accetta in ingresso:
un nome per identificare il sensore, il tipo del sensore
e il PIN al quale e' collegato sul Raspberry.
Viene creato un oggetto di tipo Subject che rappresenta lo Stream di valori in uscita dal sensore.
Necessario per implementare il paradigma reactive
'''
Thread.__init__(self, None, None, name, None, None, None)
self.name=name
self.typeSensor=typeSensor
self.pin=pin
self.flag = Event()
self.stream=Subject()
class PagedResultHandler(object):
def __init__(self, future):
self.error = None
self.finished_event = Event()
self.future = future
self.future.add_callbacks(
callback=self.handle_page,
errback=self.handle_error)
self.res = set()
def handle_page(self, results):
for row in results:
self.res.add(row[0])
if self.future.has_more_pages:
self.future.start_fetching_next_page()
else:
self.finished_event.set()
def handle_error(self, exc):
self.error = exc
self.finished_event.set()
def _worker(queue: JoinableQueue, engine: Engine,
discovery: Dict[str, DiscoveryState], objects: List[Artifact],
lock: LockT, stopswitch: EventT):
while not stopswitch.is_set():
try:
artifact = queue.get(block=True, timeout=0.5)
except QueueEmpty:
continue
try:
new_artifacts = engine.process(artifact, discovery, objects, lock)
for new_artifact in new_artifacts:
queue.put(new_artifact)
finally:
queue.task_done()
def log(self, remote_id: 'str', stop_event: Event, **kwargs) -> None:
""" Start the log stream from remote pea/pod, will use local logger for output
:param remote_id: the identity of that pea/pod
:return:
"""
with ImportExtensions(required=True):
import requests
try:
url = f'{self.log_url}/?{self.kind}_id={remote_id}'
r = requests.get(url=url, stream=True)
for log_line in r.iter_content():
if log_line:
self.logger.info(f'🌏 {log_line.strip()}')
if stop_event.is_set():
break
except requests.exceptions.RequestException as ex:
self.logger.error(
f'couldn\'t connect with remote jinad url {repr(ex)}')
finally:
self.logger.info(f'🌏 exiting from remote logger')
def Event():
'''
Returns an event object
'''
from multiprocessing.synchronize import Event
return Event()
def __init__(self, configurationPath, outputQueue):
self.processorQueue = outputQueue
self.logger = logging.getLogger("generator")
self.lirchandle = pylirc.init("pylirc", configurationPath, False)
self.stopFlag = Event()
class Async_Optimize(object):
callback = lambda *x: None
runsignal = Event()
SENTINEL = "SENTINEL"
def async_callback_collect(self, q):
while self.runsignal.is_set():
try:
for ret in iter(lambda: q.get(timeout=1), self.SENTINEL):
self.callback(*ret)
self.runsignal.clear()
except Empty:
pass
def opt_async(self,
f,
df,
x0,
callback,
update_rule=PolakRibiere,
messages=0,
maxiter=5e3,
max_f_eval=15e3,
gtol=1e-6,
report_every=10,
*args,
**kwargs):
self.runsignal.set()
c = None
outqueue = None
if callback:
outqueue = Queue()
self.callback = callback
c = Thread(target=self.async_callback_collect, args=(outqueue, ))
c.start()
p = _CGDAsync(f,
df,
x0,
update_rule,
self.runsignal,
self.SENTINEL,
report_every=report_every,
messages=messages,
maxiter=maxiter,
max_f_eval=max_f_eval,
gtol=gtol,
outqueue=outqueue,
*args,
**kwargs)
p.start()
return p, c
def opt(self,
f,
df,
x0,
callback=None,
update_rule=FletcherReeves,
messages=0,
maxiter=5e3,
max_f_eval=15e3,
gtol=1e-6,
report_every=10,
*args,
**kwargs):
p, c = self.opt_async(f, df, x0, callback, update_rule, messages,
maxiter, max_f_eval, gtol, report_every, *args,
**kwargs)
while self.runsignal.is_set():
try:
p.join(1)
if c: c.join(1)
except KeyboardInterrupt:
# print "^C"
self.runsignal.clear()
p.join()
if c: c.join()
if c and c.is_alive():
# self.runsignal.set()
# while self.runsignal.is_set():
# try:
# c.join(.1)
# except KeyboardInterrupt:
# # print "^C"
# self.runsignal.clear()
# c.join()
print "WARNING: callback still running, optimisation done!"
return p.result
def __init__(self):
self.callback = lambda *x: None
self.runsignal = Event()
self.SENTINEL = "SENTINEL"
def __init__(self):
self.callback = lambda *x: None
self.runsignal = Event()
self.SENTINEL = "SENTINEL"
def Event():
"""
Returns an event object
"""
from multiprocessing.synchronize import Event
return Event()
def Event():
from multiprocessing.synchronize import Event
return Event()
