def _dispatch_message(self, message):
if not message:
return None
data_splits = message.split('|')
header_type = data_splits[0]
if len(data_splits) <= 1:
self._log.log_message(LogLevel.Warning(), 'message is not valid')
return None
success = self._is_message_valid(message, header_type)
if not success:
self.error_count += 1
if self.error_count == MAX_ERROR_COUNT:
raise Exception('max error count is reached')
self._log.log_message(LogLevel.Warning(),
f'message: "{message}" could not be parsed')
return None
self.error_count = 0
if self._is_command(header_type):
return self._dispatch_command(data_splits)
else:
return self._dispatch_response(data_splits)
async def receive(self, request):
ws = web.WebSocketResponse(heartbeat=1.0)
await ws.prepare(request)
connection_id = str(uuid.uuid1())
ws_connection = WebSocketConnection(ws, connection_id)
self._websockets.add(ws_connection)
await self._send_connection_id_to_ws(ws, connection_id)
# master should propagate the available settings each time a page reloaded
# or new websocket connection is required
self.handle_new_connection(connection_id)
await self.send_configuration(self._app['master_app'].configuration, ws_connection)
self._log.log_message(LogLevel.Debug(), 'websocket connection opened.')
try:
async for msg in ws:
if msg.type == WSMsgType.TEXT:
self.handle_client_message(msg.data)
elif msg.type == WSMsgType.ERROR:
self._log.log_message(LogLevel.Error(), f'ws connection closed with exception: {ws.exception()}')
finally:
pass
self._discard_ws_connection_if_needed()
def on_cmd_message(self, message):
try:
data = json.loads(message.decode('utf-8'))
assert data['type'] == 'cmd'
cmd = data['command']
sites = data['sites']
self.log.log_message(LogLevel.Debug(), f'received command: {cmd}')
if self.site_id not in sites:
self.log.log_message(
LogLevel.Warning(),
f'ignoring TestApp cmd for other sites {sites} (current site_id is {self.site_id})'
)
return
to_exec_command = self.commands.get(cmd)
if to_exec_command is None:
self.log.log_message(LogLevel.Warning(),
'received command not found')
return
to_exec_command(data)
except Exception as e:
self._statemachine.error(str(e))
def wrapped_callback():
try:
cb(*args, **kwargs)
except Exception as e:
if self._statemachine is not None:
self.logger.log_message(LogLevel.Error(), e)
self.logger.log_message(LogLevel.Error(), 'exception in mqtt callback')
self._statemachine.fail(str(e))
else:
raise
def _send_set_log_level(self):
level = {
LogLevel.Debug(): 'Debug',
LogLevel.Info(): 'Info',
LogLevel.Warning(): 'Warning',
LogLevel.Error(): 'Error',
}[self.loglevel]
self.log.log_message(LogLevel.Info(), f'set loglevel to {level}')
self.connectionHandler.send_set_log_level(self.loglevel)
def _execute_cmd_init(self):
self.logger.log_message(LogLevel.Debug(), 'COMMAND: init')
# ToDo: How should we perform the selftest?
self.logger.log_message(LogLevel.Info(), 'running self test...')
selftest_result = self._sequencer_instance.init()
self.logger.log_message(LogLevel.Info(), f'self test completed: {selftest_result}')
self.send_status("IDLE")
# TODO: how to report positive init command results? we could also write the testresult
# self.publish_result(selftest_result, "<insert init result data here>")
# note that currently once "init" failed the status will not be restored to ALIVE
if not selftest_result:
self._mqtt.publish_status(TheTestAppStatusAlive.INITFAIL, {'state': self._statemachine.state, 'payload': {'state': self._statemachine.state, 'message': 'selftest is failed'}})
def does_handler_id_exist(self, param_data: dict):
if not (param_data.get("HANDLER_PROBER") == self.device_handler):
self.log.log_message(LogLevel.Warning(),
'HANDLER name does not match')
return False
return True
def is_lot_number_valid(self, param_data: dict, lot_number):
if not (param_data.get("LOTNUMBER") == lot_number):
self.log.log_message(LogLevel.Warning(),
'lot number could not be found')
return False
return True
def receive(self, timeout_ms):
self.sock.settimeout(timeout_ms)
buffer = BytesIO()
while True:
# read exactly one line from the socket:
received = select.select([self.sock], [], [], timeout_ms / 1000.0)
if not received[0]: # timeout!
return ""
data = self.sock.recv(255)
if len(data) == 0:
# We return NO data in case of a connectionloss,
# which is basically the same as a timeout.
# We'll try to reconnect on the next send. Doing this
# here makes no sense, as we probably have missed parts
# of the packet we were trying to receive anyway.
self.connected = False
self._log.log_message(LogLevel.Warning(),
"Lost connection to DCS6k")
return ""
buffer.write(data)
buffer.seek(0)
for line in buffer:
return line
# If we got here, we did not receive a complete line yet, so we
# move the file pointer to the end and keep on receiving.
buffer.seek(0, 2)
def dispatch_testapp_message(self, topic, msg):
siteid = self.__extract_siteid_from_testapp_topic(topic)
if siteid is None:
# Hacky: To limit the number of routs we subscribed to # on control, thus we get served
# the messages in cmd as well, which don't have a siteid in the topicname. To avoid
# confusing logmessage we return early in that case:
if "cmd" in topic:
return
self.log.log_message(
LogLevel.Warning(),
f'unexpected message on testapp topic {topic}: extracting siteid failed'
)
return
if "testresult" in topic:
self.status_consumer.on_testapp_testresult_changed(siteid, msg)
elif "testsummary" in topic:
self.status_consumer.on_testapp_testsummary_changed(msg)
elif "io-control" in topic:
assert 'type' in msg
assert msg['type'] == 'io-control-request'
assert 'periphery_type' in msg
assert 'ioctl_name' in msg
assert 'parameters' in msg
self.status_consumer.on_testapp_resource_changed(siteid, msg)
elif "status" in topic:
self.status_consumer.on_testapp_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
else:
assert False
def dispatch_testapp_message(self, topic, msg):
siteid = self.__extract_siteid_from_testapp_topic(topic)
if siteid is None:
self.log.log_message(
LogLevel.Warning(), 'unexpected message on testapp topic ' /
+ f'"{topic}": extracting siteid failed')
return
if "testresult" in topic:
self.status_consumer.on_testapp_testresult_changed(siteid, msg)
elif "testsummary" in topic:
self.status_consumer.on_testapp_testsummary_changed(msg)
elif "peripherystate" in topic:
assert 'type' in msg
assert msg['type'] == 'io-control-request'
assert 'periphery_type' in msg
assert 'ioctl_name' in msg
assert 'parameters' in msg
self.status_consumer.on_testapp_resource_changed(siteid, msg)
elif "status" in topic:
self.status_consumer.on_testapp_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
elif "binsettings" in topic:
self.status_consumer.on_testapp_bininfo_message(siteid, msg)
else:
assert False
def on_handler_command_message(self, msg: dict):
cmd = msg.get('type')
payload = msg.get('payload')
try:
{
'load':
lambda param_data: self.load_command(param_data),
'next':
lambda param_data: self.next(param_data),
'unload':
lambda param_data: self.unload(param_data),
'reset':
lambda param_data: self.reset(param_data),
'identify':
lambda param_data: self._send_tester_identification(param_data
),
'get-state':
lambda param_data: self._send_tester_state(param_data),
}[cmd](payload)
except KeyError:
self.log.log_message(LogLevel.Error(),
f'Failed to execute command: {cmd}')
except MachineError:
self.on_error(
f'cannot trigger command "{cmd}" from state "{self.fsm.model.state}"'
)
def get_test_information(self, param_data: dict):
for _, value in param_data.get("STATION").items():
if self.does_device_id_exist(value):
return self.remove_digit_from_keys(value)
self.log.log_message(LogLevel.Warning(), 'device information in station section are missed')
return None
def on_message(self, client, userdata, message):
try:
result = self.on_request(message.payload)
result_json = json.dumps(result)
self.mqtt_client.publish(f"ate/{self.device_id}/{self.actuator_type}/io-control/response", result_json)
except json.JSONDecodeError as error:
self._log.log_message(LogLevel.Error(), f'{error}')
def dispatch_command(self, json_data):
cmd = json_data.get('command')
try:
{
'load':
lambda param_data: self.load_command(param_data),
'next':
lambda param_data: self.next(param_data),
'unload':
lambda param_data: self.unload(param_data),
'reset':
lambda param_data: self.reset(param_data),
'usersettings':
lambda param_data: self.usersettings_command(param_data),
'getresults':
lambda param_data: self._handle_command_with_response(
param_data),
'getlogs':
lambda param_data: self._handle_command_with_response(
param_data),
'getlogfile':
lambda param_data: self._handle_command_with_response(
param_data),
'setloglevel':
lambda param_data: self.setloglevel(param_data),
}[cmd](json_data)
except Exception as e:
self.log.log_message(LogLevel.Error(),
f'Failed to execute command {cmd}: {e}')
def does_file_exist(self):
if not os.path.exists(self.fullpath):
self.log.log_message(LogLevel.Error(),
f'file not found: {self.fullpath}')
return False
return True
def publish_state(self, site_id=None, param_data=None):
if self.prev_state == self.state:
return
self.prev_state = self.state
self.log.log_message(LogLevel.Info(), f'Master state is {self.state}')
self.connectionHandler.publish_state(self.external_state)
def __get_config_parameters(self, config: dict, parameters: List[str]) -> Optional[str]:
for param in parameters:
if config.get(param) is None:
self.log.log_message(LogLevel.Error(), f"failed to get '{param}' parameter")
return None
return config
def is_station_valid(self, param_data: dict):
for _, value in param_data.items():
if self.does_device_id_exist(value):
return True
self.log.log_message(LogLevel.Warning(), 'device id mismatch in station section')
return False
def does_device_id_exist(self, param_data: dict):
for key, value in param_data.items():
if key == 'TESTER' or re.match(r'TESTER\d', key) is not None:
if self.device_id == value:
return True
self.log.log_message(LogLevel.Warning(), 'tester could not be found')
return False
def publish_current_state(self, info):
self._conhandler.publish_state(self.state, self._error_message)
if self.prev_state != self.state:
self.log.log_message(LogLevel.Info(),
f'control state is: {self.state}')
self.prev_state = self.state
def _execute_cmd_next(self, job_data: Optional[dict]):
self.logger.log_message(LogLevel.Debug(), 'COMMAND: next')
self.send_status("TESTING")
result = self._sequencer_instance.run(self._execution_policy, job_data)
self.send_status("IDLE")
self.send_testresult(result)
def decode_payload(self, payload_bytes):
try:
payload = json.loads(payload_bytes)
return payload
except json.JSONDecodeError as error:
self.log.log_message(LogLevel.Error(), f'{error}')
return None
def on_reset(self, _):
try:
if self.process:
self._terminate()
except Exception as e:
self.log.log_message(LogLevel.Error(),
f"could not terminate testapp properly: {e}")
self.to_idle(_)
def __init__(self, configuration, comm):
self._mqtt = None
self._log = Logger('handler', self._mqtt)
loglevel = LogLevel.Warning() if configuration.get('loglevel') is None else configuration['loglevel']
self._log.set_logger_level(loglevel)
self.comm = comm
self._get_configuration(configuration)
self._machine = HandlerStateMachine()
def on_cmd_message(self, message):
payload = self.decode_message(message)
to_exec_command = self.commands.get(payload.get("value"))
if to_exec_command is None:
self.log.log_message(LogLevel.Warning(),
'received command not found')
return False
return to_exec_command()
def apply_configuration(self, configuration: dict):
try:
self.configuredSites = configuration['sites']
# Sanity check for bad configurations:
if len(self.configuredSites) == 0:
self.log.log_message(LogLevel.Error(),
'Master got no sites assigned')
sys.exit()
self.device_id = configuration['device_id']
self.broker_host = configuration['broker_host']
self.broker_port = configuration['broker_port']
self.enableTimeouts = configuration['enable_timeouts']
self.handler_id = configuration['Handler']
self.env = configuration['environment']
except KeyError as e:
self.log.log_message(LogLevel.Error(),
f'Master got invalid configuration: {e}')
sys.exit()
def handle_uncaught_exceptions_from_executor(f):
try:
f.result()
except Exception:
self.logger.log_message(LogLevel.Error(), "executor exception")
# TODO: we probably don't want to keep running if any
# exception escaped. using os._exit may not be
# the best way to do this (because no cleanup/io
# flushing at all)
os._exit(1)
def send_load_test_to_all_sites(self, testapp_params):
topic = f'ate/{self.device_id}/Control/cmd'
params = {
'type': 'cmd',
'command': 'loadTest',
'testapp_params': testapp_params,
'sites': self.sites,
}
self.log.log_message(LogLevel.Info(), 'Send LoadLot to sites...')
self.mqtt.publish(topic, json.dumps(params), 0, False)
def _get_configuration(self, configuration):
try:
self.handler_type = configuration['handler_type']
self.handler_id = configuration['handler_id']
self.device_ids = configuration['device_ids']
self.broker = configuration['broker_host']
self.broker_port = configuration['broker_port']
except KeyError as e:
self._log.log_message(LogLevel.Error(), f'Handler got invalid configuration: {e}')
raise