def _dispatch_message(self, message):
if not message:
return None
data_splits = message.split('|')
header_type = data_splits[0]
if len(data_splits) <= 1:
self._log.log_message(LogLevel.Warning(), 'message is not valid')
return None
success = self._is_message_valid(message, header_type)
if not success:
self.error_count += 1
if self.error_count == MAX_ERROR_COUNT:
raise Exception('max error count is reached')
self._log.log_message(LogLevel.Warning(),
f'message: "{message}" could not be parsed')
return None
self.error_count = 0
if self._is_command(header_type):
return self._dispatch_command(data_splits)
else:
return self._dispatch_response(data_splits)
def on_cmd_message(self, message):
try:
data = json.loads(message.decode('utf-8'))
assert data['type'] == 'cmd'
cmd = data['command']
sites = data['sites']
self.log.log_message(LogLevel.Debug(), f'received command: {cmd}')
if self.site_id not in sites:
self.log.log_message(
LogLevel.Warning(),
f'ignoring TestApp cmd for other sites {sites} (current site_id is {self.site_id})'
)
return
to_exec_command = self.commands.get(cmd)
if to_exec_command is None:
self.log.log_message(LogLevel.Warning(),
'received command not found')
return
to_exec_command(data)
except Exception as e:
self._statemachine.error(str(e))
def dispatch_testapp_message(self, topic, msg):
siteid = self.__extract_siteid_from_testapp_topic(topic)
if siteid is None:
# Hacky: To limit the number of routs we subscribed to # on control, thus we get served
# the messages in cmd as well, which don't have a siteid in the topicname. To avoid
# confusing logmessage we return early in that case:
if "cmd" in topic:
return
self.log.log_message(
LogLevel.Warning(),
f'unexpected message on testapp topic {topic}: extracting siteid failed'
)
return
if "testresult" in topic:
self.status_consumer.on_testapp_testresult_changed(siteid, msg)
elif "testsummary" in topic:
self.status_consumer.on_testapp_testsummary_changed(msg)
elif "io-control" in topic:
assert 'type' in msg
assert msg['type'] == 'io-control-request'
assert 'periphery_type' in msg
assert 'ioctl_name' in msg
assert 'parameters' in msg
self.status_consumer.on_testapp_resource_changed(siteid, msg)
elif "status" in topic:
self.status_consumer.on_testapp_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
else:
assert False
def get_test_information(self, param_data: dict):
for _, value in param_data.get("STATION").items():
if self.does_device_id_exist(value):
return self.remove_digit_from_keys(value)
self.log.log_message(LogLevel.Warning(), 'device information in station section are missed')
return None
def is_station_valid(self, param_data: dict):
for _, value in param_data.items():
if self.does_device_id_exist(value):
return True
self.log.log_message(LogLevel.Warning(), 'device id mismatch in station section')
return False
def does_handler_id_exist(self, param_data: dict):
if not (param_data.get("HANDLER_PROBER") == self.device_handler):
self.log.log_message(LogLevel.Warning(),
'HANDLER name does not match')
return False
return True
def is_lot_number_valid(self, param_data: dict, lot_number):
if not (param_data.get("LOTNUMBER") == lot_number):
self.log.log_message(LogLevel.Warning(),
'lot number could not be found')
return False
return True
def dispatch_testapp_message(self, topic, msg):
siteid = self.__extract_siteid_from_testapp_topic(topic)
if siteid is None:
self.log.log_message(
LogLevel.Warning(), 'unexpected message on testapp topic ' /
+ f'"{topic}": extracting siteid failed')
return
if "testresult" in topic:
self.status_consumer.on_testapp_testresult_changed(siteid, msg)
elif "testsummary" in topic:
self.status_consumer.on_testapp_testsummary_changed(msg)
elif "peripherystate" in topic:
assert 'type' in msg
assert msg['type'] == 'io-control-request'
assert 'periphery_type' in msg
assert 'ioctl_name' in msg
assert 'parameters' in msg
self.status_consumer.on_testapp_resource_changed(siteid, msg)
elif "status" in topic:
self.status_consumer.on_testapp_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
elif "binsettings" in topic:
self.status_consumer.on_testapp_bininfo_message(siteid, msg)
else:
assert False
def receive(self, timeout_ms):
self.sock.settimeout(timeout_ms)
buffer = BytesIO()
while True:
# read exactly one line from the socket:
received = select.select([self.sock], [], [], timeout_ms / 1000.0)
if not received[0]: # timeout!
return ""
data = self.sock.recv(255)
if len(data) == 0:
# We return NO data in case of a connectionloss,
# which is basically the same as a timeout.
# We'll try to reconnect on the next send. Doing this
# here makes no sense, as we probably have missed parts
# of the packet we were trying to receive anyway.
self.connected = False
self._log.log_message(LogLevel.Warning(),
"Lost connection to DCS6k")
return ""
buffer.write(data)
buffer.seek(0)
for line in buffer:
return line
# If we got here, we did not receive a complete line yet, so we
# move the file pointer to the end and keep on receiving.
buffer.seek(0, 2)
def does_device_id_exist(self, param_data: dict):
for key, value in param_data.items():
if key == 'TESTER' or re.match(r'TESTER\d', key) is not None:
if self.device_id == value:
return True
self.log.log_message(LogLevel.Warning(), 'tester could not be found')
return False
def __init__(self, configuration, comm):
self._mqtt = None
self._log = Logger('handler', self._mqtt)
loglevel = LogLevel.Warning() if configuration.get('loglevel') is None else configuration['loglevel']
self._log.set_logger_level(loglevel)
self.comm = comm
self._get_configuration(configuration)
self._machine = HandlerStateMachine()
def _send_set_log_level(self):
level = {
LogLevel.Debug(): 'Debug',
LogLevel.Info(): 'Info',
LogLevel.Warning(): 'Warning',
LogLevel.Error(): 'Error',
}[self.loglevel]
self.log.log_message(LogLevel.Info(), f'set loglevel to {level}')
self.connectionHandler.send_set_log_level(self.loglevel)
def __init__(self, configuration):
self.site_id = configuration['site_id']
self.device_id = configuration['device_id']
self.broker_host = configuration["broker_host"]
self.broker_port = configuration["broker_port"]
self.configuration = configuration
self.log = Logger(f'control {self.site_id}')
self.log_level = LogLevel.Warning() if configuration.get(
'loglevel') is None else configuration['loglevel']
self.log.set_logger_level(self.log_level)
def on_cmd_message(self, message):
payload = self.decode_message(message)
to_exec_command = self.commands.get(payload.get("value"))
if to_exec_command is None:
self.log.log_message(LogLevel.Warning(),
'received command not found')
return False
return to_exec_command()
def on_unexpected_control_state(self, site_id, state):
self.log.log_message(
LogLevel.Warning(),
f'Site {site_id} reported state {state}. This state is ignored during startup.'
)
self.error_message = f'Site {site_id} reported state {state}'
if self.state == 'connecting' and state == 'busy':
self.on_error(
f"master can't handle control {site_id}'s state 'busy' during statup"
)
def on_unexpected_testapp_state(self, site_id, state):
self.log.log_message(
LogLevel.Warning(),
f'TestApp for site {site_id} reported state {state}. This state is ignored during startup.'
)
self.error_message = f'TestApp for site {site_id} reported state {state}'
if self.state == 'connecting' and state == 'idle':
self.on_error(
f"master can't handle testApp {site_id}'s state 'idle' during statup"
)
def send(self, data):
totalsent = 0
msglen = len(data)
while totalsent < msglen:
sent = self.sock.send(data[totalsent:])
if sent == 0:
self._log.log_message(
LogLevel.Warning(),
"Lost connection to DCS6k, reconnect in progress..")
self.do_connect(self.target_ip, self.target_port)
sent = 0 # Restart sending in case of a connection loss.
totalsent = totalsent + sent
def generate_command_msg(self, cmd_type: str, job_data: dict) -> Optional[str]:
try:
command = {"load": lambda: self.__generate_load_test_command(job_data),
"next": lambda: self.__generate_next_command(job_data),
"endlot": lambda: self.__generate_endlot_command(),
"get_state": lambda: self.__generate_get_state_command(),
"identify": lambda: self.__generate_identify_command(),
}[cmd_type]()
except KeyError:
self._log.log_message(LogLevel.Warning(), f"failed to interpret command '{cmd_type}' in {sys._getframe().f_code.co_filename}")
return None
return self.__dump_message(self.__generate_msg(cmd_type, command))
def _generate_command_from_message(self, message):
try:
command = {
'identify': lambda: self._generate_get_handler_name_command(),
'get-state':
lambda: self._generate_get_handler_state_command()
}[message['type']]()
return command
except KeyError:
self._log.log_message(
LogLevel.Warning(),
f'master command type could not be recognized: {message["type"]}'
)
return None
async def _communicate(self):
while True:
command = await self._handler.read()
if not command:
continue
try:
self._message_queue.put_nowait(command)
self._event.set()
except QueueFull:
self._log.log_message(
LogLevel.Warning(),
'communication handler cannot handle commands any more, queue is full!'
)
def dispatch_control_message(self, topic, msg):
siteid = self.__extract_siteid_from_control_topic(topic)
if siteid is None:
self.log.log_message(
LogLevel.Warning(), 'unexpected message on control topic ' /
+ f'"{topic}": extracting siteid failed')
return
if "status" in topic:
self.status_consumer.on_control_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
else:
assert False
def run(self, site_num: int):
start = time.time()
exception = False
try:
self.do()
except (NameError, AttributeError) as e:
raise Exception(e)
except Exception as e:
self.logger.log_message(LogLevel.Warning(), e)
exception = True
end = time.time()
self._execution_time += end - start
self._test_executions += 1
return self.aggregate_test_result(site_num, exception), exception
def get_test_information(self, param_data: dict):
test_information = {}
test_information['PACKAGE_ID'] = param_data['MAIN']['PACKAGEID_SHORT']
test_information['SUBLOT_ID'] = param_data['MAIN']['LOTNUMBER'].split(
'.')[1]
for _, value in param_data.get("STATION").items():
if self.does_device_id_exist(value):
test_information.update(self.remove_digit_from_keys(value))
test_information[
'PART_ID'] = f"{param_data['MAIN']['MATCHCODE']}_{test_information['TESTERPRG']}"
return test_information
self.log.log_message(
LogLevel.Warning(),
'device information in station section are missed')
return None
def _generate_response_from_message(self, message):
try:
response = {
'status': lambda: self._generate_status_response(
message
), # TODO: define a share state interface between handle and master
'identify': lambda: self._generate_test_name_response(message),
'next': lambda: self._generate_end_test_response(message),
'get-state':
lambda: self._generate_tester_status_response(message),
'error': lambda: self._generate_error_message(message)
}[message['type']]()
return response
except KeyError:
self._log.log_message(
LogLevel.Warning(),
f'master response type could not be recognized: {message["type"]}"'
)
def __init__(self, configuration):
self.sites = configuration['sites']
super().__init__(self.sites)
self.fsm = Machine(model=self,
states=MasterApplication.states,
transitions=MasterApplication.transitions,
initial="startup",
after_state_change='publish_state')
self.configuration = configuration
self.log = Logger('master')
self.loglevel = LogLevel.Warning() if configuration.get(
'loglevel') is None else configuration['loglevel']
self.log.set_logger_level(self.loglevel)
self.apply_configuration(configuration)
self.init()
self.connectionHandler = MasterConnectionHandler(
self.broker_host, self.broker_port, self.configuredSites,
self.device_id, self.handler_id, self)
self.peripheral_controller = PeripheralController(
self.connectionHandler.mqtt, self.device_id)
self.received_site_test_results = []
self.received_sites_test_results = ResultsCollector(
MAX_NUM_OF_TEST_PROGRAM_RESULTS)
self.loaded_jobname = ""
self.loaded_lot_number = ""
self.error_message = ''
self.logs = []
self.prev_state = ''
self.summary_counter = 0
self.sites_to_test = []
self.command_queue = Queue(maxsize=50)
self.prr_rec_information = {}
self.sites_yield_information = {}
# TODO: bin settings should be initialized in an other stage
self._bin_settings = {'type 1': {'sbins': [1, 2, 3], 'hbins': [1]}}
self._yield_info_handler = YieldInformationHandler()
self._yield_info_handler.set_bin_settings(self._bin_settings)
self.test_results = []
self.dummy_partid = 1
def dispatch_control_message(self, topic, msg):
siteid = self.__extract_siteid_from_control_topic(topic)
if siteid is None:
# Hacky: To limit the number of routs we subscribed to # on control, thus we get served
# the messages in cmd as well, which don't have a siteid in the topicname. To avoid
# confusing logmessage we return early in that case:
if "cmd" in topic:
return
self.log.log_message(
LogLevel.Warning(),
f'unexpected message on control topic {topic}: extracting siteid failed'
)
return
if "status" in topic:
self.status_consumer.on_control_status_changed(siteid, msg)
elif "log" in topic:
self.status_consumer.on_log_message(siteid, msg)
else:
assert False
def __init__(self, configuration):
self.sites = configuration['sites']
super().__init__(self.sites)
self.fsm = Machine(model=self,
states=MasterApplication.states,
transitions=MasterApplication.transitions,
initial="startup",
after_state_change='publish_state')
self.configuration = configuration
self.log = Logger('master')
self.loglevel = LogLevel.Warning() if configuration.get(
'loglevel') is None else configuration['loglevel']
self.log.set_logger_level(self.loglevel)
self.apply_configuration(configuration)
self.init()
self.connectionHandler = MasterConnectionHandler(
self.broker_host, self.broker_port, self.configuredSites,
self.device_id, self.handler_id, self)
self.peripheral_controller = PeripheralController(
self.connectionHandler.mqtt, self.device_id)
self.received_site_test_results = []
self.received_sites_test_results = ResultsCollector(
MAX_NUM_OF_TEST_PROGRAM_RESULTS)
self.loaded_jobname = ""
self.loaded_lot_number = ""
self.error_message = ''
self.prev_state = ''
self.summary_counter = 0
self.sites_to_test = []
self.command_queue = Queue(maxsize=50)
self._result_info_handler = ResultInformationHandler(self.sites)
self.test_results = []
self.dummy_partid = 1
self._first_part_tested = False
self._stdf_aggregator = None
def handle_message(self, message):
message_type = message['type']
try:
expected_master_state = {'load': lambda: self._on_load_command_issued(),
'next': lambda: self._on_next_command_issued(),
'unload': lambda: self._on_unload_command_issued()}[message_type]()
except KeyError:
expected_master_state = [MasterStates.unknown()]
except MachineError:
self._log.log_message(LogLevel.Warning(), f'cannot trigger command "{message_type}" from state {self.state}')
return self._get_error_message(message_type)
finally:
self._log.log_message(LogLevel.Info(), f'Handler reports message: {message}')
if expected_master_state[0] == MasterStates.unknown():
return {}
self.pending_transistion_master = SequenceContainer(expected_master_state, self._device_ids, lambda: self._get_call_back(expected_master_state[0]),
lambda site, state: self._on_unexpected_master_state(site, state))
return {}
def on_load_error(self):
self.log.log_message(LogLevel.Warning(), self.error_message)