def __init__(self, args):
"""
Create instance of application controller.
:param args: raw arguments dictionary
:param logger: logger function to use
"""
# parse queue configuration based on additional arguments passed
conf = QueueConf()
conf.setAllConf(args)
# log options processed
all_options = [
" %s -> %s" % (key, value) for key, value in conf.copy().items()
]
logger.debug("Configuration:\n%s", "\n".join(all_options))
# resolve options either directly or through session and other services
self.mongodb_url = self._validate_mongodb_url(
conf.getConfString(const.OPT_MONGODB_URL))
self.working_dir = self._validate_working_dir(
conf.getConfString(const.OPT_WORKING_DIR))
self.service_dir = self._validate_service_dir(
conf.getConfString(const.OPT_SERVICE_DIR))
self.session = self._create_session(conf)
self.client = pymongo.MongoClient(self.mongodb_url)
self.api = RestApiDispatcher(self.session)
def init_tcp(self):
self._tp = paramiko.Transport(self._socket, gss_kex=True)
self._tp.set_gss_host(socket.getfqdn(""))
self._tp.load_server_moduli()
host_key = paramiko.RSAKey(filename=conf.SSH_SERVER_RSA_EKY)
self._tp.add_server_key(host_key)
server = Server()
logger.debug('Create Server Class')
try:
self._tp.start_server(server=server)
except paramiko.SSHException:
logger.error("SSH negotiation failed.")
gvar.manager.close_connection(self)
exit()
self._channel = self._tp.accept(200)
if self._channel is None:
logger.error('No channel')
gvar.manager.close_connection(self)
exit()
logger.info('Authenticatied!')
def iteration(self):
"""
Run single iteration, entire logic of executor should be specified in this method, unless
there is an additional logic between iterations. Iteration is cancelled, if executor is
terminated.
:return: boolean flag, True - run next iteration, False - terminate
"""
# we process special case of terminated executor in case someone would launch it again.
if self._terminated:
logger.warning("Executor %s has been terminated", self.name)
return False
logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout)
try:
# send reponse to the scheduler that this executor is up and processing tasks
self._respond_is_alive()
# check if there are any messages in connection, process one message per iteration
if self.conn.poll():
self._process_message(self.conn.recv())
# check if there is any outstanding task to run, otherwise poll data for current task
self._process_task()
except ExecutorInterruptedException:
logger.info("%s - Requested termination of executor", self.name)
self._terminated = True
# cancel task that is currently running and clean up state
self._cancel_active_task()
return False
# pylint: disable=W0703,broad-except
except Exception as e:
logger.exception("%s - Unrecoverable error %s, terminating", self.name, e)
self._terminated = True
return False
# pylint: enable=W0703,broad-except
else:
return True
def run(self):
for i in range(0, self._iterations):
if self.is_cancelled():
logger.debug("Cancelled on iteration %s", i)
break
logger.debug("Iteration %s", i)
time.sleep(1.0)
def parse_or_expression(self) -> Result:
"""
or_exp := or_exp OR and_exp
"""
r1 = self.parse_and_expression()
if not r1.name and not r1.value:
return r1
while True:
token = self.pop_token()
if token.type == TOKEN_TYPE['eof']:
break
elif token.type != TOKEN_TYPE['or']:
self.push_token(token)
return r1
r2 = self.parse_and_expression()
if not r2.name and not r2.value:
raise ParseException('invalid condition "%s"', self.condstr)
r1 = Result('({} or {})'.format(
r1.name, r2.name), r1.value or r2.value)
logger.debug('[*] {}'.format(r1))
return r1
def recv(self):
try:
stream = self._socket.recv(1024)
except OSError:
logger.debug('telnet connect socket was closed.')
raise SocketClosed
return stream
def send_request(self):
bs = self.get_proto_head(0x0)
self.send(bs)
self.status = self.WAIT_ROOM_ID_STATUS
logger.debug('Regist: send request, bs:%s ,now status %s' %
(bs.get_packet(), self.status))
def measure_point(self) -> Optional[Tuple[float, float]]:
self.received = 0
rssi_values: List[int] = []
start = time.time()
while time.time() - start < self.timeout and self.received < self.points_number:
bytes_to_read = self.serial_conn.ser.inWaiting()
text_read = self.serial_conn.ser.read(bytes_to_read).decode(
encoding="utf-8"
)
if text_read:
args = text_read.split(" ")
for arg in args:
if "RSSI" in arg:
rssi = arg.split("=")[1].strip()
logger.debug(f"Read RSSI value from serial: {rssi}")
try:
rssi_values.append(int(rssi))
self.received += 1
except Exception as ex:
print(ex)
time.sleep(0.1)
try:
return (
statistics.mean(rssi_values),
(len(rssi_values) / self.points_number) * 100,
)
except statistics.StatisticsError:
return None
def start_listening(self):
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.listener.bind(('127.0.0.1', self._listen_port))
self.listener.listen()
logger.debug('Telnet.Start_linsten OK...')
def run(self):
for i in range(0, self._iterations):
if self.is_cancelled():
logger.debug("Cancelled on iteration %s", i)
break
logger.debug("Iteration %s", i)
time.sleep(1.0)
def write(self, c: int):
c = chr(c).encode()
try:
self.port.write(c)
# except serial.serialutil.SerialTimeoutException:
except Exception:
logger.debug("Serial write timeout")
self.close()
def recv(self):
res = self.room.recv()
logger.debug('Register recv message: %s' % res)
p = Protocol(res)
first_code = p.get_int8()
if first_code != 0x1:
return b'' # ! bad idea
return p
def test(self, components: Tuple[str]):
self.load_plugins()
for component in self.iter_components():
if not component.name in components:
continue
logger.debug("test '%s' check matches", component.name)
self._process_check_matches_result(component)
self._process_implies()
return self.results
def recv_from_control(self, msg):
"""
recv from control, then send to channel
"""
try:
self.send_to_channel(msg)
except OSError:
logger.debug("send error, close this room: %s" % self.id)
gvar.mansion.del_room_by_id(self.id)
def send_room_id(self):
"""
| 0x1 | room_id | server terminal listening port |
"""
bs = self.get_protocol_head(0x1)
bs.add_str(self._room.id)
bs.add_str(str(conf.SSH_SERVER_LISTENING_TERMINIAL_PORT))
self.send(bs)
self.status = self.WAIT_ROOM_ID_STATUS
logger.debug('Register: send room id: %s' % bs.get_packet())
def run(self):
"""
alias self.recv()
"""
try:
self.recv()
except Exception:
logger.debug("connection got a error, closed")
gvar.manager.close_connection(self)
exit()
def update_map_with_rssi_values(
self, id: int, rssi: float, perc: float
) -> shp.ShapeRecord:
logger.debug(f"Update id {id} with RSSI {rssi} and {perc}%")
for shaperec in self.shape_records:
if shaperec.record["ID"] == id:
shaperec.record["RSSI"] = rssi
shaperec.record["PERC"] = perc
return shaperec
raise IndexError(f"Given id ({id}) was not found")
def get_fin_room_id(self, pro: Protocol):
if not self.status == self.WAIT_ROOM_ID_STATUS:
self.send_error()
return
room_id = pro.get_str()
if not self._room.id == room_id:
self.send_error()
return
self.status = self.FULL_STATUS
logger.debug('Register: Full Status')
def interpret_group_8(self, instruction: int) -> None:
# Logic and arithmetic operations between Vx and Vy
x = (instruction & 0x0F00) >> 8
y = (instruction & 0x00F0) >> 4
last_nibble = instruction & 0x000F
logger.debug(
f"instruction:{instruction:X} x:{x:X}(0x{self.reg_v[x]:X})"
f" y:{y:X}(0x{self.reg_v[y]:X})")
if last_nibble == 0x0: # LD Vx, Vy
self.reg_v[x] = self.reg_v[y]
elif last_nibble == 0x1: # OR Vx, Vy
self.reg_v[x] = self.reg_v[x] | self.reg_v[y]
elif last_nibble == 0x2: # AND Vx, Vy
self.reg_v[x] = self.reg_v[x] & self.reg_v[y]
elif last_nibble == 0x3: # XOR Vx, Vy
self.reg_v[x] = self.reg_v[x] ^ self.reg_v[y]
elif last_nibble == 0x4: # ADD Vx, Vy
# Vf is used as carry
if self.reg_v[x] + self.reg_v[y] > 0xFF:
self.reg_v[0xF] = 1
else:
self.reg_v[0xF] = 0
self.reg_v[x] = (self.reg_v[x] + self.reg_v[y]) % 0x100
elif last_nibble == 0x5: # SUB Vx, Vy
if self.reg_v[x] <= self.reg_v[y]:
tmp = self.reg_v[x] - self.reg_v[y]
self.reg_v[x] = 0x100 + tmp # + since tmp is negative
self.reg_v[0xF] = 0
else:
self.reg_v[x] = self.reg_v[x] - self.reg_v[y]
self.reg_v[0xF] = 1
elif last_nibble == 0x6: # SHR Vx {, Vy}
# Vf shall be set to same value as the bit that is shifted out
if self.shift_quirks:
self.reg_v[x] = self.reg_v[y]
self.reg_v[0xF] = self.reg_v[x] & 0x0001
self.reg_v[x] = self.reg_v[x] >> 1
elif last_nibble == 0x7: # SUBN Vx, Vy
if self.reg_v[y] <= self.reg_v[x]:
tmp = self.reg_v[y] - self.reg_v[x]
self.reg_v[x] = 0x100 + tmp
self.reg_v[0xF] = 0
else:
self.reg_v[x] = self.reg_v[y] - self.reg_v[x]
self.reg_v[0xF] = 1
elif last_nibble == 0xE: # SHL Vx {, Vy}
# Vf shall be set to same value as the bit that is shifted out
if self.shift_quirks:
self.reg_v[x] = self.reg_v[y]
self.reg_v[0xF] = (self.reg_v[x] & 0x80) >> 7
self.reg_v[x] = (self.reg_v[x] << 1) % 0x100
else:
logger.warning(f"OpCode {instruction:X} supported ")
def close_connection(self, conn):
"""
close a connection
"""
logger.debug("close the connection")
if not conn.in_room(): # if not in a room
conn.close()
return
rid = conn.room_id()
self._rlist[rid].close_connection(conn)
def iter_files(root_dir: str, ignore_dirs=[]) -> Generator[Tuple[str, str], None, None]:
"""Iterate all components in directory `root_dir`
:returns (root, filename)
"""
logger.debug("iter components: '%s'", root_dir)
for (root, dirs, files) in os.walk(root_dir):
for ignore_dir in ignore_dirs:
if ignore_dir in dirs:
dirs.remove(ignore_dir)
for filename in files:
yield (root, filename)
def _update_is_alive(self, messages):
"""
Update 'is alive' status for executors. Currently just updates datetime of message.
:param messages: list of Message instances with EXECUTOR_IS_ALIVE status
"""
for msg in messages:
if "name" in msg.arguments:
exc_name = msg.arguments["name"]
self.is_alive_statuses[exc_name] = util.utcnow()
logger.debug("Updated 'is alive' status for executor %s", exc_name)
def read(self):
parser = configparser.ConfigParser()
if os.path.isfile(self.config_file_path):
parser.read(self.config_file_path)
program_data = ProgramData(**parser._sections["Settings"])
else:
logger.debug(
f"Config file not found. Creating file {self.config_file_path}"
)
self.update(DEFAULT_PROGRAM_DATA)
program_data = DEFAULT_PROGRAM_DATA
return program_data
def _update_is_alive(self, messages):
"""
Update 'is alive' status for executors. Currently just updates datetime of message.
:param messages: list of Message instances with EXECUTOR_IS_ALIVE status
"""
for msg in messages:
if "name" in msg.arguments:
exc_name = msg.arguments["name"]
self.is_alive_statuses[exc_name] = util.utcnow()
logger.debug("Updated 'is alive' status for executor %s",
exc_name)
def select_room(self, conn):
"""
Enter a room, need select a room ID, and give a username
"""
logger.debug("into select room process...")
se = Select(self._rlist.keys(), conn)
room_id, username = se.process()
logger.info('get room_id: %s, username: %s' % (room_id, username))
room = self._rlist[room_id]
conn.username = username
room.add_connection(conn)
del se
def wait_keyboard_interrupt(self):
import time
logger.debug("Into waiting for keyboard interrupt")
try:
while gvar.thread.has_alive_thread():
gvar.thread.clean_stoped_thread()
time.sleep(50)
except KeyboardInterrupt:
logger.info("shutdown the program...")
self.shutdown()
gvar.thread.kill_all_thread()
logger.info("Bye!")
def _safe_exec(self, func, **kwargs):
"""
Safely execute function with a list of arguments. Function is assumed not to return any
result.
:param func: function to execute
:param kwargs: dictionary of method parameters
"""
# pylint: disable=W0703,broad-except
try:
if func:
func(**kwargs)
except Exception as e:
logger.debug("%s - Failed to execute '%s(%s)', reason=%s", self.name, func, kwargs, e)
def load_script_data(self):
error = None
xpath = (
'/html/body/div[3]/div[2]/div/div[2]/div[1]/div[7]/table/tbody/tr/td/a'
)
res = self.driver.find_element_by_xpath(xpath)
while not error:
try:
res.click()
sleep(1)
except BaseException:
logger.exception('i know about it!')
error = True
logger.debug('load script data')
def _process_task(self):
"""
Process individual task, returns exit code for each task following available API. One of
the checks is performed to test current task_id against cancelled list, and discard task,
if it has been marked as cancelled, or terminate running task.
"""
if not self._active_task:
self._active_task = self._get_new_task()
logger.info("%s - New task registered", self.name)
# before checking statuses and proceed execution, we check if current task was
# requested to be cancelled, if yes, we remove it from set of ids.
if self._active_task and self._active_task.uid in self._cancel_task_ids:
self._cancel_task_ids.discard(self._active_task.uid)
self._cancel_active_task()
# check general task processing
if self._active_task:
task_id = self._active_task.uid
info = self._active_task.task_info
task_status = self._active_task.status
# perform action based on active task status
if task_status is TASK_PENDING:
# check if external system is available to run task (Developer API)
if self.external_system_available():
self._active_task.start()
self.conn.send(Message(EXECUTOR_TASK_STARTED, task_id=task_id, info=info))
logger.info("%s - Started task %s", self.name, task_id)
else:
logger.info("%s - External system is not available", self.name)
elif task_status is TASK_STARTED:
# task has started and running
if self._active_task.is_alive(): # pragma: no branch
logger.debug("%s - Ping task %s is alive", self.name, task_id)
elif task_status is TASK_SUCCEEDED:
# task finished successfully
self.conn.send(Message(EXECUTOR_TASK_SUCCEEDED, task_id=task_id, info=info))
logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status)
self._active_task = None
elif task_status is TASK_FAILED:
# task failed
self.conn.send(Message(EXECUTOR_TASK_FAILED, task_id=task_id, info=info))
logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status)
self._active_task = None
elif task_status is TASK_CANCELLED:
# task has been cancelled
if self._active_task: # pragma: no branch
self._active_task = None
else:
logger.warning("%s - Unknown status %s for %s", self.name, task_status, task_id)
else:
logger.debug("%s - No active task registered", self.name)
def start():
logger.critical("free disk monitoring service started")
schedule.every(config.CHECK_PERIOD_SECONDS).seconds.do(check_hdds)
try:
while True:
schedule.run_pending()
time.sleep(0.5)
except (KeyboardInterrupt, SystemExit) as e:
logger.debug(repr(e))
except Exception as e:
logger.error(repr(e))
finally:
logger.critical("free disk monitoring service ended")
time.sleep(5)
def _safe_exec(self, func, **kwargs):
"""
Safely execute function with a list of arguments. Function is assumed not to return any
result.
:param func: function to execute
:param kwargs: dictionary of method parameters
"""
# pylint: disable=W0703,broad-except
try:
if func:
func(**kwargs)
except Exception as e:
logger.debug("%s - Failed to execute '%s(%s)', reason=%s",
self.name, func, kwargs, e)
def parse_not_expression(self) -> Result:
"""
n_exp := NOT n_exp | NOT p_exp
"""
token = self.pop_token()
if token.type == TOKEN_TYPE['eof']:
return Result(name='', value=False)
elif token.type != TOKEN_TYPE['not']:
self.push_token(token)
return self.parse_primary_expression()
r1 = self.parse_not_expression()
r = Result('(not {})'.format(r1.name), not r1.value)
logger.debug('[*] {}'.format(r))
return r
def get_room_id(self):
bs = self.recv()
if not bs:
return
code = bs.get_int8()
if code != 0x1:
logger.error('Register: ger error room id code: %s' % code)
return self.send_error()
self.room_id = bs.get_str()
logger.info('Register: Get a room id: %s' % self.room_id)
conf._SSH_SERVER_TERMINAL_PORT = bs.get_str()
logger.debug('Register: Get Server Terminal Port: %s' %
conf._SSH_SERVER_TERMINAL_PORT)
self.send_room_id()
def iter_components(self,
ignore_dirs=["tests"],
needpath=False) -> Generator[Component, None, None]:
"""Iterate out all components in the `self.directory`
"""
for root, filename in iter_files(self.directory, ignore_dirs):
if not filename.endswith('.json'):
continue
c_path = os.path.join(root, filename)
component = Component.make(c_path)
if component is None:
continue
logger.debug("iter_components: %s", component)
if needpath is False:
yield component
else:
yield component, c_path
def _on_map_click(self, event):
if event.inaxes is not None:
x = event.xdata
y = event.ydata
logger.debug(f"Clicked plot: {x}, {y}")
self._update_measurement_progress_label()
self.thread = threading.Thread(
target=self._presenter.measure_point_by_coordinates,
args=(
x,
y,
),
daemon=True,
)
self.thread.start()
else:
logger.debug("Clicked outside axes bounds but inside plot window")
def __init__(self, args):
"""
Create instance of application controller.
:param args: raw arguments dictionary
:param logger: logger function to use
"""
# parse queue configuration based on additional arguments passed
conf = QueueConf()
conf.setAllConf(args)
# log options processed
all_options = [" %s -> %s" % (key, value) for key, value in conf.copy().items()]
logger.debug("Configuration:\n%s", "\n".join(all_options))
# resolve options either directly or through session and other services
self.mongodb_url = self._validate_mongodb_url(conf.getConfString(const.OPT_MONGODB_URL))
self.working_dir = self._validate_working_dir(conf.getConfString(const.OPT_WORKING_DIR))
self.service_dir = self._validate_service_dir(conf.getConfString(const.OPT_SERVICE_DIR))
self.session = self._create_session(conf)
self.client = pymongo.MongoClient(self.mongodb_url)
self.api = RestApiDispatcher(self.session)
def _process_message(self, msg):
"""
Process message and take action, e.g. terminate process, execute callback, etc. Message
types are defined above in the package. Note that this can take actions on tasks, e.g.
when task is cancelled, so the subsequent processing of task, will work with updated state.
:param msg: message to process
"""
logger.debug("%s - Received message %s", self.name, msg)
if isinstance(msg, Message):
if msg.status == EXECUTOR_SHUTDOWN: # pragma: no branch
raise ExecutorInterruptedException("Executor shutdown")
elif msg.status == EXECUTOR_CANCEL_TASK: # pragma: no branch
# update set of tasks to cancel
if "task_id" in msg.arguments: # pragma: no branch
task_id = msg.arguments["task_id"]
self._cancel_task_ids.add(task_id)
logger.debug("%s - Registered cancelled task %s", self.name, task_id)
else:
# valid but unrecognized message, no-op
pass
else:
logger.info("%s - Invalid message %s is ignored", self.name, msg)
def start(self):
"""
Start all session services.
"""
# getting server info validates connection
logger.info("Trying to connect to Mongo instance")
server_info = self.client.server_info()
logger.info(server_info)
# database and tables' names are fixed for now
logger.debug("Setting up database and indexes")
db = self.client.queue
db.submissions.create_index([
("uid", pymongo.ASCENDING),
("status", pymongo.ASCENDING),
("createtime", pymongo.DESCENDING)
])
db.tasks.create_index([
("uid", pymongo.ASCENDING)
])
# subscribe to events
cherrypy.engine.subscribe(EVENT_CREATE, self.event_create)
# start scheduler
self.session.scheduler.start_maintenance()
self.session.scheduler.start()
def run(self):
# update task metrics and set status
self._set_metric("starttime", time.time())
self._set_metric("duration", 0)
self.__status = TASK_STARTED
# try launching listener callback, note that failure should not affect execution of task
self._safe_exec(self.on_task_started, uid=self.__uid, info=self.__info)
logger.debug("%s - Started, time=%s", self.name, self._get_metric("starttime"))
try:
msg_queue = threadqueue.Queue()
wprocess = WorkerThread(self.__task, msg_queue)
wprocess.start()
next_iteration = True
while next_iteration:
time.sleep(self.refresh_timeout)
if self.is_cancelled:
wprocess.cancel()
raise TaskInterruptedException()
if not wprocess.is_alive():
next_iteration = False
wprocess.join()
if not msg_queue.empty():
# we only care about the first exception occuried
error = msg_queue.get_nowait()
raise error
except TaskInterruptedException:
# task has been cancelled or requested termination
self.__status = TASK_CANCELLED
self._safe_exec(self.on_task_cancelled, uid=self.__uid, info=self.__info)
# pylint: disable=W0703,broad-except
except Exception as e:
# any other exception is considered a failure
self._set_metric("reason", "%s" % e)
logger.debug("%s - Failure reason=%s", self.name, self._get_metric("reason"))
self.__status = TASK_FAILED
self._safe_exec(self.on_task_failed, uid=self.__uid, info=self.__info,
reason=self._get_metric("reason"))
# pylint: enable=W0703,broad-except
else:
self.__status = TASK_SUCCEEDED
self._safe_exec(self.on_task_succeeded, uid=self.__uid, info=self.__info)
finally:
# set post-execution metrics for task
self._set_metric("endtime", time.time())
duration = self._get_metric("endtime") - self._get_metric("starttime")
self._set_metric("duration", duration)
logger.debug("%s - Finished, status=%s, time=%s, duration=%s", self.name, self.__status,
self._get_metric("endtime"), self._get_metric("duration"))
def _get_new_task(self):
"""
Extract new task from priority list of queues. If no tasks found for priority or priority
does not exist in dictionary, next priority is checked. If task is found, it is returned,
otherwise None. For each task TaskThread is created to provide status and metrics updates.
:return: new available task across priorities
"""
task_block = None
for priority in const.PRIORITIES:
logger.debug("%s - Searching task in queue for priority %s", self.name, priority)
try:
task_block = self.task_queue_map[priority].get(block=False)
except threadqueue.Empty:
logger.debug("%s - No tasks available for priority %s", self.name, priority)
except KeyError:
logger.debug("%s - Non-existent priority %s skipped", self.name, priority)
else:
if task_block: # pragma: no branch
break
# create thread for task
task_thread = TaskThread(task_block) if task_block else None
return task_thread
def cancel(self):
"""
Cancel current thread and potentially running task.
"""
logger.debug("%s - Requested cancellation of task", self.name)
self.__cancel.set()
