def thread_run(self):
self._thread_stop = False
th = threading.Thread(target=self.run, name='Connection run')
th.start()
gvar.thread.append(th)
logger.info('thread start -> Connection.run()')
def launch_process(self):
"""
Launch process for spark-submit.
"""
# spark-submit command to launch
command = self.cmd()
logger.info("%s - Launch command %s", self.name, command)
# only create stdout and stderr when working directory is provided
if self.__working_directory:
stdout_path = util.concat(self.__working_directory, "stdout")
stderr_path = util.concat(self.__working_directory, "stderr")
stdout = util.open(stdout_path, "wb")
stderr = util.open(stderr_path, "wb")
self.__ps = subprocess.Popen(command,
bufsize=4096,
stdout=stdout,
stderr=stderr,
close_fds=True)
else:
self.__ps = subprocess.Popen(command,
bufsize=4096,
stdout=None,
stderr=None,
close_fds=True)
logger.info("%s - Process pid=%s", self.name, self.__ps.pid)
def init_tcp(self):
self._tp = paramiko.Transport(self._socket, gss_kex=True)
self._tp.set_gss_host(socket.getfqdn(""))
self._tp.load_server_moduli()
host_key = paramiko.RSAKey(filename=conf.SSH_SERVER_RSA_EKY)
self._tp.add_server_key(host_key)
server = Server()
logger.debug('Create Server Class')
try:
self._tp.start_server(server=server)
except paramiko.SSHException:
logger.error("SSH negotiation failed.")
gvar.manager.close_connection(self)
exit()
self._channel = self._tp.accept(200)
if self._channel is None:
logger.error('No channel')
gvar.manager.close_connection(self)
exit()
logger.info('Authenticatied!')
def iteration(self):
"""
Run single iteration, entire logic of executor should be specified in this method, unless
there is an additional logic between iterations. Iteration is cancelled, if executor is
terminated.
:return: boolean flag, True - run next iteration, False - terminate
"""
# we process special case of terminated executor in case someone would launch it again.
if self._terminated:
logger.warning("Executor %s has been terminated", self.name)
return False
logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout)
try:
# send reponse to the scheduler that this executor is up and processing tasks
self._respond_is_alive()
# check if there are any messages in connection, process one message per iteration
if self.conn.poll():
self._process_message(self.conn.recv())
# check if there is any outstanding task to run, otherwise poll data for current task
self._process_task()
except ExecutorInterruptedException:
logger.info("%s - Requested termination of executor", self.name)
self._terminated = True
# cancel task that is currently running and clean up state
self._cancel_active_task()
return False
# pylint: disable=W0703,broad-except
except Exception as e:
logger.exception("%s - Unrecoverable error %s, terminating", self.name, e)
self._terminated = True
return False
# pylint: enable=W0703,broad-except
else:
return True
def _check_hdd(mount_path: str):
try:
# Get disk space
hdd = psutil.disk_usage(mount_path)
free_space = hdd.free / (1024 * 1024 * 1024)
total_space = hdd.total / (1024 * 1024 * 1024)
free_space_pct = free_space / total_space * 100
# Get disk inodes
inode_hdd_mount = os.statvfs(mount_path)
total_inode = inode_hdd_mount.f_files # inodes
free_inode = inode_hdd_mount.f_ffree # free inodes
free_inode_pct = free_inode / total_inode * 100
except FileNotFoundError:
logger.warning(f"failed to find '{mount_path}'")
else:
# Log and alert
logger.info(
f"space left on '{mount_path}': {free_space:.2f}GB ({free_space_pct:.1f}%) | inodes {free_inode:,} ({free_inode_pct:.1f}%)"
)
if (free_space_pct <= config.FREE_SPACE_PCT_WARNING) and (
free_space_pct > config.FREE_SPACE_PCT_ERROR):
send_alarm_warning(free_space, free_space_pct, mount_path)
elif free_space_pct <= config.FREE_SPACE_PCT_ERROR:
send_alarm_error(free_space, free_space_pct, mount_path)
if (free_inode_pct <= config.FREE_INODE_PCT_WARNING) and (
free_inode_pct > config.FREE_INODE_PCT_ERROR):
send_alarm_warning_inode(free_inode, free_inode_pct, mount_path)
elif free_inode_pct <= config.FREE_INODE_PCT_ERROR:
send_alarm_error_inode(free_inode, free_inode_pct, mount_path)
def load_rom(self, filename: str) -> None:
rom = open(filename, 'rb').read()
for i, val in enumerate(rom):
self.memory[PROGRAM_START + i] = val
self.program_counter = PROGRAM_START
self.rom_loaded = True
logger.info(f"Loaded {filename} into memory")
def close(self):
if self.channel:
self.thread_stop()
self.channel.close()
self.control.remove(self)
self.channel = None
logger.info("room connection was closed")
def send(self, msg):
try:
self._channel.send(msg)
except OSError:
logger.info("socket is closed")
gvar.manager.close_connection(self)
return
def create_extrapolation_dataset(src: PathLike, dest: PathLike, seed=None):
if seed is not None:
np.random.seed(seed=seed)
else:
np.random.seed(datetime.now().microsecond)
logger.info("开始制作外推数据集...")
# 存储文件编号
train_set_file_no = 1
test_set_file_no = 1
# 每个 folder 代表一天的序列
for folder in os.listdir(src):
step = 40 # 这个可以改，假定一天 240 个数据都有的话，按照 40 个一组划分为 6 组 block
r: List[str] = os.listdir(f"{src}\\{folder}") # 找出日期文件夹下所有的数据文件
r.sort() # 没办法，时序问题必须要按照实际序列取数据集，以防万一，排个序
r = list(map(lambda p: f"{src}\\{folder}\\{p}",
r)) # 每个元素变成可直接使用的地址字符串
split_list = [r[i:i + step]
for i in range(0, len(r), step)] # 一天划分成几个时间段
np.random.shuffle(split_list)
# 分配测试集一个 block
test_queue = deque(maxlen=30)
test_block = split_list[0]
for i in range(0, 30):
test_queue.append(test_block[i])
__copy(list(test_queue),
dest=f"{dest}\\test\\example{test_set_file_no:05d}")
test_set_file_no += 1
for i in range(30, len(test_block)):
test_queue.append(test_block[i])
__copy(list(test_queue),
dest=f"{dest}\\test\\example{test_set_file_no:05d}")
test_set_file_no += 1
split_list = split_list[1:]
# 其余 block 分配给训练集
train_queue = deque(maxlen=20)
for train_block in split_list:
train_queue.clear()
for i in range(0, 20):
train_queue.append(train_block[i])
__copy(list(train_queue),
dest=f"{dest}\\train\\example{train_set_file_no:05d}")
train_set_file_no += 1
for i in range(20, len(train_block)):
train_queue.append(train_block[i])
__copy(list(train_queue),
dest=f"{dest}\\train\\example{train_set_file_no:05d}")
train_set_file_no += 1
logger.info("外推数据集制作完毕...")
def shutdown(self):
import sys
logger.info('shutdown the programe.')
self.close_ssh_server()
self.close_mansion()
gvar.thread.kill_all_thread()
logger.info('bye!')
sys.exit(0)
def __init__(self) -> None:
self.volume = 0.75
pygame.mixer.init()
square_wave = bytearray([128] * 400 + [0] * 400)
self.sound = pygame.mixer.Sound(buffer=square_wave)
self.sound.play(loops=-1)
self.sound.set_volume(0.0)
logger.info("Sound initialized")
def close_room(self):
if not self._room:
logger.info('not in room')
return
self.ssh_client.close()
self._room.close()
logger.info("close the room")
def start(self):
"""
Start scheduler, launches executors asynchronously.
"""
logger.info("Starting %s '%s' executors", self.num_executors, self.executor_class())
# Launch executors and save pipes per each
for i in range(self.num_executors):
exc = self._prepare_executor("#%s" % i)
exc.start()
def start(self):
"""
Start scheduler, launches executors asynchronously.
"""
logger.info("Starting %s '%s' executors", self.num_executors,
self.executor_class())
# Launch executors and save pipes per each
for i in range(self.num_executors):
exc = self._prepare_executor("#%s" % i)
exc.start()
def create_restore_file(path_queue_file, path_crawled_file, path_restore_file):
queue_set = set(_open(path_queue_file))
crawled_set = set(_open(path_crawled_file))
restore_queue = list(queue_set - crawled_set)
append_list_to_file(path_restore_file, restore_queue, to_new_file=True)
logger.info('restore file created')
return True
def __init__(self, socket):
super().__init__(socket)
self._tp = None
self._control = None # src.control or src.channel
self._channel: paramiko.channel.Channel = None
self._thread_stop = False
logger.info('establish a new ssh session.')
self.init_tcp()
def create_room(self, conn):
logger.info('createing a room...')
while True:
i = self.random_id()
if self.check_id(i):
break
logger.info('random a room id -> %s', i)
room = Room(i)
room.channel = conn
self._rlist[i] = room
def start_listening(self, port):
self._port = port
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.listener.bind((conf.SSH_SERVER_LISTENING_IP, self._port))
try:
self.listener.listen()
logger.info('SSH.start listening port:%s OK ...' % self._port)
except:
logger.error(
'SSH.start listening port:%s Error, Exit...' % self._port)
def _cancel_active_task(self):
"""
Cancel current running task, if available.
"""
if self._active_task:
task_id = self._active_task.uid
info = self._active_task.task_info
self._active_task.cancel()
self.conn.send(Message(EXECUTOR_TASK_CANCELLED, task_id=task_id, info=info))
logger.info("%s - Cancelled task %s", self.name, task_id)
self._active_task = None
else:
logger.info("%s - No active task to cancel", self.name)
def run(self):
"""
Method to run tasks on executor, this runs in iterations with each timeout interval. Each
iteration polls new messages from connection and checks running task. If iteration fails we
immediately return status False.
"""
logger.info("Start executor %s, time=%s", self.name, time.time())
proceed = True
while proceed: # pragma: no branch
proceed = self.iteration()
if not proceed:
return False
time.sleep(self.timeout)
def connection(self):
try:
# params: timeout = None means forever
self.port = serial.Serial(conf.SERIAL_DEVICE, baudrate=conf.SERIAL_BAUDRATE, timeout=None, write_timeout=2,
parity=serial.PARITY_NONE, bytesize=serial.EIGHTBITS, stopbits=serial.STOPBITS_ONE, xonxoff=False)
except serial.serialutil.SerialException:
logger.error(
"can't open serial %s, Please check the COM is open and retry..." % conf.SERIAL_DEVICE)
raise(serial.SerialException)
self.port.flushInput()
logger.info('serial port \'%s\' connection complete' %
conf.SERIAL_DEVICE)
def run(self):
"""
Method to run tasks on executor, this runs in iterations with each timeout interval. Each
iteration polls new messages from connection and checks running task. If iteration fails we
immediately return status False.
"""
logger.info("Start executor %s, time=%s", self.name, time.time())
proceed = True
while proceed: # pragma: no branch
proceed = self.iteration()
if not proceed:
return False
time.sleep(self.timeout)
def select_room(self, conn):
"""
Enter a room, need select a room ID, and give a username
"""
logger.debug("into select room process...")
se = Select(self._rlist.keys(), conn)
room_id, username = se.process()
logger.info('get room_id: %s, username: %s' % (room_id, username))
room = self._rlist[room_id]
conn.username = username
room.add_connection(conn)
del se
def wait_keyboard_interrupt(self):
import time
logger.debug("Into waiting for keyboard interrupt")
try:
while gvar.thread.has_alive_thread():
gvar.thread.clean_stoped_thread()
time.sleep(50)
except KeyboardInterrupt:
logger.info("shutdown the program...")
self.shutdown()
gvar.thread.kill_all_thread()
logger.info("Bye!")
def _process_task(self):
"""
Process individual task, returns exit code for each task following available API. One of
the checks is performed to test current task_id against cancelled list, and discard task,
if it has been marked as cancelled, or terminate running task.
"""
if not self._active_task:
self._active_task = self._get_new_task()
logger.info("%s - New task registered", self.name)
# before checking statuses and proceed execution, we check if current task was
# requested to be cancelled, if yes, we remove it from set of ids.
if self._active_task and self._active_task.uid in self._cancel_task_ids:
self._cancel_task_ids.discard(self._active_task.uid)
self._cancel_active_task()
# check general task processing
if self._active_task:
task_id = self._active_task.uid
info = self._active_task.task_info
task_status = self._active_task.status
# perform action based on active task status
if task_status is TASK_PENDING:
# check if external system is available to run task (Developer API)
if self.external_system_available():
self._active_task.start()
self.conn.send(Message(EXECUTOR_TASK_STARTED, task_id=task_id, info=info))
logger.info("%s - Started task %s", self.name, task_id)
else:
logger.info("%s - External system is not available", self.name)
elif task_status is TASK_STARTED:
# task has started and running
if self._active_task.is_alive(): # pragma: no branch
logger.debug("%s - Ping task %s is alive", self.name, task_id)
elif task_status is TASK_SUCCEEDED:
# task finished successfully
self.conn.send(Message(EXECUTOR_TASK_SUCCEEDED, task_id=task_id, info=info))
logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status)
self._active_task = None
elif task_status is TASK_FAILED:
# task failed
self.conn.send(Message(EXECUTOR_TASK_FAILED, task_id=task_id, info=info))
logger.info("%s - Finished task %s, status %s", self.name, task_id, task_status)
self._active_task = None
elif task_status is TASK_CANCELLED:
# task has been cancelled
if self._active_task: # pragma: no branch
self._active_task = None
else:
logger.warning("%s - Unknown status %s for %s", self.name, task_status, task_id)
else:
logger.debug("%s - No active task registered", self.name)
def download_zip_file(url: str) -> str:
"""
Streams remote ZIP file to disk without using excessive memory, extract its content,
then deletes it.
:param url: remote file URL
:return: extracted file name
"""
local_filename = f"tmp/{url.split('/')[-1]}"
# Downloading ZIP file
logger.info(
f"Downloading ZIP file from {url} into local file '{local_filename}' ..."
)
with requests.get(url, stream=True) as r, open(local_filename, "wb") as f:
shutil.copyfileobj(r.raw, f)
logger.info(
f"Download of ZIP data from {url} into local file '{local_filename}' successfull !"
)
# Extracting ZIP file data
logger.info(f"Extracting ZIP data from {local_filename} ...")
with ZipFile(local_filename) as zf:
zf.extractall()
logger.info(f"Extraction of ZIP data from {local_filename} successfull !")
# Removing ZIP file after extraction and returning the extracted filename
remove(local_filename)
return local_filename.split(".zip")[0]
def check_server_port_is_open(self):
"""
check with server connection and port is open
but there have some problem, server point will print SSH Error on logbuf
"""
try:
tel = telnetlib.Telnet(host=conf.SSH_SERVER_IP_ADDRESS,
port=conf.SSH_SERVER_PORT, timeout=1)
tel.close()
return True
except:
logger.info(
"can't connect ssh server, please check the internet setting.")
return False
def _cancel_active_task(self):
"""
Cancel current running task, if available.
"""
if self._active_task:
task_id = self._active_task.uid
info = self._active_task.task_info
self._active_task.cancel()
self.conn.send(
Message(EXECUTOR_TASK_CANCELLED, task_id=task_id, info=info))
logger.info("%s - Cancelled task %s", self.name, task_id)
self._active_task = None
else:
logger.info("%s - No active task to cancel", self.name)
def create_ssh_server(self, port):
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((conf.SSH_SERVER_LISTENING_IP, port))
try:
listener.listen()
logger.info('SSH.start listening port:%s OK ...' % port)
except:
logger.error('SSH.start listening port:%s Error, Exit...' % port)
var = SSHServer()
var.manager = self # ! maybe not need
var.listener = listener
var._port = port
var.thread_run()
self.ssh_servers.append(var)
def __init__(self) -> None:
pygame.init()
self.PIXEL_SIZE = 20
self.MENU_HEIGHT = 40
self.WIDTH = 64 * self.PIXEL_SIZE
self.HEIGHT = 32 * self.PIXEL_SIZE
self.TOTAL_HEIGHT = self.HEIGHT + self.MENU_HEIGHT
self.DISPLAY = pygame.display.set_mode((self.WIDTH, self.TOTAL_HEIGHT),
0, 1)
self.WHITE: tuple[int, int, int] = (230, 230, 230)
self.BLACK: tuple[int, int, int] = (20, 20, 20)
self.pixel_matrix: list[list[int]] = [[0] * 32 for _ in range(64)]
self.paused = False
logger.info("Screen initialized")
def stop(self):
"""
Stop scheduler, terminates executors, and all tasks that were running at the time.
"""
for conn in self.pipe.values():
conn.send(Message(EXECUTOR_SHUTDOWN))
# timeout to terminate processes and process remaining messages in Pipe by polling thread
logger.info("Waiting for termination...")
time.sleep(5)
for exc in self.executors:
if exc.is_alive():
exc.terminate()
exc.join()
logger.info("Terminated executors, cleaning up internal data")
self.pipe = None
self.executors = None
self.task_queue_map = None
def stop(self):
"""
Stop scheduler, terminates executors, and all tasks that were running at the time.
"""
for conn in self.pipe.values():
conn.send(Message(EXECUTOR_SHUTDOWN))
# timeout to terminate processes and process remaining messages in Pipe by polling thread
logger.info("Waiting for termination...")
time.sleep(5)
for exc in self.executors:
if exc.is_alive():
exc.terminate()
exc.join()
logger.info("Terminated executors, cleaning up internal data")
self.pipe = None
self.executors = None
self.task_queue_map = None
def get_room_id(self):
bs = self.recv()
if not bs:
return
code = bs.get_int8()
if code != 0x1:
logger.error('Register: ger error room id code: %s' % code)
return self.send_error()
self.room_id = bs.get_str()
logger.info('Register: Get a room id: %s' % self.room_id)
conf._SSH_SERVER_TERMINAL_PORT = bs.get_str()
logger.debug('Register: Get Server Terminal Port: %s' %
conf._SSH_SERVER_TERMINAL_PORT)
self.send_room_id()
def check_version(self):
new_ver = self.get_new_version()
if not new_ver:
return
logger.debug("remote repository recent version is: %s" % new_ver)
logger.debug("the running version is %s" % banner.VERSION)
new_ver_int = calc_version(new_ver)
run_ver_int = calc_version(banner.VERSION)
if not (new_ver_int and run_ver_int): # check if format is bad
return False
if (new_ver_int > run_ver_int): # check if recent version is new
return True
else:
logger.info("it's the latest version")
return False
def launch_process(self):
"""
Launch process for spark-submit.
"""
# spark-submit command to launch
command = self.cmd()
logger.info("%s - Launch command %s", self.name, command)
# only create stdout and stderr when working directory is provided
if self.__working_directory:
stdout_path = util.concat(self.__working_directory, "stdout")
stderr_path = util.concat(self.__working_directory, "stderr")
stdout = util.open(stdout_path, "wb")
stderr = util.open(stderr_path, "wb")
self.__ps = subprocess.Popen(command, bufsize=4096, stdout=stdout, stderr=stderr,
close_fds=True)
else:
self.__ps = subprocess.Popen(command, bufsize=4096, stdout=None, stderr=None,
close_fds=True)
logger.info("%s - Process pid=%s", self.name, self.__ps.pid)
def parse_txt_files(paths: List[str]) -> Iterator[str]:
"""
Parses a NCDC data file stored locally and yields database records from it.
:param paths: list of .txt files paths
:return: database records generator
"""
for p in paths:
with open(p, "r") as f:
for line in f:
if line.startswith("#"):
header = parse_header(line)
else:
data = parse_data_record(line)
yield transform_record({**header, **data})
remove(p)
logger.info(f"Successfully parsed and deleted text file {p} !")
def get_new_version(self):
try:
resp = requests.get(url=banner.REPOSITORY_API_ADDRESS)
except Exception as e:
logger.error("network anomaly")
return False
if resp.status_code != 200:
logger.info("unable to get a valid page")
return False
_ = str(resp.content.decode())
new_ver = re.search(r'"name":"(.*?)",', _).group(1)
if not new_ver:
logger.info("unable to get valid version information")
return False
return new_ver
def _process_message(self, msg):
"""
Process message and take action, e.g. terminate process, execute callback, etc. Message
types are defined above in the package. Note that this can take actions on tasks, e.g.
when task is cancelled, so the subsequent processing of task, will work with updated state.
:param msg: message to process
"""
logger.debug("%s - Received message %s", self.name, msg)
if isinstance(msg, Message):
if msg.status == EXECUTOR_SHUTDOWN: # pragma: no branch
raise ExecutorInterruptedException("Executor shutdown")
elif msg.status == EXECUTOR_CANCEL_TASK: # pragma: no branch
# update set of tasks to cancel
if "task_id" in msg.arguments: # pragma: no branch
task_id = msg.arguments["task_id"]
self._cancel_task_ids.add(task_id)
logger.debug("%s - Registered cancelled task %s", self.name, task_id)
else:
# valid but unrecognized message, no-op
pass
else:
logger.info("%s - Invalid message %s is ignored", self.name, msg)
def start(self):
"""
Start all session services.
"""
# getting server info validates connection
logger.info("Trying to connect to Mongo instance")
server_info = self.client.server_info()
logger.info(server_info)
# database and tables' names are fixed for now
logger.debug("Setting up database and indexes")
db = self.client.queue
db.submissions.create_index([
("uid", pymongo.ASCENDING),
("status", pymongo.ASCENDING),
("createtime", pymongo.DESCENDING)
])
db.tasks.create_index([
("uid", pymongo.ASCENDING)
])
# subscribe to events
cherrypy.engine.subscribe(EVENT_CREATE, self.event_create)
# start scheduler
self.session.scheduler.start_maintenance()
self.session.scheduler.start()
