class TcpAcceptor:
def __init__(self, host, port, header_size):
self.host = host
self.port = port
self.header_size = header_size
self.thread = threading.Thread(target=thread_entry,
args=(self, ),
daemon=True)
self.receivers = []
self.receiver_queue = SimpleQueue()
self.accept_callbacks = []
def start(self):
self.thread.start()
def update(self):
if not self.receiver_queue.empty():
receiver = self.receiver_queue.get_nowait()
self.receivers.append(receiver)
for fn in self.accept_callbacks:
fn(receiver)
pass
for receiver in self.receivers:
receiver.update()
return True
def subscribe(self, accept_callback):
self.accept_callbacks.append(accept_callback)
def thread_entry(self):
logging.info("TcpAcceptor : thread_entry")
s = socket.socket()
s.bind((self.host, self.port))
s.listen(5)
while True:
conn, addr = s.accept()
receiver = TcpReceiver(conn, addr, self.header_size)
receiver.start()
self.receiver_queue.put(receiver)
async def _single_iteration_queue(self,
source: _queue.SimpleQueue,
target: asyncio.Queue):
"""Transfer one queue item.
If a *stop* command is encountered, self-cancel after transfering command.
To avoid race conditions while stopping queue processing,
place a *stop* command in *source* and asyncio.shield() a call
to this coroutine in a *try: ... except: ...* block.
Note that the caller will then receive CancelledError after *stop* command has
been transferred.
Raises:
queue.Empty if *source* is empty
asyncio.CancelledError when cancelled or *stop* is received.
"""
command: QueueItem = source.get_nowait()
logger.debug(f'Processing command {repr(command)}')
await target.put(command)
# TODO: Use formal RPC protocol.
if 'control' in command:
# Note that we don't necessarily need to stop managing the dispatcher queue
# at this point, but the Executor will be directed to shut down,
# so we must not put anything else onto the command queue until we have a
# new command queue or a new executor.
if command['control'] == 'stop':
raise asyncio.CancelledError()
else:
raise ProtocolError('Unknown command: {}'.format(command['control']))
else:
if 'add_item' not in command:
# TODO: We might want a call-back or Event to force errors before the
# queue-runner task is awaited.
raise MissingImplementationError(
f'Executor has no implementation for {str(command)}'
)
return command
class TcpReceiver:
def __init__(self, conn, addr, header_size):
self.conn = conn
self.addr = addr
self.queue = SimpleQueue()
self.header_size = header_size
self.thread = threading.Thread(target=thread_entry,
args=(self, ),
daemon=True)
def start(self):
self.thread.start()
def update(self):
pass
def take(self):
if self.queue.empty():
return None
return self.queue.get_nowait()
def thread_entry(self):
logging.info("TcpReceiver : thread_entry")
self._receive_loop()
def _receive_loop(self):
conn = self.conn
queue = self.queue
header_size = self.header_size
while True:
size_str = conn.recv(header_size)
if not size_str:
break
size_int = int.from_bytes(size_str, byteorder='big')
logging.info('Receiving : size_str = %s, size = %d', size_str,
size_int)
msg_bytes = conn.recv(size_int)
queue.put(msg_bytes)
class MakeCallbackAwaitable:
DEFAULT_TIMEOUT = 3 # in seconds
def __init__(self, loop):
self._loop = loop or asyncio.get_event_loop()
self._queue = None
def create_pair(self) -> tuple[Callable, Callable]:
self._queue = SimpleQueue() # maxsize=1)
def putter(*args): # callback
self._loop.call_soon_threadsafe(self._queue.put_nowait, args)
async def getter(timeout=self.DEFAULT_TIMEOUT) -> tuple:
timeout = self.DEFAULT_TIMEOUT if timeout is None else timeout
dt_expired = dt.now() + td(seconds=timeout)
while dt.now() < dt_expired:
try:
return self._queue.get_nowait()
except Empty:
await asyncio.sleep(0.005)
raise TimeoutError
return getter, putter # awaitable, callback
class Tasks():
def __init__(self, maxtasks=5, eco=False):
self.queue = SimpleQueue()
self.__status = False
self.__task = [] # running task ( as threads instance )
self.__left = [] # list of tasks in queue (tskname,func,args,kwargs)
self.__done = [] # list of tasks done (tskname,*perf[todo])
self.__aborted = [
] # list of tasks aborted when stop() issued and left items still in queue (tskname,func,args,kwargs)
self.__stoppending = False # True when we empty the remaining queue after a task.stop(), doing nothing.
self.__reports = {}
self.__polls = []
self.__pollslatency = 1.0
self.__pollslast = 0
self.__eco = eco
self.__verbose = False
self.maxtasks = maxtasks
if not (self.__eco): self.start()
@property
def task(self):
tsks = []
for t in self.__task:
tsks.append(t.getName())
return tsks
@property
def left(self):
return self.__left
@property
def done(self):
return self.__done
@property
def aborted(self):
return self.__aborted
@property
def eco(self):
return self.__eco
@eco.setter
def eco(self, v):
self.__eco = bool(v)
@property
def maxtasks(self):
return self.__maxtasks - 2
@property
def pollslatency(self):
return self.__pollslatency
@pollslatency.setter
def pollslatency(self, v):
self.__pollslatency = int(v)
@property
def verbose(self):
return self.__verbose
@verbose.setter
def verbose(self, v):
self.__verbose = bool(v)
@maxtasks.setter
def maxtasks(self, v):
self.__maxtasks = max(
1, v) + 2 # main process and task manager are alive, so add 2
@property
def status(self):
return self.__status
def statusinfo(self):
if self.__status: print('tasks manager started and waiting.')
else: print('tasks manager stopped.')
def start(self):
if not (self.__status):
if self.__stoppending:
print('tasks manager is leaving, can\'t start for now.')
else:
self.__status = True
p = threading.Thread(target=self.__queue, name='TaskManager')
p.start()
else:
print('tasks manager already started.')
def stop(self):
if self.__stoppending:
print('tasks manager is already leaving.')
elif not (self.__status):
print('tasks manager already stopped.')
else:
self.__stoppending = True
# task.report(task basename,number of tasks awaited)
# prepare a train of tasks after the call on can
# task.info(task basename) to known the current status of
# these tasks
def report(self, tskbasename, count=1, poll=False):
self.__reports[tskbasename] = {
'total': count,
'count': 0,
'start': -1,
'time': -1,
'poll': poll,
}
if poll:
self.__polls.append(tskbasename)
def add(self, tskname, func=None, *args, **kwargs):
self.queue.put((tskname, func, args, kwargs))
self.__left.append(tskname)
if self.__eco and not (self.status) and len(self.__left) == 1:
self.start()
# task.info()
# task.info(train task basename,rtn)
# is train task basename is given, returns infos about a train defined by task.report()
# if rtn is False (default), print information in the console
# if True, returns a tuple with ( status, tasks done, total tasks, start time, end time )
def info(self, tskbasename=False, rtn=False):
if tskbasename:
train = "tasks train '%s'" % tskbasename
if tskbasename in self.__reports:
rep = self.__reports[tskbasename]
since = perf_counter() - rep['start']
if rep['start'] == -1:
trainstacli = '%s is waiting for first task' % (train)
trainsta = 'waiting'
elif rep['time'] != -1:
trainstacli = '%s done in %.3f secs' % (train, rep['time'])
trainsta = 'done'
else:
trainstacli = '%s running since %.3f secs %s/%s' % (
train, since, rep['count'], rep['total'])
trainsta = 'running'
else:
trainstacli = '%s does not exist' % (train)
if rtn:
return (trainsta, rep['count'], rep['total'], since,
rep['time'])
print(trainstacli)
else:
self.statusinfo()
print('tasks left : %s (check : %s)' %
(self.queue.qsize(), len(
self.__left))) # qsize is reported as non-reliable
print('current tasks : %s' %
(','.join(self.task) if self.task else 'None'))
print(f'total done : {len(self.__done)}')
print(f'total aborted : {len(self.__aborted)}')
print(f'threads : {active_count()}')
def __taskwrapper(self, func, tskname, *args, **kwargs):
if 'callback' in kwargs: callback = kwargs.pop('callback')
else: callback = False
rtn = func(*args, **kwargs)
self.__done.append((tskname, [rtn]))
if callback: callback(tskname, rtn)
def __queue(self):
self.statusinfo()
while self.__status:
# got something to do
try:
args = list(self.queue.get_nowait())
tskname, func, args, kwargs = args
tskbasename = '\\'.join(
tskname.split('\\')[:-1]) # for tasks trains
tskcheck = self.__left.pop(0)
if not (self.__stoppending):
if active_count() < self.__maxtasks:
# active_count()-1 rather than -2 : the about to run thread is counted as already running
if self.__verbose:
print("run task '%s' (%s/%s tasks)" %
(tskname, active_count() - 1,
self.__maxtasks - 2))
# counter of train of tasks
if tskbasename in self.__reports:
rep = self.__reports[tskbasename]
if rep['start'] == -1:
rep['start'] = perf_counter()
if self.__verbose: self.info(tskbasename)
# t = Thread(target=lambda q, arg1: q.put(foo(arg1)), args=(que, 'world!'))
# args.insert(self.returnsqueue)
# t = threading.Thread(target=lambda q, *args, **kwargs: q.put(func(*args,**kwargs)), name=tskname, args=args, kwargs=kwargs)
# t = threading.Thread(target=func, name=tskname, args=args, kwargs=kwargs)
if not (args): args = []
args = list(args)
args.insert(0, tskname)
args.insert(0, func)
t = threading.Thread(target=self.__taskwrapper,
name=tskname,
args=args,
kwargs=kwargs)
t.start()
self.__task.append(t)
while active_count() >= self.__maxtasks:
sleep(0.25)
else:
print('SHOULD NEVER BE HERE !!') # well
# stop is pending, abort queue
else:
self.__aborted.append((tskname, func, args, kwargs))
if self.__verbose: print('task %s aborted' % (tskname))
# empty queue
except:
# print('empty queue, left is %s'%self.__left) # check
# print('empty queue, task is %s'%self.task) # check
if not (self.__task):
if self.__stoppending:
self.__status = False
self.__stoppending = False
elif self.__eco:
self.__stoppending = True
if self.__verbose: print('eco mode, ask for stop()')
else: sleep(0.25)
# task completion check
if self.__task:
tasknext = []
for t in self.__task:
if t not in threading.enumerate():
tskname = t.getName()
tskbasename = '\\'.join(
tskname.split('\\')[:-1]) # for tasks trains
if self.__verbose: print("task '%s' ended." % tskname)
# train
if tskbasename in self.__reports:
rep = self.__reports[tskbasename]
rep['count'] += 1
if rep['count'] == rep['total']:
rep['time'] = perf_counter() - rep['start']
if self.__verbose:
self.info(tskbasename)
else:
tasknext.append(t)
self.__task = tasknext
# train task completion for callback.
if self.__polls and (perf_counter() - self.__pollslast >=
self.__pollslatency or not (self.__status)):
pollnext = []
for tskbasename in self.__polls:
rep = self.__reports[tskbasename]
trainsta, count, total, since, duration = self.info(
tskbasename, True)
if trainsta == 'done':
func = rep['poll']
t = threading.Thread(
target=func,
name='tasksTrain.%s.%s' % (tskbasename, trainsta),
args=(tskbasename, trainsta, count, total, since,
duration))
t.start()
rep['poll'] = False
else:
pollnext.append(tskbasename)
self.__pollslast = perf_counter()
self.__polls = pollnext
self.statusinfo()
class Target:
name: str
local_port: int
remote_ip: str
_messages_received: 'SimpleQueue[Message]'
_messages_to_send: 'SimpleQueue[Message]'
def __init__(self, name: str, remote_ip: str):
self.name = name
self.remote_ip = remote_ip
# self._status = TargetStatus.INITIALISED
# self._status_lock = threading.Lock()
self._messages_to_send = SimpleQueue()
self._messages_received = SimpleQueue()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
last_used_port_lock.acquire()
def try_port():
try:
global last_used_port
last_used_port += 1
self.local_port = last_used_port
sock.bind(('0.0.0.0', last_used_port))
return True
except:
return False
while try_port():
pass
last_used_port_lock.release()
print(self.local_port)
def do_work():
sock.listen(1)
communication.initialise_with_sock(sock.accept()[0],
self._messages_to_send,
self._messages_received)
same_ip_targets = list(
filter(lambda x: x.remote_ip == self.remote_ip, targets))
if len(same_ip_targets) > 0:
for target in same_ip_targets:
target.send_message(
Message(MESSAGE_SOURCE_TYPES.RAVAGE_CORE,
MESSAGE_TYPES.ACTION,
sub_type=ACTIONS.SHUTDOWN.value))
targets.append(self)
self.send_message(
Message(MESSAGE_SOURCE_TYPES.SOUNDWAVE,
MESSAGE_TYPES.ACTION,
sub_type=ACTIONS.HELLO.value))
print(f'Target {remote_ip} created and connected')
threading.Thread(target=do_work, daemon=True).start()
def send_message(self, message: Message):
self._messages_to_send.put(message)
def get_received_message(self, block=True):
if block:
return self._messages_received.get()
else:
try:
return self._messages_received.get_nowait()
except Empty:
return None
class ActionStatistics:
def __init__(self):
self.queue_add_action = SimpleQueue()
self.queue_get_stats = SimpleQueue()
self.queue_get_stats_response = SimpleQueue()
self._dict = dict(
) # Should only be accessed by the thread running self.worker
threading.Thread(target=self.worker, daemon=True).start()
def worker(self):
"""Manages queue needs."""
while True:
if self.queue_add_action.empty() and self.queue_get_stats.empty():
sleep(0.5)
continue
if not self.queue_add_action.empty():
item = self.queue_add_action.get_nowait()
if item:
self._add_action(item)
if not self.queue_get_stats.empty(
) and self.queue_get_stats.get_nowait():
self.queue_get_stats_response.put(self._get_stats())
def addAction(self, action: str):
"""Add an action's time to the average for that action.
Parameters
----------
action : str
A JSON serialized string with a format like: {"action":"jump", "time":100}. The "action" field is
case sensitive.
"""
self.queue_add_action.put(action)
def getStats(self):
"""Return the average times for each action.
Returns
-------
str
JSON serialized string containing a list of the actions and their respective average times with the format:
[{"action":"jump", "avg":150}, {"action":"run", "avg":75}]
Raises
------
queue.Empty
If this is unable to get a response for the query within the timeout, then a
queue.Empty exception will be raised.
"""
self.queue_get_stats.put(True)
return self.queue_get_stats_response.get(block=True, timeout=5)
def _add_action(self, action: str):
"""Add an action to the internal storage object.
Parameter
---------
action : str
JSON serialized action to add.
"""
action = loads(action)
action_time = action['time']
action = action['action']
internal_action = self._dict.get(action, {'total': 0, 'count': 0})
internal_action['total'] += action_time
internal_action['count'] += 1
self._dict[action] = internal_action
def _get_stats(self):
"""Return the stats."""
return dumps([{
'action': k,
'avg': v['total'] / v['count']
} for k, v in self._dict.items()])
class Connection:
"""
ACI Connection
"""
def __init__(self, loop, ip, port, name):
"""
:param ip:
:param port:
:param loop:
"""
global connections
self.ip = ip
self.port = port
self.ws = 0
self.responses = SimpleQueue()
self.loop = loop
self.name = name
self.id = "not-authed"
self.interfaces = {}
self.event_callbacks = []
connections[name] = self
async def start(self):
await self._create(self.port, self.ip, self.loop, self.responses)
async def wait_for_response(self, _, key="none", db_key="none", cmd_type="any"):
"""
Waits for a response
:param _:
:param key:
:param db_key:
:return:
"""
while True:
if not self.responses.empty():
value = self.responses.get_nowait()
cmd = json.loads(value)
if cmd["cmd_typ"] == cmd_type or cmd_type == "any":
if cmd["cmd_typ"] == "get_val" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
elif cmd["cmd_typ"] == "set_val":
return cmd["val"]
elif cmd["cmd_typ"] == "ld":
return cmd["val"]
elif cmd["cmd_typ"] == "auth_msg":
return cmd["val"]
elif cmd["cmd_typ"] == "get_indexResp" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
elif cmd["cmd_typ"] == "set_indexResp" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
elif cmd["cmd_typ"] == "app_indexResp" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
elif cmd["cmd_typ"] == "get_len_indexResp" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
elif cmd["cmd_typ"] == "get_recent_indexResp" and cmd["key"] == key and cmd["db_key"] == db_key:
return cmd["val"]
async def _create(self, port, ip, loop, responses):
"""
Initializes the connection
:param port:
:param ip:
:param loop:
:param responses:
:return:
"""
await self.handler(loop, responses, ip, port)
async def handler(self, loop, responses, ip="127.0.0.1", port=8765):
"""
Creates a handler
:param loop:
:param responses:
:param ip:
:param port:
:return:
"""
asyncio.set_event_loop(loop)
uri = "ws://%s:%s" % (ip, port)
async with websockets.connect(uri) as websocket:
# print(websocket)
self.ws = websocket
time.sleep(0.25)
while True:
consumer_task = asyncio.ensure_future(_recv_handler(self.ws, uri, responses))
done, pending = await asyncio.wait([consumer_task], return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
def _get_interface(self, database_key):
"""
Gets an interface to the Database with the given keys
:param database_key:
:return:
"""
return DatabaseInterface(self, database_key)
def __getitem__(self, key):
if key not in self.interfaces:
self.interfaces[key] = self._get_interface(key)
return self.interfaces[key]
async def create_database(self, db_key):
await self.ws.send(json.dumps({"cmd": "cdb", "db_key": db_key}))
@allow_sync
async def authenticate(self, id, token):
self.id = id
await self.ws.send(json.dumps({"cmd":"a_auth", "id":id, "token":token}))
return await self.wait_for_response("auth_msg", None, None)
@allow_sync
async def send_event(self, destination, event_id, data):
await self.ws.send(json.dumps({"cmd":"event", "event_id":event_id, "destination": destination, "data": data, "origin":self.id}))
def add_event_callback(self, event_callback):
self.event_callbacks.append(event_callback)
class Builder(object):
def __init__(self, build_os, image):
self.__client = docker.from_env()
self.__image = image
self.__build_os = build_os
self.__container = None
self.__container_logs = None
self.__cancel_lock = threading.Lock()
self.__cancelled = False
self.__logs = SimpleQueue()
self.build_output = dict()
def __enter__(self):
return self
@property
def script_template(self):
if self.__build_os == "Linux":
return "build.sh.in"
else:
return "build.ps1.in"
@property
def build_package_dir(self):
if self.__build_os == "Linux":
return "/{0}".format(build_package_dir_name)
else:
return "C:\\{0}".format(build_package_dir_name)
@property
def build_package_dir(self):
if self.__build_os == "Linux":
return "/{0}".format(build_package_dir_name)
else:
return "C:\\{0}".format(build_package_dir_name)
@property
def escaped_build_package_dir(self):
if self.__build_os == "Linux":
return "/{0}".format(build_package_dir_name)
else:
return "C:\\\\{0}".format(build_package_dir_name)
@property
def root_dir(self):
if self.__build_os == "Linux":
return "/"
else:
return "C:\\"
@property
def build_output_dir(self):
if self.__build_os == "Linux":
return "/tmp/{0}".format(build_output_dir_name)
else:
return "C:\\{0}".format(build_output_dir_name)
@property
def build_command(self):
if self.__build_os == "Linux":
return "sh {0}/build.sh".format(self.build_package_dir)
else:
return 'cmd /s /c "powershell -File {0}\\build.ps1"'.format(
self.build_package_dir)
def pull(self, parameters):
m = re.match(docker_image_pattern, self.__image)
if not m:
raise Exception("The image '{0}' is not a valid "
"docker image name".format(self.__image))
tag = m.group(4)
repository = m.group(1)
if tag == "local":
logger.info("Do not pull local image '%s'", self.__image)
return
auth_config = None
if parameters['docker_user']:
auth_config = {
"username": parameters['docker_user'],
"password": parameters['docker_password']
}
logger.info("Pull docker image '%s'", self.__image)
self.__client.images.pull(repository=repository,
tag=tag,
auth_config=auth_config)
def setup(self, parameters):
logger.info("Setup docker container")
self.__container = self.__client.containers.create(
image=self.__image, command=self.build_command)
logger.info("Created docker container '%s'", self.__container.short_id)
config_url = "{0} --type=git".format(parameters["conan_config_url"])
config_branch = "--args \"-b {0}\"".format(parameters["conan_config_branch"])\
if parameters["conan_config_branch"] else ""
config_path = "-sf {0}".format(parameters["conan_config_path"])\
if parameters["conan_config_path"] else ""
patched_parameters = {
**parameters, "conan_config_args":
" ".join([config_url, config_branch, config_path]),
"build_package_dir":
self.build_package_dir,
"escaped_build_package_dir":
self.escaped_build_package_dir,
"build_output_dir":
self.build_output_dir
}
build_tar = create_build_tar(self.script_template, patched_parameters)
result = self.__container.put_archive(self.root_dir, data=build_tar)
if not result:
raise Exception("Failed to copy build files to container '{0}'"\
.format(self.__container.short_id))
logger.info("Copied build files to container '%s'",
self.__container.short_id)
def run(self):
with self.__cancel_lock:
if self.__cancelled:
logger.info("Build was cancelled")
return
logger.info("Start build in container '{0}'" \
.format(self.__container.short_id))
self.__container.start()
self.__container_logs = self.__container.logs(stream=True,
follow=True)
for byte_data in self.__container_logs:
line = byte_data.decode("utf-8").strip('\n\r')
self.__logs.put(line)
with self.__cancel_lock:
self.__container_logs = None
if self.__cancelled:
logger.info("Build was cancelled")
return
result = self.__container.wait()
try:
data, _ = self.__container.get_archive(self.build_output_dir)
self.build_output = extract_output_tar(data)
except docker.errors.APIError:
logger.error("Failed to obtain build output from container '%s'",
self.__container.short_id)
if result.get("StatusCode"):
raise Exception(
"Build in container '{0}' failed with status '{1}'".format(
self.__container.short_id, result.get("StatusCode")))
def cancel(self):
with self.__cancel_lock:
logger.info("Cancel build")
self.__cancelled = True
if self.__container_logs:
logger.info("Close logs")
self.__container_logs.close()
def __exit__(self, type, value, traceback):
if not self.__container:
return
try:
logger.info("Stop docker container '%s'",
self.__container.short_id)
self.__container.stop()
except docker.errors.APIError:
pass
try:
logger.info("Remove docker container '%s'",
self.__container.short_id)
self.__container.remove()
except docker.errors.APIError:
pass
def get_log_lines(self):
try:
while True:
yield self.__logs.get_nowait()
except Empty:
pass
class ManaWallDialog(basedialog.BaseDialog):
'''
manafirewall main dialog
'''
def __init__(self):
gettext.install('manafirewall',
localedir='/usr/share/locale',
names=('ngettext', ))
basedialog.BaseDialog.__init__(self,
_("Manatools - firewalld configurator"),
"", basedialog.DialogType.POPUP, 80, 10)
self._application_name = _(
"{} - ManaTools firewalld configurator").format(PROJECT)
# most used text
self.connected_label = _("Connection to firewalld established.")
self.trying_to_connect_label = \
_("Trying to connect to firewalld, waiting...")
self.failed_to_connect_label = \
_("Failed to connect to firewalld. Please make sure that the "
"service has been started correctly and try again.")
self.changes_applied_label = _("Changes applied.")
self.used_by_label = _("Used by network connection '%s'")
self.default_zone_used_by_label = _("Default zone used by network "
"connection '%s'")
self.enabled = _("enabled")
self.disabled = _("disabled")
self.connection_lost = False
self.log_denied = ""
self.automatic_helpers = ""
self.active_zones = {}
self.runtime_view = True
self.replacePointWidgetsAndCallbacks = []
self.fwEventQueue = SimpleQueue()
def UIlayout(self, layout):
'''
layout implementation called in base class to setup UI
'''
# Let's test a Menu widget
self.file_menu = self.factory.createMenuButton(
self.factory.createLeft(layout), _("&File"))
qm = yui.YMenuItem(_("&Quit"))
self.file_menu.addItem(qm)
self.file_menu.rebuildMenuTree()
sendObjOnEvent = True
self.eventManager.addMenuEvent(qm, self.onQuitEvent, sendObjOnEvent)
# _______
#| | |
#
cols = self.factory.createHBox(layout)
col1 = self.factory.createVBox(cols)
col2 = self.factory.createVBox(cols)
# Column 1
self.activeBindingsTree = self.factory.createTree(
col1, _("Active bindings"))
col1.setWeight(yui.YD_HORIZ, 30)
changeBindingsButton = self.factory.createPushButton(
col1, _("&Change binding"))
self.eventManager.addWidgetEvent(changeBindingsButton,
self.onChangeBinding, sendObjOnEvent)
#### editFrameBox contains button to modify zones (add, remove, edit, load defaults)
self.editFrameBox = self.factory.createFrame(col1, _("Edit zones"))
hbox = self.factory.createHBox(self.editFrameBox)
vbox1 = self.factory.createVBox(hbox)
vbox2 = self.factory.createVBox(hbox)
editFrameAddButton = self.factory.createPushButton(vbox1, _("&Add"))
self.eventManager.addWidgetEvent(editFrameAddButton,
self.onEditFrameAddButtonEvent)
editFrameEditButton = self.factory.createPushButton(vbox1, _("&Edit"))
self.eventManager.addWidgetEvent(editFrameEditButton,
self.onEditFrameEditButtonEvent)
editFrameRemoveButton = self.factory.createPushButton(
vbox2, _("&Remove"))
self.eventManager.addWidgetEvent(editFrameRemoveButton,
self.onEditFrameRemoveButtonEvent)
editFrameLoadDefaultsButton = self.factory.createPushButton(
vbox2, _("&Load default"))
self.eventManager.addWidgetEvent(
editFrameLoadDefaultsButton,
self.onEditFrameLoadDefaultsButtonEvent)
self.editFrameBox.setEnabled(False)
# Column 2
align = self.factory.createTop(col2) #self.factory.createLeft(col2)
align = self.factory.createLeft(align)
hbox = self.factory.createHBox(align)
col2.setWeight(yui.YD_HORIZ, 80)
#label = self.factory.createLabel(hbox, _("Configuration:"),False,False)
self.views = {
'runtime': {
'title': _("Runtime")
},
'permanent': {
'title': _("Permanent")
},
}
ordered_views = ['runtime', 'permanent']
self.currentViewCombobox = self.factory.createComboBox(
hbox, _("Configuration"))
itemColl = yui.YItemCollection()
for v in ordered_views:
item = yui.YItem(self.views[v]['title'], False)
show_item = 'runtime'
if show_item == v:
item.setSelected(True)
# adding item to views to find the item selected
self.views[v]['item'] = item
itemColl.push_back(item)
item.this.own(False)
self.currentViewCombobox.addItems(itemColl)
self.currentViewCombobox.setNotify(True)
self.eventManager.addWidgetEvent(self.currentViewCombobox,
self.onChangeView)
# mainNotebook (configure combo box)
# TODO icmp_types, helpers, direct_configurations, lockdown_whitelist
self.configureViews = {
'zones': {
'title': _("Zones")
},
'services': {
'title': _("Services")
},
'ipsets': {
'title': _("IP Sets")
},
}
ordered_configureViews = ['zones', 'services', 'ipsets']
self.configureViewCombobox = self.factory.createComboBox(
hbox, _("View"))
itemColl = yui.YItemCollection()
for v in ordered_configureViews:
item = yui.YItem(self.configureViews[v]['title'], False)
show_item = 'zones'
if show_item == v:
item.setSelected(True)
# adding item to views to find the item selected
self.configureViews[v]['item'] = item
itemColl.push_back(item)
item.this.own(False)
self.configureViewCombobox.addItems(itemColl)
self.configureViewCombobox.setNotify(True)
self.eventManager.addWidgetEvent(self.configureViewCombobox,
self.onConfigurationViewChanged)
# selectedConfigurationCombo is filled if requested by selected configuration view (which zones, services ...)
self.selectedConfigurationCombo = self.factory.createComboBox(
hbox, " ")
# adding a dummy item to enlarge combobox
item = yui.YItem("--------------------", False)
item.this.own(False)
self.selectedConfigurationCombo.addItem(item)
self.selectedConfigurationCombo.setEnabled(False)
self.selectedConfigurationCombo.setNotify(True)
self.eventManager.addWidgetEvent(
self.selectedConfigurationCombo,
self.onSelectedConfigurationComboChanged)
###
# ZoneNotebook and other (combo box to configure selected thing)
self.zoneConfigurationView = {
'services': {
'title': _("Services")
},
'ports': {
'title': _("Ports")
},
'protocols': {
'title': _("Protocols")
},
'source_ports': {
'title': _("Source Ports")
},
'masquerading': {
'title': _("Masquerading")
},
'port_forwarding': {
'title': _("Port Forwarding")
},
'icmp_filter': {
'title': _("ICMP Filter")
},
'rich_rules': {
'title': _("Rich Rules")
},
'interfaces': {
'title': _("Interfaces")
},
'sources': {
'title': _("Sources")
},
}
# ServiceNotebook
self.serviceConfigurationView = {
'ports': {
'title': _("Ports")
},
'protocols': {
'title': _("Protocols")
},
'source_ports': {
'title': _("Source Ports")
},
'modules': {
'title': _("Modules")
},
'destinations': {
'title': _("Destinations")
},
}
# ServiceNotebook
self.ipsecConfigurationView = {
'entries': {
'title': _("Entries")
},
}
self.configureCombobox = self.factory.createComboBox(
hbox, _("Configure"))
# adding a dummy item to enlarge combobox
itemColl = self._zoneConfigurationViewCollection()
self.configureCombobox.addItems(itemColl)
self.configureCombobox.setNotify(True)
self.eventManager.addWidgetEvent(self.configureCombobox,
self.onSelectedConfigurationChanged)
###
#### Replace Point to change configuration view
#self.rightPaneFrame = self.factory.createFrame(col2, "TEST")
self.replacePoint = self.factory.createReplacePoint(
col2) #self.rightPaneFrame)
self.configurationPanel = self.factory.createVBox(self.replacePoint)
self._replacePointServices()
#### bottom status lines
align = self.factory.createLeft(layout)
statusLine = self.factory.createHBox(align)
self.statusLabel = self.factory.createLabel(
statusLine, self.failed_to_connect_label)
align = self.factory.createLeft(layout)
statusLine = self.factory.createHBox(align)
self.defaultZoneLabel = self.factory.createLabel(
statusLine,
_("Default Zone: {}").format("--------"))
self.logDeniedLabel = self.factory.createLabel(
statusLine,
_("Log Denied: {}").format("--------"))
self.panicLabel = self.factory.createLabel(
statusLine,
_("Panic Mode: {}").format("--------"))
self.automaticHelpersLabel = self.factory.createLabel(
statusLine,
_("Automatic Helpers: {}").format("--------"))
self.lockdownLabel = self.factory.createLabel(
statusLine,
_("Lockdown: {}").format("--------"))
#### buttons on the last line
align = self.factory.createRight(layout)
bottomLine = self.factory.createHBox(align)
aboutButton = self.factory.createPushButton(bottomLine, _("&About"))
self.eventManager.addWidgetEvent(aboutButton, self.onAbout)
quitButton = self.factory.createPushButton(bottomLine, _("&Quit"))
self.eventManager.addWidgetEvent(quitButton, self.onQuitEvent,
sendObjOnEvent)
# Let's test a cancel event
self.eventManager.addCancelEvent(self.onCancelEvent)
# Let's check external events every 100 msec
self.timeout = 100
#self.eventManager.addTimeOutEvent(self.onTimeOutEvent)
# End Dialof layout
self.initFWClient()
def _replacePointServices(self):
'''
draw services frame
'''
if len(self.replacePointWidgetsAndCallbacks) > 0:
print("Error there are still widget events for ReplacePoint"
) #TODO log
return
if self.configurationPanel.hasChildren():
print("Error there are still widgets into ReplacePoint") #TODO log
return
#self.mgaFactory.create
services_header = yui.YTableHeader()
columns = [_('Service')]
services_header.addColumn("")
for col in (columns):
services_header.addColumn(col)
self.serviceList = self.mgaFactory.createCBTable(
self.configurationPanel, services_header,
yui.YCBTableCheckBoxOnFirstColumn)
self.serviceList.setImmediateMode(True)
if isinstance(self.serviceList, yui.YMGA_CBTable):
print("YMGA_CBTable")
self.replacePointWidgetsAndCallbacks.append({
'widget':
self.serviceList,
'action':
self.onRPServiceChecked
})
self.eventManager.addWidgetEvent(self.serviceList,
self.onRPServiceChecked)
def _fillRPServices(self):
services = None
if self.runtime_view:
services = self.fw.listServices()
else:
services = self.fw.config().getServiceNames()
v = []
for service in services:
item = yui.YCBTableItem(service)
item.check(False)
item.this.own(False)
v.append(item)
#NOTE workaround to get YItemCollection working in python
itemCollection = yui.YItemCollection(v)
self.serviceList.startMultipleChanges()
# cleanup old changed items since we are removing all of them
self.serviceList.setChangedItem(None)
self.serviceList.addItems(itemCollection)
self.serviceList.doneMultipleChanges()
def onRPServiceChecked(self, widgetEvent):
'''
works on enabling/disabling service for zone
'''
if (widgetEvent.reason() == yui.YEvent.ValueChanged):
item = self.serviceList.changedItem()
if item:
if item.checked():
print("%s checked" % item.cell(0).label())
else:
print("%s unchecked" % item.cell(0).label())
def _zoneConfigurationViewCollection(self):
'''
returns an YItemCollection containing Zone configuration views
'''
ordered_configureViews = [
'services', 'ports', 'protocols', 'source_ports', 'masquerading',
'port_forwarding', 'icmp_filter', 'rich_rules', 'interfaces',
'sources'
]
itemColl = yui.YItemCollection()
for v in ordered_configureViews:
item = yui.YItem(self.zoneConfigurationView[v]['title'], False)
show_item = 'services'
if show_item == v:
item.setSelected(True)
# adding item to views to find the item selected
self.zoneConfigurationView[v]['item'] = item
itemColl.push_back(item)
item.this.own(False)
return itemColl
def _serviceConfigurationViewCollection(self):
'''
returns an YItemCollection containing Service configuration views
'''
ordered_Views = [
'ports', 'protocols', 'source_ports', 'modules', 'destinations'
]
itemColl = yui.YItemCollection()
for v in ordered_Views:
item = yui.YItem(self.serviceConfigurationView[v]['title'], False)
show_item = 'ports'
if show_item == v:
item.setSelected(True)
# adding item to views to find the item selected
self.serviceConfigurationView[v]['item'] = item
itemColl.push_back(item)
item.this.own(False)
return itemColl
def _ipsecConfigurationViewCollection(self):
'''
returns an YItemCollection containing IPSEC configuration views
'''
ordered_Views = [
'entries',
]
itemColl = yui.YItemCollection()
for v in ordered_Views:
item = yui.YItem(self.ipsecConfigurationView[v]['title'], False)
show_item = 'entries'
if show_item == v:
item.setSelected(True)
# adding item to views to find the item selected
self.ipsecConfigurationView[v]['item'] = item
itemColl.push_back(item)
item.this.own(False)
return itemColl
def _exception_handler(self, exception_message):
if not self.__use_exception_handler:
raise
def initFWClient(self):
'''
initialize firewall client
'''
self.fw = client.FirewallClient(wait=1)
self.__use_exception_handler = True
self.fw.setExceptionHandler(self._exception_handler)
self.fw.setNotAuthorizedLoop(True)
self.fw.connect("connection-changed", self.fwConnectionChanged)
self.fw.connect("lockdown-enabled", self.lockdown_enabled_cb)
self.fw.connect("lockdown-disabled", self.lockdown_disabled_cb)
self.fw.connect("panic-mode-enabled", self.panic_mode_enabled_cb)
self.fw.connect("panic-mode-disabled", self.panic_mode_disabled_cb)
self.fw.connect("default-zone-changed", self.default_zone_changed_cb)
self.fw.connect("config:zone-added", self.conf_zone_changed_cb)
self.fw.connect("config:zone-updated", self.conf_zone_changed_cb)
self.fw.connect("config:zone-removed", self.conf_zone_changed_cb)
self.fw.connect("config:zone-renamed", self.conf_zone_changed_cb)
self.fw.connect("config:service-added", self.conf_service_changed_cb)
self.fw.connect("config:service-updated", self.conf_service_changed_cb)
self.fw.connect("config:service-removed", self.conf_service_changed_cb)
self.fw.connect("config:service-renamed", self.conf_service_changed_cb)
def load_zones(self, selected=None):
'''
load zones into selectedConfigurationCombo
'''
self.selectedConfigurationCombo.startMultipleChanges()
self.selectedConfigurationCombo.deleteAllItems()
self.selectedConfigurationCombo.setEnabled(True)
self.selectedConfigurationCombo.setLabel(
self.configureViews['zones']['title'])
zones = []
if self.runtime_view:
zones = self.fw.getZones()
else:
zones = self.fw.config().getZoneNames()
selected_zone = selected
if selected not in zones:
selected_zone = self.fw.getDefaultZone()
# zones
itemColl = yui.YItemCollection()
for zone in zones:
item = yui.YItem(zone, False)
if zone == selected_zone:
item.setSelected(True)
itemColl.push_back(item)
item.this.own(False)
self.selectedConfigurationCombo.addItems(itemColl)
self.selectedConfigurationCombo.doneMultipleChanges()
def load_services(self, service_name=None):
'''
load services into selectedConfigurationCombo
'''
self.selectedConfigurationCombo.startMultipleChanges()
self.selectedConfigurationCombo.deleteAllItems()
self.selectedConfigurationCombo.setEnabled(True)
self.selectedConfigurationCombo.setLabel(
self.configureViews['services']['title'])
services = []
if self.runtime_view:
services = self.fw.listServices()
else:
services = self.fw.config().getServiceNames()
# services
itemColl = yui.YItemCollection()
for service in services:
item = yui.YItem(service, False)
if service == service_name:
item.setSelected(True)
itemColl.push_back(item)
item.this.own(False)
self.selectedConfigurationCombo.addItems(itemColl)
self.selectedConfigurationCombo.doneMultipleChanges()
def load_ipsets(self):
'''
load ipsets into selectedConfigurationCombo
'''
self.selectedConfigurationCombo.startMultipleChanges()
self.selectedConfigurationCombo.deleteAllItems()
self.selectedConfigurationCombo.setEnabled(True)
self.selectedConfigurationCombo.setLabel(
self.configureViews['ipsets']['title'])
ipsets = []
if self.runtime_view:
ipsets = self.fw.getIPSets()
else:
ipsets = self.fw.config().getIPSetNames()
# ipsets
itemColl = yui.YItemCollection()
for ipset in ipsets:
item = yui.YItem(ipset, False)
itemColl.push_back(item)
item.this.own(False)
self.selectedConfigurationCombo.addItems(itemColl)
self.selectedConfigurationCombo.doneMultipleChanges()
#### Firewall events
def fwConnectionChanged(self):
'''
connection changed
'''
if self.fw.connected:
self.fwEventQueue.put({
'event': "connection-changed",
'value': True
})
print("connected")
else:
self.fwEventQueue.put({
'event': "connection-changed",
'value': False
})
print("disc")
def lockdown_enabled_cb(self):
'''
manage lockdown enabled evend from firewalld
'''
self.fwEventQueue.put({'event': "lockdown-changed", 'value': True})
def lockdown_disabled_cb(self):
'''
manage lockdown disabled evend from firewalld
'''
self.fwEventQueue.put({'event': "lockdown-changed", 'value': False})
def panic_mode_enabled_cb(self):
'''
manage panicmode enabled evend from firewalld
'''
self.fwEventQueue.put({'event': "panicmode-changed", 'value': True})
def panic_mode_disabled_cb(self):
'''
manage panicmode disabled evend from firewalld
'''
self.fwEventQueue.put({'event': "panicmode-changed", 'value': False})
def default_zone_changed_cb(self, zone):
'''
manage default zone changed from firewalld
'''
self.fwEventQueue.put({'event': "default-zone-changed", 'value': zone})
def conf_zone_changed_cb(self, zone):
'''
zones have been modified
'''
self.fwEventQueue.put({'event': "config-zone-changed", 'value': zone})
def conf_service_changed_cb(self, service):
'''
service have been modified
'''
self.fwEventQueue.put({
'event': "config-service-changed",
'value': service
})
#### GUI events
def onCancelEvent(self):
'''
Exit by using cancel event
'''
print("Got a cancel event")
def onQuitEvent(self, obj):
'''
Exit by using quit button or menu
'''
if isinstance(obj, yui.YItem):
print("Quit menu pressed")
else:
print("Quit button pressed")
# BaseDialog needs to force to exit the handle event loop
self.ExitLoop()
def onChangeBinding(self, obj):
'''
manages changeBindingsButton button pressed
'''
print("TODO: Change binding pressed")
def onAbout(self):
'''
About dialog invoked
'''
ok = common.AboutDialog({
'name':
self._application_name,
'dialog_mode':
common.AboutDialogMode.TABBED,
'version':
VERSION,
'credits':
_("Credits {}").format("2019 Angelo Naselli"),
'license':
'GPLv2+',
'authors':
'Angelo Naselli <[email protected]>',
'description':
_("{} is a graphical configuration tool for firewalld.").format(
PROJECT),
'size': {
'column': 50,
'lines': 6
},
})
def onChangeView(self):
'''
manages currentViewCombobox chenges
'''
item = self.currentViewCombobox.selectedItem()
self.runtime_view = item == self.views['runtime']['item']
# Let's change view as if a new configuration has been chosen
self.onConfigurationViewChanged()
self.editFrameBox.setEnabled(not self.runtime_view)
def onEditFrameAddButtonEvent(self):
'''
from edit frame Add has benn pressed, let's call the right add event
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
self.onAddZone()
elif item == self.configureViews['services']['item']:
#Services selected
self.onServiceConfAddService()
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onEditFrameEditButtonEvent(self):
'''
from edit frame Edit has benn pressed, let's call the right edit event
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
self.onEditZone()
elif item == self.configureViews['services']['item']:
#Services selected
self.onServiceConfEditService()
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onEditFrameRemoveButtonEvent(self):
'''
from edit frame Remove has benn pressed, let's call the right remove event
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
self.onRemoveZone()
elif item == self.configureViews['services']['item']:
#Services selected
self.onServiceConfRemoveService()
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onEditFrameLoadDefaultsButtonEvent(self):
'''
from edit frame Remove has benn pressed, let's call the right remove event
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
self.onLoadDefaultsZone()
elif item == self.configureViews['services']['item']:
#Services selected
self.onServiceConfLoadDefaultsService()
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onAddZone(self):
'''
manages add zone button
'''
if self.runtime_view:
return
self._add_edit_zone(True)
self.load_zones()
def onRemoveZone(self):
'''
manages remove zone button
'''
if self.runtime_view:
return
selected_zoneitem = self.selectedConfigurationCombo.selectedItem()
if selected_zoneitem:
selected_zone = selected_zoneitem.label()
zone = self.fw.config().getZoneByName(selected_zone)
zone.remove()
self.load_zones()
# TODO self.onChangeZone()
def onEditZone(self):
'''
manages edit zone button
'''
if self.runtime_view:
return
self._add_edit_zone(False)
selected_zone = None
selected_zoneitem = self.selectedConfigurationCombo.selectedItem()
if selected_zoneitem:
selected_zone = selected_zoneitem.label()
self.load_zones(selected_zone)
def onLoadDefaultsZone(self):
'''
manages load defaults zone Button
'''
if self.runtime_view:
return
selected_zoneitem = self.selectedConfigurationCombo.selectedItem()
if selected_zoneitem:
selected_zone = selected_zoneitem.label()
zone = self.fw.config().getZoneByName(selected_zone)
zone.loadDefaults()
# TODO self.onChangeZone()
def _add_edit_zone(self, add):
'''
adds or edit zone (parameter add True if adding)
'''
zoneBaseInfo = {}
zoneBaseInfo['max_zone_name_len'] = functions.max_zone_name_len()
if not add:
# fill zoneBaseInfo for zoneBaseDialog fields
selected_zoneitem = self.selectedConfigurationCombo.selectedItem()
if selected_zoneitem:
selected_zone = selected_zoneitem.label()
zone = self.fw.config().getZoneByName(selected_zone)
settings = zone.getSettings()
props = zone.get_properties()
zoneBaseInfo['name'] = zone.get_property("name")
zoneBaseInfo['version'] = settings.getVersion()
zoneBaseInfo['short'] = settings.getShort()
zoneBaseInfo['description'] = settings.getDescription()
zoneBaseInfo['default'] = props["default"]
zoneBaseInfo['builtin'] = props["builtin"]
zoneBaseInfo['target'] = settings.getTarget()
if zoneBaseInfo['target'] == DEFAULT_ZONE_TARGET:
zoneBaseInfo['target'] = 'default'
zoneBaseDlg = zoneBaseDialog.ZoneBaseDialog(zoneBaseInfo)
newZoneBaseInfo = zoneBaseDlg.run()
# Cancelled if None is returned
if newZoneBaseInfo is None:
return
if not add:
if zoneBaseInfo['name'] == newZoneBaseInfo['name'] and \
zoneBaseInfo['version'] == newZoneBaseInfo['version'] and \
zoneBaseInfo['short'] == newZoneBaseInfo['short'] and \
zoneBaseInfo['description'] == newZoneBaseInfo['description'] and \
zoneBaseInfo['target'] == newZoneBaseInfo['target']:
# no changes
return
selected_zoneitem = self.selectedConfigurationCombo.selectedItem()
if selected_zoneitem:
selected_zone = selected_zoneitem.label()
zone = self.fw.config().getZoneByName(selected_zone)
if zoneBaseInfo['version'] != newZoneBaseInfo['version'] or \
zoneBaseInfo['short'] != newZoneBaseInfo['short'] or \
zoneBaseInfo['description'] != newZoneBaseInfo['description'] or \
zoneBaseInfo['target'] != newZoneBaseInfo['target']:
settings = zone.getSettings()
settings.setVersion(newZoneBaseInfo['version'])
settings.setShort(newZoneBaseInfo['short'])
settings.setDescription(newZoneBaseInfo['description'])
settings.setTarget(newZoneBaseInfo['target'])
zone.update(settings)
if zoneBaseInfo['name'] == newZoneBaseInfo['name']:
return
zone.rename(newZoneBaseInfo['name'])
else:
settings = client.FirewallClientZoneSettings()
settings.setVersion(newZoneBaseInfo['version'])
settings.setShort(newZoneBaseInfo['short'])
settings.setDescription(newZoneBaseInfo['description'])
settings.setTarget(newZoneBaseInfo['target'])
self.fw.config().addZone(newZoneBaseInfo['name'], settings)
def onServiceConfAddService(self, *args):
'''
manages add service button
'''
if self.runtime_view:
return
self._add_edit_service(True)
def onServiceConfRemoveService(self, *args):
'''
manages remove zone button
'''
if self.runtime_view:
return
selected_item = self.selectedConfigurationCombo.selectedItem()
if selected_item:
active_service = selected_item.label()
service = self.fw.config().getServiceByName(active_service)
service.remove()
self.load_services()
# TODO self.onChangeService()
def onServiceConfEditService(self, *args):
if self.runtime_view:
return
self._add_edit_service(False)
def onServiceConfLoadDefaultsService(self, *args):
'''
manages load defaults service Button
'''
if self.runtime_view:
return
selected_item = self.selectedConfigurationCombo.selectedItem()
if selected_item:
active_service = selected_item.label()
service = self.fw.config().getServiceByName(active_service)
service.loadDefaults()
# TODO self.onChangeService()
def _add_edit_service(self, add):
'''
adds or edit service (parameter add True if adding)
'''
serviceBaseInfo = {}
if not add:
# fill serviceBaseInfo for serviceBaseDialog fields
selected_serviceitem = self.selectedConfigurationCombo.selectedItem(
)
if selected_serviceitem:
active_service = selected_serviceitem.label()
service = self.fw.config().getServiceByName(active_service)
settings = service.getSettings()
props = service.get_properties()
serviceBaseInfo['default'] = props["default"]
serviceBaseInfo['builtin'] = props["builtin"]
serviceBaseInfo['name'] = service.get_property("name")
serviceBaseInfo['version'] = settings.getVersion()
serviceBaseInfo['short'] = settings.getShort()
serviceBaseInfo['description'] = settings.getDescription()
serviceBaseDlg = serviceBaseDialog.ServiceBaseDialog(serviceBaseInfo)
newServiceBaseInfo = serviceBaseDlg.run()
# Cancelled if None is returned
if newServiceBaseInfo is None:
return
if not add:
if serviceBaseInfo['name'] == newServiceBaseInfo['name'] and \
serviceBaseInfo['version'] == newServiceBaseInfo['version'] and \
serviceBaseInfo['short'] == newServiceBaseInfo['short'] and \
serviceBaseInfo['description'] == newServiceBaseInfo['description']:
# no changes
return
selected_serviceitem = self.selectedConfigurationCombo.selectedItem(
)
if selected_serviceitem:
selected_service = selected_serviceitem.label()
service = self.fw.config().getServiceByName(active_service)
if serviceBaseInfo['version'] != newServiceBaseInfo['version'] or \
serviceBaseInfo['short'] != newServiceBaseInfo['short'] or \
serviceBaseInfo['description'] != newServiceBaseInfo['description']:
settings = service.getSettings()
settings.setVersion(newServiceBaseInfo['version'])
settings.setShort(newServiceBaseInfo['short'])
settings.setDescription(newServiceBaseInfo['description'])
service.update(settings)
if serviceBaseInfo['name'] == newServiceBaseInfo['name']:
return
service.rename(newServiceBaseInfo['name'])
else:
settings = client.FirewallClientServiceSettings()
settings.setVersion(newServiceBaseInfo['version'])
settings.setShort(newServiceBaseInfo['short'])
settings.setDescription(newServiceBaseInfo['description'])
self.fw.config().addService(newServiceBaseInfo['name'], settings)
def onSelectedConfigurationComboChanged(self):
'''
depending on what configuration view is selected it manages zones,
services, etc
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
# self.onChangeZone()
pass
elif item == self.configureViews['services']['item']:
#Services selected
pass
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onConfigurationViewChanged(self):
'''
manages configureViewCombobox changes
'''
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
self.load_zones()
self.configureCombobox.startMultipleChanges()
self.configureCombobox.deleteAllItems()
itemColl = self._zoneConfigurationViewCollection()
self.configureCombobox.addItems(itemColl)
self.configureCombobox.setEnabled(True)
self.configureCombobox.doneMultipleChanges()
self.editFrameBox.setLabel(_("Edit zones"))
elif item == self.configureViews['services']['item']:
#Services selected
self.load_services()
self.configureCombobox.startMultipleChanges()
self.configureCombobox.deleteAllItems()
itemColl = self._serviceConfigurationViewCollection()
self.configureCombobox.addItems(itemColl)
self.configureCombobox.setEnabled(True)
self.configureCombobox.doneMultipleChanges()
self.editFrameBox.setLabel(_("Edit services"))
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
self.load_ipsets()
self.configureCombobox.startMultipleChanges()
self.configureCombobox.deleteAllItems()
itemColl = self._ipsecConfigurationViewCollection()
self.configureCombobox.addItems(itemColl)
self.configureCombobox.setEnabled(True)
self.configureCombobox.doneMultipleChanges()
self.editFrameBox.setLabel(_("Edit ipsets"))
else:
# disabling info combo
self.selectedConfigurationCombo.startMultipleChanges()
self.selectedConfigurationCombo.deleteAllItems()
self.selectedConfigurationCombo.setLabel(" ")
self.selectedConfigurationCombo.setEnabled(False)
self.selectedConfigurationCombo.doneMultipleChanges()
#disabling configure view
self.configureCombobox.startMultipleChanges()
self.configureCombobox.deleteAllItems()
self.configureCombobox.setEnabled(False)
self.configureCombobox.doneMultipleChanges()
self.editFrameBox.setLabel("")
def onSelectedConfigurationChanged(self):
'''
manages configureCombobox changes
'''
config_item = self.configureCombobox.selectedItem()
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
#Zones selected
if config_item == self.zoneConfigurationView['services']['item']:
self._fillRPServices()
elif item == self.configureViews['services']['item']:
#Services selected
pass
elif item == self.configureViews['ipsets']['item']:
# ip sets selected
pass
def onTimeOutEvent(self):
print("Timeout occurred")
def doSomethingIntoLoop(self):
'''
check on internal queue if any fw event has been managed
'''
try:
item = self.fwEventQueue.get_nowait()
# managing deferred firewall events
if item['event'] == 'connection-changed':
connected = item['value']
self.connection_lost = not connected
t = self.connected_label if connected else self.trying_to_connect_label
self.statusLabel.setText(t)
if connected:
self.fw.authorizeAll()
default_zone = self.fw.getDefaultZone()
self.defaultZoneLabel.setText(
_("Default Zone: {}").format(default_zone))
self.log_denied = self.fw.getLogDenied()
self.logDeniedLabel.setText(
("Log Denied: {}").format(self.log_denied))
self.automatic_helpers = self.fw.getAutomaticHelpers()
self.automaticHelpersLabel.setText(
_("Automatic Helpers: {}").format(
self.automatic_helpers))
#### TODO self.set_automaticHelpersLabel(self.automatic_helpers)
lockdown = self.fw.queryLockdown()
t = self.enabled if lockdown else self.disabled
self.lockdownLabel.setText(_("Lockdown: {}").format(t))
panic = self.fw.queryPanicMode()
t = self.enabled if panic else self.disabled
self.panicLabel.setText(_("Panic Mode: {}").format(t))
else:
self.defaultZoneLabel.setText(
_("Default Zone: {}").format("--------"))
self.logDeniedLabel.setText(
("Log Denied: {}").format("--------"))
self.automaticHelpersLabel.setText(
_("Automatic Helpers: {}").format("--------"))
self.lockdownLabel.setText(
_("Lockdown: {}").format("--------"))
self.panicLabel.setText(
_("Panic Mode: {}").format("--------"))
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
self.load_zones()
elif item['event'] == 'lockdown-changed':
t = self.enabled if item['value'] else self.disabled
self.lockdownLabel.setText(_("Lockdown: {}").format(t))
# TODO manage menu items if needed
elif item['event'] == 'panicmode-changed':
t = self.enabled if item['value'] else self.disabled
self.panicLabel.setText(_("Panic Mode: {}").format(t))
# TODO manage menu items if needed
elif item['event'] == 'default-zone-changed':
zone = item['value']
self.defaultZoneLabel.setText(
_("Default Zone: {}").format(zone))
# TODO self.update_active_zones()
elif item['event'] == 'config-zone-changed':
zone = item['value']
if not self.runtime_view:
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['zones']['item']:
# Zones selected
selected_zone = None
selected_item = self.selectedConfigurationCombo.selectedItem(
)
if selected_item:
selected_zone = selected_item.label()
self.load_zones(selected_zone)
elif item['event'] == 'config-service-changed':
service = item['value']
if not self.runtime_view:
item = self.configureViewCombobox.selectedItem()
if item == self.configureViews['services']['item']:
# Services selected
selected_service = None
selected_item = self.selectedConfigurationCombo.selectedItem(
)
if selected_item:
selected_service = selected_item.label()
self.load_services(selected_service)
except Empty as e:
pass
class NetworkConnection(object):
class State(object):
kCreated = 0
kInit = 1
kHandshake = 2
kSynchronized = 3
kActive = 4
kDead = 5
def __init__(self, uid, stream, notifier, handshake, get_entry_type, verbose=False):
# logging debugging
self.m_verbose = verbose
self.m_uid = uid
self.m_stream = stream
self.m_notifier = notifier
self.m_handshake = handshake
self.m_get_entry_type = get_entry_type
self.m_active = False
self.m_proto_rev = 0x0300
self.state = self.State.kCreated
self.m_state_mutex = threading.Lock()
self.m_last_update = 0
self.m_outgoing = Queue()
self.m_process_incoming = None
self.m_read_thread = None
self.m_write_thread = None
self.m_remote_id_mutex = threading.Lock()
self.m_remote_id = None
self.m_last_post = 0
self.m_pending_mutex = threading.Lock()
self.m_pending_outgoing = []
self.m_pending_update = {}
# Condition variables for shutdown
self.m_shutdown_mutex = threading.Lock()
# Not needed in python
# self.m_read_shutdown_cv = threading.Condition()
# self.m_write_shutdown_cv = threading.Condition()
self.m_read_shutdown = False
self.m_write_shutdown = False
# turn off Nagle algorithm; we bundle packets for transmission
try:
self.m_stream.setNoDelay()
except IOError as e:
logger.warning("Setting TCP_NODELAY: %s", e)
def start(self):
if self.m_active:
return
self.m_active = True
self.set_state(self.State.kInit)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
# reset shutdown flags
with self.m_shutdown_mutex:
self.m_read_shutdown = False
self.m_write_shutdown = False
# start threads
self.m_write_thread = SafeThread(
target=self._writeThreadMain, name="nt-net-write"
)
self.m_read_thread = SafeThread(target=self._readThreadMain, name="nt-net-read")
def __repr__(self):
try:
return "<NetworkConnection 0x%x %s>" % (id(self), self.info())
except Exception:
return "<NetworkConnection 0x%x ???>" % id(self)
def stop(self):
logger.debug("NetworkConnection stopping (%s)", self)
if not self.m_active:
return
self.set_state(self.State.kDead)
self.m_active = False
# closing the stream so the read thread terminates
self.m_stream.close()
# send an empty outgoing message set so the write thread terminates
self.m_outgoing.put([])
# wait for threads to terminate, timeout
self.m_write_thread.join(1)
if self.m_write_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
self.m_read_thread.join(1)
if self.m_read_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
def get_proto_rev(self):
return self.m_proto_rev
def get_stream(self):
return self.m_stream
def info(self):
return ConnectionInfo(
self.remote_id(),
self.m_stream.getPeerIP(),
self.m_stream.getPeerPort(),
self.m_last_update,
self.m_proto_rev,
)
def is_connected(self):
return self.state == self.State.kActive
def last_update(self):
return self.m_last_update
def set_process_incoming(self, func):
self.m_process_incoming = func
def set_proto_rev(self, proto_rev):
self.m_proto_rev = proto_rev
def set_state(self, state):
with self.m_state_mutex:
State = self.State
# Don't update state any more once we've died
if self.state == State.kDead:
return
# One-shot notify state changes
if self.state != State.kActive and state == State.kActive:
info = self.info()
self.m_notifier.notifyConnection(True, info)
logger.info(
"CONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
elif self.state != State.kDead and state == State.kDead:
info = self.info()
self.m_notifier.notifyConnection(False, info)
logger.info(
"DISCONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
if self.m_verbose:
logger.debug(
"%s: %s -> %s", self, _state_map[self.state], _state_map[state]
)
self.state = state
# python optimization: don't use getter here
# def state(self):
# return self.m_state
def remote_id(self):
with self.m_remote_id_mutex:
return self.m_remote_id
def set_remote_id(self, remote_id):
with self.m_remote_id_mutex:
self.m_remote_id = remote_id
def uid(self):
return self.m_uid
def _sendMessages(self, msgs):
self.m_outgoing.put(msgs)
def _readThreadMain(self):
decoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
def _getMessage():
decoder.set_proto_rev(self.m_proto_rev)
try:
return Message.read(self.m_stream, decoder, self.m_get_entry_type)
except IOError as e:
logger.warning("read error in handshake: %s", e)
# terminate connection on bad message
self.m_stream.close()
return None
self.set_state(self.State.kHandshake)
try:
handshake_success = self.m_handshake(self, _getMessage, self._sendMessages)
except Exception:
logger.exception("Unhandled exception during handshake")
handshake_success = False
if not handshake_success:
self.set_state(self.State.kDead)
self.m_active = False
else:
self.set_state(self.State.kActive)
try:
while self.m_active:
if not self.m_stream:
break
decoder.set_proto_rev(self.m_proto_rev)
try:
msg = Message.read(
self.m_stream, decoder, self.m_get_entry_type
)
except Exception as e:
if not isinstance(e, StreamEOF):
if verbose:
logger.exception("read error")
else:
logger.warning("read error: %s", e)
# terminate connection on bad message
self.m_stream.close()
break
if verbose:
logger.debug(
"%s received type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
self.m_last_update = monotonic()
self.m_process_incoming(msg, self)
except IOError as e:
# connection died probably
logger.debug("IOError in read thread: %s", e)
except Exception:
logger.warning("Unhandled exception in read thread", exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
# also kill write thread
self.m_outgoing.put([])
with self.m_shutdown_mutex:
self.m_read_shutdown = True
def _writeThreadMain(self):
encoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
out = []
try:
while self.m_active:
msgs = self.m_outgoing.get()
if verbose:
logger.debug("write thread woke up")
if msgs:
logger.debug(
"%s sending %s messages", self.m_stream.sock_type, len(msgs)
)
if not msgs:
continue
encoder.set_proto_rev(self.m_proto_rev)
# python-optimization: checking verbose causes extra overhead
if verbose:
for msg in msgs:
if msg:
logger.debug(
"%s sending type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
Message.write(msg, out, encoder)
else:
for msg in msgs:
if msg:
Message.write(msg, out, encoder)
if not self.m_stream:
break
if not out:
continue
self.m_stream.send(b"".join(out))
del out[:]
# if verbose:
# logger.debug('send %s bytes', encoder.size())
except IOError as e:
# connection died probably
if not isinstance(e, StreamEOF):
logger.debug("IOError in write thread: %s", e)
except Exception:
logger.warning("Unhandled exception in write thread", exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
self.m_stream.close() # also kill read thread
with self.m_shutdown_mutex:
self.m_write_shutdown = True
def queueOutgoing(self, msg):
with self.m_pending_mutex:
# Merge with previous. One case we don't combine: delete/assign loop.
msgtype = msg.type
if msgtype in [kEntryAssign, kEntryUpdate]:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.first != 0:
# overwrite the previous one for this id
oldidx = mpend.first - 1
oldmsg = self.m_pending_outgoing[oldidx]
if (
oldmsg
and oldmsg.type == kEntryAssign
and msgtype == kEntryUpdate
):
# need to update assignment with seq_num and value
oldmsg = Message.entryAssign(
oldmsg.str, msg_id, msg.seq_num_uid, msg.value, oldmsg.flags
)
else:
oldmsg = msg # easy update
self.m_pending_outgoing[oldidx] = oldmsg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(pos + 1, 0)
elif msgtype == kEntryDelete:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
# clear previous updates
mpend = self.m_pending_update.get(msg_id)
if mpend is not None:
if mpend.first != 0:
self.m_pending_outgoing[mpend.first - 1] = None
if mpend.second != 0:
self.m_pending_outgoing[mpend.second - 1] = None
self.m_pending_update[msg_id] = _empty_pair
# add deletion
self.m_pending_outgoing.append(msg)
elif msgtype == kFlagsUpdate:
# don't do this for unassigned id's
msg_id = msg.id
if id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.second != 0:
# overwrite the previous one for this id
self.m_pending_outgoing[mpend.second - 1] = msg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(0, pos + 1)
elif msgtype == kClearEntries:
# knock out all previous assigns/updates!
for i, m in enumerate(self.m_pending_outgoing):
if not m:
continue
t = m.type
if t in [
kEntryAssign,
kEntryUpdate,
kFlagsUpdate,
kEntryDelete,
kClearEntries,
]:
self.m_pending_outgoing[i] = None
self.m_pending_update.clear()
self.m_pending_outgoing.append(msg)
else:
self.m_pending_outgoing.append(msg)
def postOutgoing(self, keep_alive):
with self.m_pending_mutex:
# optimization: don't call monotonic unless needed
# now = monotonic()
if not self.m_pending_outgoing:
if not keep_alive:
return
# send keep-alives once a second (if no other messages have been sent)
now = monotonic()
if (now - self.m_last_post) < 1.0:
return
self.m_outgoing.put((Message.keepAlive(),))
else:
now = monotonic()
self.m_outgoing.put(self.m_pending_outgoing)
self.m_pending_outgoing = []
self.m_pending_update.clear()
self.m_last_post = now
class Port:
def __init__(self, parser, portNotOpenException, PortExceptionType = None):
self.__PortExceptionType = PortExceptionType or type(portNotOpenException)
self.__autoOpenOnWrite = False
self.__connectionListeners = set()
self.__debugRead = False
self.__debugWrite = False
self.__errorProcessor = print
self.__packet = None
self.__parser = parser
self.__path = None
self.__portNotOpenException = portNotOpenException
self.__queue = SimpleQueue()
self.__throw = False
@property
def autoOpenOnWrite(self):
return self.__autoOpenOnWrite
@autoOpenOnWrite.setter
def autoOpenOnWrite(self, autoOpenOnWrite):
self.__autoOpenOnWrite = autoOpenOnWrite
@property
def debugRead(self):
return self.__debugRead
@debugRead.setter
def debugRead(self, debugRead):
self.__debugRead = debugRead
@property
def debugWrite(self):
return self.__debugWrite
@debugWrite.setter
def debugWrite(self, debugWrite):
self.__debugWrite = debugWrite
@property
def errorProcessor(self):
return self.__errorProcessor
@errorProcessor.setter
def errorProcessor(self, errorProcessor):
self.__errorProcessor = errorProcessor
@property
def parser(self):
return self.__parser
@property
def path(self):
return self.__path
@path.setter
def path(self, path):
self.__path = path
@property
def throw(self):
return self.__throw
@throw.setter
def throw(self, throw):
self.__throw = throw
def Packet(self, **kw):
return Packet(self.__parser.format, **kw)
def addConnectionListener(self, connectionListener):
self.__addRemoveConnectionListener(True, connectionListener)
def addQueueItem(self, item):
self.__queue.put_nowait(item)
def close(self):
if self.isOpen():
self._close()
def isOpen(self):
StaticUtils.notImplemented()
def open(self, path = None, **kw):
try:
self._open(path, **kw)
return True
except self.__PortExceptionType as e:
self._processError(e)
finally:
self.__throw = False
def packet(self, **kw):
self.__packet = self.Packet(**kw)
return self
def removeConnectionListener(self, connectionListener):
self.__addRemoveConnectionListener(False, connectionListener)
def processQueue(self):
try:
item = self.__queue.get_nowait()
if isinstance(item, ConnectionEstablished):
for connectionListener in self.__connectionListeners:
connectionListener.connectionEstablished(self)
elif isinstance(item, ConnectionLost):
for connectionListener in self.__connectionListeners:
connectionListener.connectionLost(self, item.e)
elif isinstance(item, DataReceived):
self.__parser.parse(item.data)
elif isinstance(item, ProcessError):
self.__errorProcessor(item.e)
else:
raise UnknownItem(item)
except Empty:
pass
def write(self, packet = None, throw = None):
if not packet:
packet = self.__packet
if throw is not None:
self.__throw = throw
try:
if self.__debugWrite:
print(packet)
else:
if not self.isOpen():
if self.__autoOpenOnWrite:
self._open()
else:
raise self.__portNotOpenException
self._write(packet)
return True
except self.__PortExceptionType as e:
self._processError(e)
finally:
self.__packet = None
self.__throw = False
def _close(self):
StaticUtils.notImplemented()
def _open(self, path = None, **kw):
StaticUtils.notImplemented()
def _processError(self, e):
if self.__throw:
raise e
if self.__errorProcessor:
self.addQueueItem(ProcessError(e))
def _write(self, packet):
StaticUtils.notImplemented()
def __addRemoveConnectionListener(self, add, connectionListener):
StaticUtils.assertInheritance(connectionListener, ConnectionListener, "connectionListener")
getattr(self.__connectionListeners, "add" if add else "remove")(connectionListener)
def exec(self):
logging.info('Start to find requests')
visited = set()
session = HTMLSession()
queue = SimpleQueue()
queue.put_nowait(self.args['url'])
visited.add(self.args['url'])
count = 0
while not queue.empty():
count += 1
if count > self.args['max_page_count']:
break
url = queue.get_nowait()
logging.info('Request on {}'.format(url))
r = session.get(url, cookies=self.cookies)
links = r.html.absolute_links
for link in links:
def is_exclude():
for exc in self.args['exclude']:
if link.startswith(exc):
return True
return False
if link.startswith(self.args['url'])\
and not is_exclude()\
and link not in visited:
visited.add(link)
queue.put_nowait(link)
forms = r.html.find('form')
for form in forms:
request = {
'uuid':
randuuid(),
'location':
url,
'url':
urljoin(r.url, form.attrs['action'])
if 'action' in form.attrs else url,
'method':
form.attrs['method'].upper()
if 'method' in form.attrs else 'GET',
'content-type':
form.attrs['enctype'] if 'enctype' in form.attrs else
'application/x-www-form-urlencoded',
'fields': {},
'form':
form
}
inputs = form.find('input')
for inp in inputs:
if 'name' not in inp.attrs:
continue
typ = inp.attrs['type'] if 'type' in inp.attrs else 'text'
name = inp.attrs['name']
value = inp.attrs['value'] if 'value' in inp.attrs else ''
required = True if 'required' in inp.attrs else False
if typ == 'radio':
if name in request['fields']:
request['fields'][name]['values'].append(value)
else:
request['fields'][name] = {
'type': 'radio',
'name': name,
'required': required,
'values': [value]
}
elif typ == 'checkbox':
checked = True if 'checked' in inp.attrs and (
inp.attrs['checked'] == 'checked'
or inp.attrs['checked'] == '') else False
request['fields'][name] = {
'type': 'checkbox',
'name': name,
'required': required,
'value': value,
'checked': checked
}
else:
request['fields'][name] = {
'type': typ,
'name': name,
'required': required,
'default': value
}
textareas = form.find('textarea')
for textarea in textareas:
if 'name' not in textarea.attrs:
continue
name = textarea.attrs['name']
request['fields'][name] = {
'type': 'textarea',
'name': name,
'required':
True if 'required' in textarea.attrs else False,
'default': textarea.text
}
selects = form.find('select')
for select in selects:
if 'name' not in select.attrs:
continue
name = select.attrs['name']
multiple = True if 'multiple' in select.attrs else False
required = True if 'required' in select.attrs else False
values = []
options = select.find('option')
for option in options:
if 'value' not in option.attrs:
continue
values.append(option.attrs['value'])
request['fields'][name] = {
'type': 'select',
'name': name,
'multiple': multiple,
'required': required,
'values': values
}
logging.info('Found form {}'.format(request['uuid']))
self.requests[request['uuid']] = request
self.results['requests'] = self.requests
self.results['urls'] = list(visited)
logging.info('Found {} forms'.format(len(self.results['requests'])))
class IntcodeRunner(object):
# Halt running at input/output
def __init__(self, program):
self.ip = 0
self.program = program
self.input_buffer = SimpleQueue()
self.output_buffer = SimpleQueue()
self.halted = False
def add_input(self, input):
self.input_buffer.put(input)
def run_program(self):
if self.halted:
return
while True:
inst = [int(x) for x in str(self.program[self.ip])]
if len(inst) == 1:
opcode = inst[0]
mode = []
else:
opcode = int(''.join([str(x) for x in inst[-2:]]))
mode = inst[:-2]
mode.reverse()
if opcode == 1:
self._add(mode)
elif opcode == 2:
self._mul(mode)
elif opcode == 3:
if not self._input():
break
elif opcode == 4:
self._output(mode)
break
elif opcode == 5:
self._jmp_true(mode)
elif opcode == 6:
self._jmp_false(mode)
elif opcode == 7:
self._lt(mode)
elif opcode == 8:
self._eq(mode)
elif opcode == 99:
self.halted = True
print('Halting!')
break
else:
print(f'Unknown opcode.. {opcode}')
sys.exit(1)
def adjust_mode(self, mode_array, nr_param):
if len(mode_array) == nr_param:
return mode_array
missing = [0 for _ in range(nr_param - len(mode_array))]
return mode_array + missing
def _add(self, mode):
mode = self.adjust_mode(mode, 3)
param_1 = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
param_2 = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
self.program[self.program[self.ip + 3]] = param_1 + param_2
self.ip += 4
def _mul(self, mode):
mode = self.adjust_mode(mode, 3)
param_1 = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
param_2 = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
self.program[self.program[self.ip + 3]] = param_1 * param_2
self.ip += 4
def _input(self):
# Never value mode as we write
# Throws a queue.empty() if empty. We use this to input values
if self.input_buffer.empty():
return False
self.program[self.program[self.ip +
1]] = self.input_buffer.get_nowait()
self.ip += 2
return True
def _output(self, mode):
mode = self.adjust_mode(mode, 1)
output = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
self.output_buffer.put(output)
self.ip += 2
def _jmp_true(self, mode):
# Jump if true
mode = self.adjust_mode(mode, 2)
jmp = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
if jmp > 0:
self.ip = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
else:
self.ip += 3
def _jmp_false(self, mode):
# Jump if false
mode = self.adjust_mode(mode, 2)
jmp = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
if jmp == 0:
self.ip = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
else:
self.ip += 3
def _lt(self, mode):
# less then
mode = self.adjust_mode(mode, 2)
param_1 = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
param_2 = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
if param_1 < param_2:
self.program[self.program[self.ip + 3]] = 1
else:
self.program[self.program[self.ip + 3]] = 0
self.ip += 4
def _eq(self, mode):
# equals
mode = self.adjust_mode(mode, 2)
param_1 = (self.program[self.program[self.ip + 1]]
if mode[0] == 0 else self.program[self.ip + 1])
param_2 = (self.program[self.program[self.ip + 2]]
if mode[1] == 0 else self.program[self.ip + 2])
if param_1 == param_2:
self.program[self.program[self.ip + 3]] = 1
else:
self.program[self.program[self.ip + 3]] = 0
self.ip += 4
async def chat_watcher(tips_queue, broadcaster):
log = logging.getLogger('chat_watcher')
try:
api_info = None
with urllib.request.urlopen(
f'https://chaturbate.com/api/chatvideocontext/{broadcaster}/'
) as url:
api_info = json.loads(url.read().decode())
# CB uses SockJS defaults of randomness
ws_uri = f"{api_info['wschat_host'].replace('https://', 'wss://')}/{random.randint(100, 999)}/{''.join(random.choices(string.ascii_letters + string.digits, k=8))}/websocket"
async with websockets.connect(ws_uri) as websocket:
# opening handshake
resp = await websocket.recv()
# print(f'<< {resp}') # 'o'
json_encoder = json.JSONEncoder()
obj2 = json_encoder.encode({
'method': 'connect',
'data': {
'user': api_info['chat_username'],
'password': api_info['chat_password'],
'room': api_info['broadcaster_username'],
'room_password': api_info['room_pass']
}
})
obj3 = json_encoder.encode([obj2])
# print(f'>> {obj3}')
await websocket.send(obj3)
resp = await websocket.recv()
assert 'onAuthResponse' in resp
# print(f'<< {resp}') # 'a["{\"args\":[\"1\"],\"callback\":null,\"method\":\"onAuthResponse\"}"]'
obj2 = json_encoder.encode({
'method': 'joinRoom',
'data': {
'room': broadcaster,
'exploringHashTag': ''
}
})
obj3 = json_encoder.encode([obj2])
# print(f'>> {obj3}')
await websocket.send(obj3)
log.info('connected to chat room')
ws_connect_time = time.time()
random_levels = None
prev_resps = SimpleQueue(
) # allow follow-up message clarifying random levels to be sent with the tip
while True:
resp = None
try:
try:
resp = prev_resps.get_nowait()
except Empty:
resp = await asyncio.wait_for(websocket.recv(), 1)
except asyncio.TimeoutError:
pass
# save all websockets messages for debugging
# with open('ws.log', 'a') as f:
# f.write(f'{datetime.now().isoformat()} {resp}\n')
if resp != None and re.search(
r'tip_alert', resp, re.IGNORECASE
) and time.time(
) - ws_connect_time > 1: # ignore initial burst of old tips
# tip notification: a["{\"args\":[\"{\\\"in_fanclub\\\": false, \\\"to_username\\\": \\\"{broadcaster}\\\", \\\"has_tokens\\\": true, \\\"message\\\": \\\"\\\", \\\"tipped_recently\\\": true, \\\"is_anonymous_tip\\\": false, \\\"dont_send_to\\\": \\\"\\\", \\\"from_username\\\": \\\"{username}\\\", \\\"send_to\\\": \\\"\\\", \\\"tipped_alot_recently\\\": true, \\\"amount\\\": 1, \\\"tipped_tons_recently\\\": true, \\\"is_mod\\\": false, \\\"type\\\": \\\"tip_alert\\\", \\\"history\\\": true}\",\"true\"],\"callback\":null,\"method\":\"onNotify\"}"]
# random level chosen a["{\"args\":[\"{broadcaster}\",\"{\\\"c\\\": \\\"rgb(120,0,175)\\\", \\\"X-Successful\\\": true, \\\"in_fanclub\\\": false, \\\"f\\\": \\\"Arial, Helvetica\\\", \\\"i\\\": \\\"HWBBR7LPLE7F7V\\\", \\\"gender\\\": \\\"f\\\", \\\"has_tokens\\\": true, \\\"m\\\": \\\"--------\\\\\\\"{username} has RANDOMLY activated level DOMI in 3 by tipping 44 tokens\\\", \\\"tipped_alot_recently\\\": false, \\\"user\\\": \\\"{broadcaster}\\\", \\\"is_mod\\\": false, \\\"tipped_tons_recently\\\": false, \\\"tipped_recently\\\": false}\"],\"callback\":null,\"method\":\"onRoomMsg\"}"]
msg = json.loads(
json.loads(json.loads(resp[1:])[0])['args'][0])
if msg['type'] == 'tip_alert':
# print(f'<<j {msg}')
amt = msg['amount']
tip: Tip = Tip(int(amt), time.time())
# one of the next few messages might have the randomly chosen level if this tip was for random level
if random_levels != None and tip.val == random_levels[
'value']:
# limit to 5 messages or 1 second
log.debug('searching for random level')
while prev_resps.qsize() < 10 and time.time(
) < tip.timestamp + 1:
try:
resp = await asyncio.wait_for(
websocket.recv(), 1)
matches = re.search(
r'[Ll]evel[^\d]+(\d+)', resp)
if matches:
random_tip_level = int(
matches.group(1))
tip.val = random_levels['selection'][
random_tip_level - 1]
log.debug(
f'random tip level found:{random_tip_level} tip.val:{tip.val}'
)
else:
if 'room subject changed to' not in resp and '"Notice: ' not in resp: # ignore easy 'spam'
prev_resps.put(resp)
except asyncio.TimeoutError:
pass
# send the tip
tips_queue.put(tip)
log.debug('sent ' + str(tip.val) + ' from ' +
msg['from_username'] + ' tip queue len ' +
str(tips_queue.len_write()))
await asyncio.sleep(0.1)
try:
ex = tips_queue.get_nowait()
if type(ex) == Exception:
raise ex
else:
if ex[0] == 'broadcaster':
broadcaster = ex[1]
log.info('new broadcaster: ' + broadcaster)
break
elif ex[0] == 'random_levels':
random_levels = ex[1]
if random_levels != None:
random_levels['selection'] = sorted(
random_levels['selection'])
except Empty:
pass
except Exception as ex:
print('watcher error')
print(traceback.format_exc())
finally:
tips_queue.put(Exception('watcher error'))
return
class Producer:
def __init__(self, *topics, broker, schema_registry, schema, logging_enabled = False):
"""
Initialization of the Producer which instatiates an AvroProducer class
Parameters
----------
broker: str
The URL of the broker (example: 'localhost:9092')
schema_registry: str
The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081')
schema: str
The default AVRO schema to use to serialize messages
logger: Logger object, optional
The logger object which will be used to log messages if provided
topics
variable length argument list of the string names of topics to produce too
"""
self.schema = avro.loads(schema)
self.__producer = AvroProducer(
{
"bootstrap.servers": broker,
"schema.registry.url": schema_registry
},
default_key_schema=self.schema
)
if logging_enabled:
self.logger = logging.getLogger(__name__)
else:
self.logger = None
self.topics = topics
self.produce_flag = True
self.production_last_stoped = 0
self.total_time_producing_stoped = 0
self.__msg_queue = SimpleQueue()
def produce(self, msg, schema = None, callback = None):
"""
Write a message to confluent kafka using
the instatiated AvroProducer to serialize
Parameters
----------
msg: str
The message to be serialized and sent
schema: str, Optional
An optional schema to overide the default
set in the constructor
callback: Function object, Optional
An optional callback which will be executed
whether the producing of the message fails or
succeeds. This function must take two parameters
the first for the error and the second for the message
(https://docs.confluent.io/current/clients/confluent-kafka-python/#producer)
"""
# TODO: function partials for better readability ?
# SOLVED: created dictionary to expand to parameters
params = {}
params["value"] = msg
if schema is not None:
params["value_schema"] = schema
else:
params["value_schema"] = self.schema
if callback is not None:
params["on_delivery"] = callback
for topic in self.topics:
params["topic"] = topic
self.__msg_queue.put_nowait(params)
try:
while not self.__msg_queue.empty():
msg = self.__msg_queue.get_nowait()
self.__producer.produce(**msg)
self.__producer.flush()
self.produce_flag = True
self.production_last_stoped = 0
except SerializerError as e:
self.__log_msg(
"ERROR",
"Message deserialization has failed {}: {} \n".format(msg,e),
"See the following trace back \n {}".format(traceback.format_exc())
)
except BufferError as e:
if self.produce_flag or (
self.production_last_stoped != 0 and ((time.time() - self.production_last_stoped) >= 3600)
):
self.produce_flag = False
self.production_last_stoped = time.time()
self.total_time_producing_stoped += time.time() - self.production_last_stoped
self.__log_msg(
"Queue Buffer has reached its maximum capacity, unable to deliver message {}: {}".format(msg,e),
"Message production will be shut down until messages can be resent",
f"Total time message producing has stopped {self.total_time_producing_stoped}",
level="CRITICAL",
delimeter="\n"
)
except KafkaException as e:
self.__log_msg(
"An unknown exception has occured specific to Kafka {}: {}".format(msg, e),
level="ERROR",
delimeter=""
)
except Exception as e:
self.__log_msg(
"An unknown exception has occured {}: {}".format(msg, e),
f"See the following traceback {traceback.format_exc()}",
level="ERROR",
delimeter="\n"
)
def __enter__(self):
"""
Context Manager for Producer, to allow custom actions for producing messages
"""
return self.__producer
def __exit__(self, *args):
"""
On exit producer is flushed
"""
self.__producer.flush()
def __log_msg(self, *messages, level="NOTSET", delimeter= " "):
levels = {
"CRITICAL": logging.CRITICAL,
"ERROR": logging.ERROR,
"WARNING": logging.WARNING,
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"NOTSET": logging.NOTSET
}
msg = delimeter.join(messages)
if self.logger is not None:
if level not in levels:
raise ValueError(
f"level {level} is not valid must be one of {list(levels.keys())}"
)
self.logger.log(
levels[level],
msg
)
else:
if level is not None:
print(f"LOGGED MESSAGE: {msg}")
else:
print(f"{level}: {msg}")
async def new_worker(queue: SimpleQueue):
driver = make_new_driver()
while True:
if queue.empty():
driver.quit()
break
next_download = queue.get_nowait()
url, filename = next_download
print(filename)
driver.get(url)
sleep(3)
if "model" in driver.current_url:
continue
# press 3D software options
find_element(
driver,
make_options_xpath("3d_softwares") + "button",
By.XPATH,
).click()
sleep(0.25)
software_options = find_element(
driver,
make_options_xpath("3d_softwares") + "div/ul",
By.XPATH,
).find_elements_by_tag_name("li")
# press RendererOptions
find_element(
driver,
make_options_xpath("renders") + "button",
By.XPATH,
).click()
sleep(0.25)
renderer_options = find_element(
driver,
make_options_xpath("renders") + "div/ul",
By.XPATH,
).find_elements_by_tag_name("li")
set_multi_option()
# press ExtraTextureSizes
texture_options = find_element(
driver,
'//*[@id="3d_softwares"]/div[2]/div/div/ul',
By.XPATH,
).find_elements_by_tag_name("li")
find_element(
driver,
'//*[@id="3d_softwares"]/div[2]/div/div/ul/li[2]/a/span[1]',
By.XPATH,
).click()
sleep(0.25)
find_element(driver, '//*[@id="texture_sizes"]/div[2]/div/button',
By.XPATH).click()
sleep(0.25)
res_options = find_element(
driver, '//*[@id="texture_sizes"]/div[2]/div/div/ul',
By.XPATH).find_elements_by_tag_name("li")
res_options[-1].click()
set_multi_option(res_options, config["texture-sizes"])
sleep(0.25)
elif "texture" in driver.current_url:
# if texture
dropdown = find_element(
driver,
'//*[@id="acquared-asset"]/div[5]/div/div[2]/div[2]/div/button',
By.XPATH,
)
dropdown.click()
sleep(0.25)
res_options = find_element(
driver,
'//*[@id="acquared-asset"]/div[5]/div/div[2]/div[2]/div/div/ul',
By.XPATH,
).find_elements_by_tag_name("li")
set_multi_option(res_options, config["texture-sizes"])
dropdown.click()
sleep(0.5)
elif "hdr" in driver.current_url:
# if texture
dropdown = find_element(
driver,
'//*[@id="acquared-asset"]/div[5]/div/div[2]/div[2]/div/button',
By.XPATH,
)
dropdown.click()
sleep(0.5)
res_options = find_element(
driver,
'//*[@id="acquared-asset"]/div[5]/div/div[2]/div[2]/div/div/ul',
By.XPATH,
).find_elements_by_tag_name("li")
set_multi_option(res_options, config["hdr-sizes"])
dropdown.click()
sleep(0.5)
find_download_btn(driver).click()
t1 = perf_counter()
sleep(3)
while not (
os.path.exists(f"{DOWNLOAD_PATH}\{filename}.zip")
or os.path.exists(f"{DOWNLOAD_PATH}\{filename}.crdownload") or
os.path.exists(f"{DOWNLOAD_PATH}\{filename}.zip.crdownload")):
elapsed = perf_counter() - t1
print(
filename,
colored(
f" waiting on{filename} | {int(elapsed)}/{DOWNLOAD_INIT_TIMEOUT}",
"yellow",
),
)
if elapsed > DOWNLOAD_INIT_TIMEOUT:
print(filename,
colored(f" Failed or timed out {filename}", "red"))
break
sleep(1)
print(filename, colored(f"started downloading {filename}", "cyan"))
driver.minimize_window()
while not os.path.exists(f"{DOWNLOAD_PATH}\{filename}.zip"):
print(filename, colored(f"downloading {filename}", "blue"))
await asyncio.sleep(5)
print(filename, colored(f" finished downloading{filename}", "green"))
driver.maximize_window()
class Consumer:
def __init__(self,
broker,
schema_registry,
topic,
logging_enabled=False,
groupId="asgardConsumerGroup",
autocommit=True):
"""
Initialiser for Confluent Consumer using AvroConsumer.
Each consumer can only be subscribed to one topic
Parameters
----------
broker: str
The URL of the broker (example: 'localhost:9092')
schema_registry: str
The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081')
topic: str
The topic to subscribe too
logger: Logger object, Optional
The logger object which will be used to log messages if provided
groupId: str, Optional
An optional groupId which can be used to loadbalance consumers default is "asgard"
"""
"""self.__consumer = AvroConsumer(
{
"bootstrap.servers": broker,
"group.id": groupId,
"schema.registry.url": schema_registry,
"enable.auto.commit": autocommit
}
)"""
self.__consumer = KafkaConsumer({
"bootstrap.servers": broker,
"group.id": groupId,
"enable.auto.commit": autocommit,
"auto.offset.reset": "latest"
})
self.autocommit = autocommit
if not autocommit:
self.consumed_messages = SimpleQueue()
self.__consumer.subscribe([topic])
if logging_enabled:
self.logger = logging.getLogger(__name__)
else:
self.logger = None
def consume(self):
"""
Method to consume and return message if exists and can be deserialized
Returns
-------
str
The recieved message payload as a string
None
No message has been recieved or an error has occured
"""
msg = None
try:
msg = self.__consumer.poll(1)
except SerializerError as e:
self.__log_msg("Message deserialization has failed {}: {}".format(
msg, e),
"See the following stack trace",
f"{traceback.format_exc()}",
delimeter="\n",
level="ERROR")
except RuntimeError as e:
self.__log_msg(
"The consumer has been closed and cannot recieve messages",
level="ERROR")
except Exception as e:
self.__log_msg("An unkown error has occured {}".format(e),
"See the following stack trace",
f"{traceback.format_exc()}",
delimeter="\n",
level="ERROR")
if not msg is None:
if msg.error():
self.__log_msg("AvroConsumer error: {}".format(msg.error()),
level="ERROR")
else:
if not self.autocommit:
self.consumed_messages.put_nowait(msg)
return json.loads(msg.value().decode()).get("payload")
def __enter__(self):
return self.__consumer
def __exit__(self, *args):
self.close()
def __log_msg(
self,
*messages,
level="NOTSET",
delimeter=" ",
):
levels = {
"CRITICAL": logging.CRITICAL,
"ERROR": logging.ERROR,
"WARNING": logging.WARNING,
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"NOTSET": logging.NOTSET
}
msg = delimeter.join(messages)
if self.logger is not None:
if level not in levels:
raise ValueError(
f"level {level} is not valid must be one of {list(levels.keys())}"
)
self.logger.log(levels[level], msg)
else:
if level is not None:
print(f"LOGGED MESSAGE: {msg}")
else:
print(f"{level}: {msg}")
def commit(self, asynchronous=True):
if not self.autocommit and not self.consumed_messages.empty():
msg = self.consumed_messages.get_nowait()
self.__consumer.commit(msg, asynchronous=asynchronous)
def close(self):
"""
Close the consumer, Once called this object cannot be reused
"""
self.__consumer.close()
class HttpSyncedDictionary:
"""
Contains a dictionary that can be updated and queried locally.
The updates are sent to a HTTP server, as reply, the current dictionary
known to the HTTP server is expected. This is used to update the
local dictionary.
In effect, the dictionary can be synchronized across multiple instances
all using the same server.
The synchronization happens in a separate thread and is limited by the
time needed for HTTP POST send/receive.
"""
def __init__(self, server_url, keys_to_filter=[]):
"""
:param keys_to_filter: Iterable of keys in the synchronized dictionary.
In order to not overwrite the values which are produced locally and
thus more accurate locally than on the server, provide the keys to
those values here. Values from the remote server for those keys will
be ignored.
"""
self.data = {}
self.inbox = SimpleQueue()
self.server_url = server_url
self.keys_to_filter = keys_to_filter
self.thread = Thread(target=self._thread_function)
self.daemon = True
self.is_thread_running = False
def _thread_function(self):
while self.is_thread_running:
new_data = {}
while not self.inbox.empty(): # only use latest queue element
new_data = self.inbox.get_nowait()
response = requests.post(self.server_url, json=new_data)
if response.ok:
remote_status = response.json()
for key in self.keys_to_filter:
remote_status.pop(key, None)
self.data.update(remote_status)
def start(self):
if not self.is_thread_running:
self.is_thread_running = True
self.thread.start()
def stop(self):
self.is_thread_running = False
self.thread.join()
def update(self, dictionary):
self.data.update(dictionary)
self.inbox.put(dictionary)
def get(self, key=None, default_value=None):
if key is not None:
return self.data.get(key, default_value)
else:
return self.data
class IntcodeRunner(object):
def __init__(self, program, break_on_output=False):
self.ip = 0
self.relative_base = 0
self.program = program + [0 for _ in range(1000)]
self.input_buffer = SimpleQueue()
self.output_buffer = SimpleQueue()
self.halted = False
self.break_on_output = break_on_output
def add_input(self, input):
self.input_buffer.put(input)
def run_program(self):
if self.halted:
return
while True:
inst = [int(x) for x in str(self.program[self.ip])]
if len(inst) == 1:
opcode = inst[0]
mode = []
else:
opcode = int(''.join([str(x) for x in inst[-2:]]))
mode = inst[:-2]
mode.reverse()
if opcode == 1:
self._add(mode)
elif opcode == 2:
self._mul(mode)
elif opcode == 3:
if not self._input(mode):
break
elif opcode == 4:
self._output(mode)
if self.break_on_output:
break
elif opcode == 5:
self._jmp_true(mode)
elif opcode == 6:
self._jmp_false(mode)
elif opcode == 7:
self._lt(mode)
elif opcode == 8:
self._eq(mode)
elif opcode == 9:
self._adjust_rb(mode)
elif opcode == 99:
self.halted = True
print('Halting!')
break
else:
print(f'Unknown opcode.. {opcode}')
sys.exit(1)
def adjust_mode(self, mode_array, nr_param):
if len(mode_array) == nr_param:
return mode_array
missing = [0 for _ in range(nr_param - len(mode_array))]
return mode_array + missing
def get_mode_value(self, mode, ip_offset):
if mode == 0:
pass
return self.program[self.program[self.ip + ip_offset]]
elif mode == 1:
return self.program[self.ip + ip_offset]
elif mode == 2:
return self.program[self.program[self.ip + ip_offset] + self.relative_base]
else:
print(f'Got illigal mode! mode: {mode}')
sys.exit(1)
def expand_memory(self, nr):
self.program = self.program + [0 for _ in range(nr*2)]
def write_mode_value(self, mode, ip_offset, value):
if mode == 0:
if self.program[self.ip + ip_offset] >= len(self.program):
self.expand_memory(self.program[self.ip + ip_offset])
self.program[self.program[self.ip + ip_offset]] = value
elif mode == 1:
print('Wrong write mode!')
sys.exit(1)
elif mode == 2:
if (self.program[self.relative_base + self.program[self.ip + ip_offset]] > len(self.program)):
self.expand_memory(self.program[self.ip + ip_offset])
self.program[self.relative_base +
self.program[self.ip + ip_offset]] = value
else:
print('Unknown write mode!')
sys.exit(1)
def _add(self, mode):
mode = self.adjust_mode(mode, 3)
param_1 = self.get_mode_value(mode[0], 1)
param_2 = self.get_mode_value(mode[1], 2)
self.write_mode_value(mode[2], 3, param_1 + param_2)
self.ip += 4
def _mul(self, mode):
mode = self.adjust_mode(mode, 3)
param_1 = self.get_mode_value(mode[0], 1)
param_2 = self.get_mode_value(mode[1], 2)
self.write_mode_value(mode[2], 3, param_1 * param_2)
self.ip += 4
def _input(self, mode):
# Throws a queue.empty() if empty. We use this to input values
if self.input_buffer.empty():
return False
self.write_mode_value(mode[0], 1, self.input_buffer.get_nowait())
self.ip += 2
return True
def _output(self, mode):
mode = self.adjust_mode(mode, 1)
output = self.get_mode_value(mode[0], 1)
self.output_buffer.put(output)
self.ip += 2
def _jmp_true(self, mode):
# Jump if true
mode = self.adjust_mode(mode, 2)
jmp = self.get_mode_value(mode[0], 1)
if jmp > 0:
self.ip = self.get_mode_value(mode[1], 2)
else:
self.ip += 3
def _jmp_false(self, mode):
# Jump if false
mode = self.adjust_mode(mode, 2)
jmp = self.get_mode_value(mode[0], 1)
if jmp == 0:
self.ip = self.get_mode_value(mode[1], 2)
else:
self.ip += 3
def _lt(self, mode):
# less then
mode = self.adjust_mode(mode, 3)
param_1 = self.get_mode_value(mode[0], 1)
param_2 = self.get_mode_value(mode[1], 2)
if param_1 < param_2:
self.write_mode_value(mode[2], 3, 1)
else:
self.write_mode_value(mode[2], 3, 0)
self.ip += 4
def _eq(self, mode):
# equals
mode = self.adjust_mode(mode, 3)
param_1 = self.get_mode_value(mode[0], 1)
param_2 = self.get_mode_value(mode[1], 2)
if param_1 == param_2:
self.write_mode_value(mode[2], 3, 1)
else:
self.write_mode_value(mode[2], 3, 0)
self.ip += 4
def _adjust_rb(self, mode):
"""Update the relative base."""
mode = self.adjust_mode(mode, 1)
self.relative_base += self.get_mode_value(mode[0], 1)
self.ip += 2
class Connection:
"""Describes a connection either from the server to some client or from the client
to the server
Attributes:
connection (socket.socket): how we communicate with the other entity
address (str): where the entity connected from / where we connected to
send_queue (queue[bytes]): the packets that we need to send
rec_queue (queue[Packet]): the packets that they have sent us
curr_send_packet (optional BytesIO): if we are currently trying to send a message
to the client, this is the serialized message we are trying to send (that has
already been removed from the send_queue)
curr_rec (deque[bytes]): the things that we have in memory received
"""
def __init__(self, connection: socket.socket, address: str) -> None:
self.connection = connection
self.address = address
self.send_queue = Queue()
self.rec_queue = Queue()
self.curr_send_packet: io.BytesIO = None
self.curr_rec = deque()
def disconnected(self):
"""Returns True if the connection is dead for whatever reason, False otherwise"""
return self.connection is None
def update(self):
"""Handles sending and receiving packets in a non-blocking way. Must be called very
regularly for send() and receive() to actually do anything
"""
if self.disconnected():
return
try:
self._handle_send()
self._handle_rec()
except BlockingIOError:
pass
except OSError:
self.connection = None
print(f'[networking.shared] connection lost')
traceback.print_exc()
def _handle_send(self):
if self.curr_send_packet is None:
if self.send_queue.empty():
return
packet_serd = self.send_queue.get_nowait()
self.curr_send_packet = io.BytesIO()
self.curr_send_packet.write(
len(packet_serd).to_bytes(4, 'big', signed=False))
self.curr_send_packet.write(packet_serd)
self.curr_send_packet.seek(0, 0)
for _ in range(128): # avoid sending more than 512kb in one go
block = self.curr_send_packet.read(BLOCK_SIZE)
if not block:
self.curr_send_packet = None
return
amt_sent = self.connection.send(block)
if amt_sent < len(block):
self.curr_send_packet.seek(amt_sent - len(block), 1)
return
def _try_from_recq(self, amt: int) -> typing.Optional[bytes]:
"""Tries to read the specified number of bytes from the receive queue.
If this fails to get that many bytes the receive queue is effectively
unaltered, otherwise the bytes are removed from the receive queue
and returned"""
if not self.curr_rec:
return None
if len(self.curr_rec) == 1 or len(self.curr_rec[0]) >= amt:
# happy / most common case
if len(self.curr_rec[0]) < amt:
return None
block = self.curr_rec.popleft()
if len(block) == amt:
return block
self.curr_rec.appendleft(block[amt:])
return block[:amt]
result = io.BytesIO()
curlen = 0
while self.curr_rec:
block = self.curr_rec.popleft()
if curlen + len(block) == amt:
# another happy / common case
result.write(block)
return result.getvalue()
if curlen + len(block) < amt:
result.write(block)
continue
result.write(block[:amt])
self.curr_rec.appendleft(block[amt:])
return result.getvalue()
# didn't get enough data, but now the curr_rec queue is all merged
# so we will get the top happy case
self.curr_rec.appendleft(result.getvalue())
return None
def _handle_rec(self):
for _ in range(128): # avoid reading more than 512kb in one go
block = self.connection.recv(BLOCK_SIZE)
if not block:
self.connection.close()
self.connection = None
break
self.curr_rec.append(block)
if len(block) < BLOCK_SIZE:
break
for _ in range(8): # avoid parsing too many packets at once
lenblock = self._try_from_recq(4)
if not lenblock:
return
explen = int.from_bytes(lenblock, 'big', signed=False)
block = self._try_from_recq(explen)
if not block:
self.curr_rec.appendleft(lenblock)
return
packet = ser.deserialize(block)
if not isinstance(packet, packets.Packet):
raise ValueError(
f'got non-packet {packet} (type={type(packet)})')
self.rec_queue.put(packet)
def send(self, packet: packets.Packet):
"""Sends this client the specified packet"""
if self.disconnected():
return
self.send_queue.put_nowait(ser.serialize(packet))
def send_serd(self, packet_serd: bytes):
"""Sends this client the serialized packet"""
if self.disconnected():
return
self.send_queue.put_nowait(packet_serd)
def read(self) -> typing.Optional[packets.Packet]:
"""Returns the packet from the client if there is one"""
return self.rec_queue.get_nowait(
) if not self.rec_queue.empty() else None
def has_pending(self, read=True, write=True) -> bool:
"""Returns True if there are pending sends / receives, False otherwise"""
if write and not self.send_queue.empty():
# have things to send still
return True
if read and not self.rec_queue.empty():
# have things that we've parsed but haven't been read() yet
return True
if write and self.curr_send_packet is not None:
# in the middle of sending something
return True
if read and self.curr_rec:
# have things not yet parsed / incomplete
return True
return False
class NetworkConnection(object):
class State(object):
kCreated = 0
kInit = 1
kHandshake = 2
kSynchronized = 3
kActive = 4
kDead = 5
def __init__(self,
uid,
stream,
notifier,
handshake,
get_entry_type,
verbose=False):
# logging debugging
self.m_verbose = verbose
self.m_uid = uid
self.m_stream = stream
self.m_notifier = notifier
self.m_handshake = handshake
self.m_get_entry_type = get_entry_type
self.m_active = False
self.m_proto_rev = 0x0300
self.state = self.State.kCreated
self.m_state_mutex = threading.Lock()
self.m_last_update = 0
self.m_outgoing = Queue()
self.m_process_incoming = None
self.m_read_thread = None
self.m_write_thread = None
self.m_remote_id_mutex = threading.Lock()
self.m_remote_id = None
self.m_last_post = 0
self.m_pending_mutex = threading.Lock()
self.m_pending_outgoing = []
self.m_pending_update = {}
# Condition variables for shutdown
self.m_shutdown_mutex = threading.Lock()
# Not needed in python
# self.m_read_shutdown_cv = threading.Condition()
# self.m_write_shutdown_cv = threading.Condition()
self.m_read_shutdown = False
self.m_write_shutdown = False
# turn off Nagle algorithm; we bundle packets for transmission
try:
self.m_stream.setNoDelay()
except IOError as e:
logger.warning("Setting TCP_NODELAY: %s", e)
def start(self):
if self.m_active:
return
self.m_active = True
self.set_state(self.State.kInit)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
# reset shutdown flags
with self.m_shutdown_mutex:
self.m_read_shutdown = False
self.m_write_shutdown = False
# start threads
self.m_write_thread = SafeThread(target=self._writeThreadMain,
name="nt-net-write")
self.m_read_thread = SafeThread(target=self._readThreadMain,
name="nt-net-read")
def __repr__(self):
try:
return "<NetworkConnection 0x%x %s>" % (id(self), self.info())
except Exception:
return "<NetworkConnection 0x%x ???>" % id(self)
def stop(self):
logger.debug("NetworkConnection stopping (%s)", self)
if not self.m_active:
return
self.set_state(self.State.kDead)
self.m_active = False
# closing the stream so the read thread terminates
self.m_stream.close()
# send an empty outgoing message set so the write thread terminates
self.m_outgoing.put([])
# wait for threads to terminate, timeout
self.m_write_thread.join(1)
if self.m_write_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
self.m_read_thread.join(1)
if self.m_read_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
def get_proto_rev(self):
return self.m_proto_rev
def get_stream(self):
return self.m_stream
def info(self):
return ConnectionInfo(
self.remote_id(),
self.m_stream.getPeerIP(),
self.m_stream.getPeerPort(),
self.m_last_update,
self.m_proto_rev,
)
def is_connected(self):
return self.state == self.State.kActive
def last_update(self):
return self.m_last_update
def set_process_incoming(self, func):
self.m_process_incoming = func
def set_proto_rev(self, proto_rev):
self.m_proto_rev = proto_rev
def set_state(self, state):
with self.m_state_mutex:
State = self.State
# Don't update state any more once we've died
if self.state == State.kDead:
return
# One-shot notify state changes
if self.state != State.kActive and state == State.kActive:
info = self.info()
self.m_notifier.notifyConnection(True, info)
logger.info(
"CONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
elif self.state != State.kDead and state == State.kDead:
info = self.info()
self.m_notifier.notifyConnection(False, info)
logger.info(
"DISCONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
if self.m_verbose:
logger.debug("%s: %s -> %s", self, _state_map[self.state],
_state_map[state])
self.state = state
# python optimization: don't use getter here
# def state(self):
# return self.m_state
def remote_id(self):
with self.m_remote_id_mutex:
return self.m_remote_id
def set_remote_id(self, remote_id):
with self.m_remote_id_mutex:
self.m_remote_id = remote_id
def uid(self):
return self.m_uid
def _sendMessages(self, msgs):
self.m_outgoing.put(msgs)
def _readThreadMain(self):
decoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
def _getMessage():
decoder.set_proto_rev(self.m_proto_rev)
try:
return Message.read(self.m_stream, decoder,
self.m_get_entry_type)
except IOError as e:
logger.warning("read error in handshake: %s", e)
# terminate connection on bad message
self.m_stream.close()
return None
self.set_state(self.State.kHandshake)
try:
handshake_success = self.m_handshake(self, _getMessage,
self._sendMessages)
except Exception:
logger.exception("Unhandled exception during handshake")
handshake_success = False
if not handshake_success:
self.set_state(self.State.kDead)
self.m_active = False
else:
self.set_state(self.State.kActive)
try:
while self.m_active:
if not self.m_stream:
break
decoder.set_proto_rev(self.m_proto_rev)
try:
msg = Message.read(self.m_stream, decoder,
self.m_get_entry_type)
except Exception as e:
if not isinstance(e, StreamEOF):
if verbose:
logger.exception("read error")
else:
logger.warning("read error: %s", e)
# terminate connection on bad message
self.m_stream.close()
break
if verbose:
logger.debug(
"%s received type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
self.m_last_update = monotonic()
self.m_process_incoming(msg, self)
except IOError as e:
# connection died probably
logger.debug("IOError in read thread: %s", e)
except Exception:
logger.warning("Unhandled exception in read thread",
exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
# also kill write thread
self.m_outgoing.put([])
with self.m_shutdown_mutex:
self.m_read_shutdown = True
def _writeThreadMain(self):
encoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
out = []
try:
while self.m_active:
msgs = self.m_outgoing.get()
if verbose:
logger.debug("write thread woke up")
if msgs:
logger.debug("%s sending %s messages",
self.m_stream.sock_type, len(msgs))
if not msgs:
continue
encoder.set_proto_rev(self.m_proto_rev)
# python-optimization: checking verbose causes extra overhead
if verbose:
for msg in msgs:
if msg:
logger.debug(
"%s sending type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
msg.write(out, encoder)
else:
for msg in msgs:
if msg:
msg.write(out, encoder)
if not self.m_stream:
break
if not out:
continue
self.m_stream.send(b"".join(out))
del out[:]
# if verbose:
# logger.debug('send %s bytes', encoder.size())
except IOError as e:
# connection died probably
if not isinstance(e, StreamEOF):
logger.debug("IOError in write thread: %s", e)
except Exception:
logger.warning("Unhandled exception in write thread",
exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
self.m_stream.close() # also kill read thread
with self.m_shutdown_mutex:
self.m_write_shutdown = True
def queueOutgoing(self, msg):
with self.m_pending_mutex:
# Merge with previous. One case we don't combine: delete/assign loop.
msgtype = msg.type
if msgtype in [kEntryAssign, kEntryUpdate]:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.first != 0:
# overwrite the previous one for this id
oldidx = mpend.first - 1
oldmsg = self.m_pending_outgoing[oldidx]
if (oldmsg and oldmsg.type == kEntryAssign
and msgtype == kEntryUpdate):
# need to update assignment with seq_num and value
oldmsg = Message.entryAssign(oldmsg.str, msg_id,
msg.seq_num_uid,
msg.value, oldmsg.flags)
else:
oldmsg = msg # easy update
self.m_pending_outgoing[oldidx] = oldmsg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(pos + 1, 0)
elif msgtype == kEntryDelete:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
# clear previous updates
mpend = self.m_pending_update.get(msg_id)
if mpend is not None:
if mpend.first != 0:
self.m_pending_outgoing[mpend.first - 1] = None
if mpend.second != 0:
self.m_pending_outgoing[mpend.second - 1] = None
self.m_pending_update[msg_id] = _empty_pair
# add deletion
self.m_pending_outgoing.append(msg)
elif msgtype == kFlagsUpdate:
# don't do this for unassigned id's
msg_id = msg.id
if id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.second != 0:
# overwrite the previous one for this id
self.m_pending_outgoing[mpend.second - 1] = msg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(0, pos + 1)
elif msgtype == kClearEntries:
# knock out all previous assigns/updates!
for i, m in enumerate(self.m_pending_outgoing):
if not m:
continue
t = m.type
if t in [
kEntryAssign,
kEntryUpdate,
kFlagsUpdate,
kEntryDelete,
kClearEntries,
]:
self.m_pending_outgoing[i] = None
self.m_pending_update.clear()
self.m_pending_outgoing.append(msg)
else:
self.m_pending_outgoing.append(msg)
def postOutgoing(self, keep_alive):
with self.m_pending_mutex:
# optimization: don't call monotonic unless needed
# now = monotonic()
if not self.m_pending_outgoing:
if not keep_alive:
return
# send keep-alives once a second (if no other messages have been sent)
now = monotonic()
if (now - self.m_last_post) < 1.0:
return
self.m_outgoing.put((Message.keepAlive(), ))
else:
now = monotonic()
self.m_outgoing.put(self.m_pending_outgoing)
self.m_pending_outgoing = []
self.m_pending_update.clear()
self.m_last_post = now
class ServerEventLoop(EventLoop):
def __init__(self, listen_socket: RDTSocket):
super().__init__(listen_socket)
self.connections: dict = {}
self.accept_queue = SimpleQueue()
self.__is_close = False
self.setName('ServerEventLoop')
def run(self) -> None:
self.send_loop.start()
self.recv_loop.start()
super(ServerEventLoop, self).run()
def accept(self) -> (RDTSocket, (str, int)):
assert not self.__is_close, 'Can not accept after close'
if not self.accept_queue.empty():
try:
return self.accept_queue.get_nowait()
except Empty as ev:
print('\033[0;31m642: Empty-> ', ev, '\033[0m')
def on_syn(self, pkt: RDTPacket):
remote = pkt.remote
if remote in self.connections:
simple_sct = self.connections[remote]
simple_sct.SEQ_ACK = max(simple_sct.SEQ_ACK, pkt.SEQ + pkt.LEN)
syn_ack_pkt = RDTPacket(SYN=1, ACK=1, remote=remote, SEQ=simple_sct.SEQ, SEQ_ACK=simple_sct.SEQ_ACK)
self.send_loop.put(syn_ack_pkt)
return
elif self.__is_close:
self.send_loop.put(RDTPacket(remote=pkt.remote, SEQ=0, SEQ_ACK=0, RST=1))
return
assert remote not in self.connections, 'Has SYN'
simple_sct = self.socket.create_simple_socket(remote, pkt.SEQ, pkt.SEQ_ACK)
simple_sct.SEQ_ACK += pkt.LEN
simple_sct.status = RDTConnectionStatus.SYN_
self.connections[remote] = simple_sct
syn_ack_pkt = RDTPacket(SYN=1, ACK=1, remote=remote, SEQ=simple_sct.SEQ, SEQ_ACK=simple_sct.SEQ_ACK,
PAYLOAD=bytes(1024))
simple_sct.SEQ += 1024
self.send_loop.put(syn_ack_pkt)
timer = self.push_timer(SYN_ACK_WAIT,
RDTEvent(RDTEventType.ACK_TIMEOUT, syn_ack_pkt))
simple_sct.wait_ack.append(timer)
if self.socket.debug:
print('\033[0;32m668: SYN<- ', remote, '\033[0m')
def on_syn_ack(self, pkt: RDTPacket):
assert False, 'SYN_ACK ???'
def on_ack(self, pkt: RDTPacket):
simple_sct = self.get_simple_sct(pkt)
if simple_sct.status == RDTConnectionStatus.SYN_:
self.accept_queue.put(simple_sct)
simple_sct.status = RDTConnectionStatus.ACK_
self.deal_ack(simple_sct=simple_sct, pkt=pkt)
def on_fin(self, pkt: RDTPacket):
if pkt.remote not in self.connections:
self.send_loop.put(RDTPacket(remote=pkt.remote, FIN=1, ACK=1, SEQ=0, SEQ_ACK=0))
return
simple_sct: SimpleRDT = self.get_simple_sct(pkt)
simple_sct.SEQ_ACK = pkt.SEQ + pkt.LEN
if simple_sct.status.value < RDTConnectionStatus.FIN.value:
simple_sct.status = RDTConnectionStatus.FIN_
if simple_sct.debug:
print('\033[0;33m690: FIN<- ', pkt.remote, '\033[0m')
self.await_send_fin(simple_sct)
elif simple_sct.status == RDTConnectionStatus.FIN:
if simple_sct.debug:
print('\033[0;33m695: FIN success', pkt.remote, '\033[0m')
self.cancel_timer(simple_sct.destroy_timer)
self.send_fin_ack_pkt(simple_sct)
self.put(RDTEventType.DESTROY_SIMPLE, simple_sct)
def on_fin_ack(self, pkt: RDTPacket):
simple_sct: SimpleRDT = self.get_simple_sct(pkt)
self.cancel_timer(simple_sct.destroy_timer)
if simple_sct.status.value < RDTConnectionStatus.FIN_ACK_.value:
simple_sct.status = RDTConnectionStatus.FIN_ACK_
else:
return # FIN ACK过了
self.put(RDTEventType.DESTROY_SIMPLE, simple_sct)
def on_send(self, r: ((str, int), bytes)):
remote, bs = r
simple_sct: SimpleRDT = self.connections[remote]
assert simple_sct.status == RDTConnectionStatus.ACK_, 'Send with a wrong state'
self.deal_send(simple_sct, bs)
def on_send_ack(self, simple_sct: SimpleRDT):
if simple_sct.last_ACK == simple_sct.SEQ_ACK and simple_sct.status == RDTConnectionStatus.ACK_:
return # ACK过了
self.send_ack_pkt(simple_sct)
def on_send_fin(self, skt: SimpleRDT):
if len(skt.wait_ack) > 0 or len(skt.wait_send) > 0:
self.await_send_fin(skt)
return
self.deal_send_fin(skt)
def on_connect(self, remote: (str, int)):
assert False, 'connect ???'
def on_rst(self, pkt: RDTPacket):
assert False, 'RST ???'
def on_ack_timeout(self, pkt: RDTPacket):
simple_sct: SimpleRDT = self.get_simple_sct(pkt)
self.deal_ack_timeout(simple_sct, pkt)
def on_sak(self, pkt: RDTPacket):
self.deal_sak(self.get_simple_sct(pkt), pkt)
def get_simple_sct(self, pkt: RDTPacket):
try:
assert pkt.remote in self.connections, 'No such connection'
except AssertionError:
self.send_loop.put(RDTPacket(remote=pkt.remote, SEQ=0, SEQ_ACK=0, RST=1))
return self.connections[pkt.remote]
def on_simple_close(self, remote: (str, int)):
assert remote in self.connections, 'No such connection'
simple_sct: SimpleRDT = self.connections[remote]
if simple_sct.status.value >= RDTConnectionStatus.FIN.value:
self.put(RDTEventType.DESTROY_SIMPLE, simple_sct)
else:
self.put(RDTEventType.SEND_FIN, simple_sct)
def on_listen_close(self):
assert not self.__is_close, 'Has closed'
self.__is_close = True
while not self.accept_queue.empty():
_, remote = self.accept_queue.get()
del self.connections[remote]
self.put(RDTEventType.DESTROY_ALL, None)
def on_destroy_simple(self, skt: SimpleRDT):
assert skt.remote in self.connections, 'No such connection'
with skt.lock:
skt.remote_close = True
for t in self.connections[skt.remote].wait_ack:
self.cancel_timer(t)
del self.connections[skt.remote]
def on_destroy_all(self):
if len(self.connections) == 0:
self.put(RDTEventType.VANISH, None)
if self.socket.debug:
print('\033[0;31m774:完全销毁 DESTROY_ALL -> VANISH')
else:
self.await_destroy_all()
state = SimpleQueue()
state.put_nowait(((choice(room_state)), choice(room_state)))
while True:
iter.clear()
cleanwriter.clear()
iter.write("Iteração/Repetição : " + str(count),
align="center",
font=("Arial", 16, "normal"))
cleanwriter.write("Limpezas : " + str(cleaned),
align="center",
font=("Arial", 16, "normal"))
sleep(1.5)
condition = state.get_nowait()
stateA = condition[0]
stateB = condition[1]
X.clear()
Y.clear()
nextA = choice(room_state)
nextB = choice(room_state)
state.put_nowait((nextA, nextB))
filler(A, Room_state[stateA])
filler(B, Room_state[stateB])
X.write("Agora : " + stateA + "\nPróximo : " + nextA,
class Crawler(Worker):
def __init__(self, scheduler):
super().__init__()
connect_to_database()
self.__scheduler = scheduler
self.__repos = SimpleQueue()
self.__next_crawl = datetime.datetime.now()
def post_repo(self, repo_id):
self.__repos.put(repo_id)
async def work(self):
try:
await self.__process_repos()
except Exception as e:
logger.error("Processing repos failed: %s", e)
async def __process_repos(self):
logger.info("Start crawling")
loop = asyncio.get_running_loop()
if not os.path.exists(data_dir):
os.makedirs(data_dir, exist_ok=True)
logger.info("Created directory '%s'", data_dir)
new_commits = False
with database.session_scope() as session:
if datetime.datetime.now() >= self.__next_crawl:
logger.info("Crawl all repos")
repos = session.query(database.Repo).all()
self.__next_crawl = datetime.datetime.now(
) + datetime.timedelta(seconds=CRAWLER_PERIOD_SECONDS)
self.reschedule_internally(CRAWLER_PERIOD_SECONDS)
else:
logger.info("Crawl manually triggered repos")
repo_ids = [repo for repo in self.__get_repos()]
repos = session.query(database.Repo).filter(
database.Repo.id.in_(repo_ids)).all()
channels = session.query(database.Channel).all()
for repo in repos:
try:
work_dir = os.path.join(data_dir, str(repo.id))
controller = RepoController(work_dir)
if not controller.is_clone_of(repo.url):
logger.info("Create repo for URL '%s' in '%s'",
repo.url, work_dir)
await loop.run_in_executor(None,
controller.create_new_repo,
repo.url)
logger.info("Setup SSH in '%s'", work_dir)
await loop.run_in_executor(None, controller.setup_ssh,
repo.ecosystem.ssh_key,
repo.ecosystem.known_hosts)
logger.info("Setup HTTP credentials in '%s'", work_dir)
credentials = [{
"url": c.url,
"username": c.username,
"password": c.password
} for c in repo.ecosystem.credentials]
await loop.run_in_executor(None, controller.setup_http,
credentials)
logger.info("Fetch repo '%s' for URL '%s'", work_dir,
repo.url)
await loop.run_in_executor(None, controller.fetch)
branches = controller.get_remote_branches()
for channel in channels:
for branch in branches:
if not re.fullmatch(channel.branch, branch):
continue
logger.info("Branch '%s' matches '%s'", branch,
channel.branch)
logger.info("Checkout branch '%s'", branch)
controller.checkout(branch)
sha = controller.get_sha()
commits = session.query(database.Commit).filter_by(
repo=repo, sha=sha, channel=channel)
# continue if this commit has already been stored
if list(commits):
logger.info("Commit '%s' exists", sha[:7])
continue
logger.info("Add commit '%s'", sha[:7])
commit = database.Commit()
commit.sha = sha
commit.message = controller.get_message()
commit.user_name = controller.get_user_name()
commit.user_email = controller.get_user_email()
commit.repo = repo
commit.channel = channel
commit.status = database.CommitStatus.new
session.add(commit)
new_commits = True
old_commits = session.query(
database.Commit).filter(
database.Commit.repo == repo,
database.Commit.channel == channel,
database.Commit.sha != sha,
database.Commit.status !=
database.CommitStatus.old)
for c in old_commits:
logger.info("Set status of '%s' to 'old'",
c.sha[:7])
c.status = database.CommitStatus.old
except git.exc.GitError as e:
logger.error(
"Failed to process repo '%s' with message '%s'",
repo.url, e)
if new_commits:
logger.info("Finish crawling with *new* commits")
logger.info('Trigger scheduler: process commits')
try:
self.__scheduler.process_commits()
except (ApiException, MaxRetryError):
logger.error("Failed to trigger scheduler")
else:
logger.info("Finish crawling with *no* new commits")
def __get_repos(self):
try:
while True:
yield self.__repos.get_nowait()
except Empty:
pass
