def runexternal_out_and_err(cmd, check_memleak=True):
# pylint: disable=unused-argument
command = shlex.split(cmd)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.stdout is not None:
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(p.stdout, q_stdout))
t_stdout.start()
else:
q_stdout = None
ret_stdout = ''
if p.stderr is not None:
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(p.stderr, q_stderr))
t_stderr.start()
else:
q_stderr = None
ret_stderr = ''
if q_stdout is not None:
ret_stdout = q_stdout.get().decode('ascii')
if q_stderr is not None:
ret_stderr = q_stderr.get().decode('ascii')
waitcode = p.wait()
if waitcode != 0:
ret_stderr = ret_stderr + '\nERROR ret code = %d' % waitcode
return (ret_stdout, ret_stderr)
def runexternal_out_and_err(cmd, check_memleak = True):
command = shlex.split(cmd)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.stdout is not None:
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(p.stdout, q_stdout))
t_stdout.start()
else:
q_stdout = None
ret_stdout = ''
if p.stderr is not None:
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(p.stderr, q_stderr))
t_stderr.start()
else:
q_stderr = None
ret_stderr = ''
if q_stdout is not None:
ret_stdout = q_stdout.get().decode('ascii')
if q_stderr is not None:
ret_stderr = q_stderr.get().decode('ascii')
p.wait()
return (ret_stdout, ret_stderr)
def test_handle_failing_upload_xlog(self):
sleeps = []
def sleep(sleep_amount):
sleeps.append(sleep_amount)
time.sleep(0.001)
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.sleep = sleep
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
with pytest.raises(Empty):
callback_queue.get(timeout=0.1)
alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20]
assert sleeps[:8] == expected_sleeps
class GdbBreakpoint(Breakpoint):
def __init__(self, system, bkpt_num):
super().__init__()
self._system = system
self._bkpt_num = bkpt_num
self._queue = Queue()
system.register_event_listener(self._event_receiver)
def wait(self, timeout = None):
if self._handler:
raise Exception("Breakpoint cannot have a handler and be waited on")
if timeout == 0:
return self._queue.get(False)
else:
return self._queue.get(True, timeout)
def delete(self):
self._system.unregister_event_listener(self._event_receiver)
self._system.get_target()._gdb_interface.delete_breakpoint(self._bkpt_num)
def _event_receiver(self, evt):
if EVENT_BREAKPOINT in evt["tags"] and \
evt["source"] == "target" and \
evt["properties"]["bkpt_number"] == self._bkpt_num:
if self._handler:
self._handler(self._system, self)
else:
self._queue.put(evt)
elif EVENT_SIGABRT in evt["tags"]:
self._queue.put(evt)
def main(filename):
username = input('User: '******': script started')
task_queue = Queue()
new_task = Task(task_queue)
new_task.read(filename)
while not task_queue.empty():
host_queue = Queue()
log_queue = Queue()
task_block = task_queue.get()
for host_ip in task_block['hosts']:
host_queue.put(host_ip)
config_list = task_block['tasks']
while not host_queue.empty():
for x in range(50):
worker = ExecuteTask(host_queue, config_list, log_queue, username, password, mode=task_block['mode'])
worker.daemon = True
worker.start()
host_queue.join()
while not log_queue.empty():
st = log_queue.get()
print(st)
print(time_stamp(), ': script stopped')
class TTS(object):
def __init__(self):
self.clients = []
self.voice_choices = []
self.queue = Queue()
if 'win32com' not in globals():
return
Thread(target=self._background).start()
def _background(self):
pythoncom.CoInitialize()
self.tts = win32com.client.Dispatch("SAPI.SpVoice")
self.voices = self.tts.GetVoices()
self.voices = [self.voices.Item(i) for i in range(self.voices.Count)]
self.voice_choices = [dict(desc=v.GetDescription(), id=i) for i, v in enumerate(self.voices)]
self.tts.Rate = -5
self.event_sink = win32com.client.WithEvents(self.tts, TTSEventSink)
self.event_sink.setTTS(self)
while True:
self._speak(self.queue.get(True))
def _speak(self, text):
self._speaking = True
self.tts.Skip("Sentence", INT32_MAX)
self.tts.Speak(text, SVSFlagsAsync)
self._pump()
def speak(self, text):
while True:
try:
self.queue.get(False)
except Empty:
break
self.queue.put(text)
def get_voice_choices(self):
return self.voice_choices
def set_voice(self, voice_id):
self.tts.Voice = self.voices[voice_id]
def handle_event(self, event, *args):
msg = dict(type=event)
if event == 'end':
self._speaking = False
elif event == 'word':
msg.update(dict(char_pos=args[0], length=args[1]))
msg = json.dumps(msg)
for c in self.clients:
c.write_message(msg)
def _pump(self):
skipped = False
while self._speaking:
if not skipped and not self.queue.empty():
self.tts.Skip("Sentence", INT32_MAX)
skipped = True
pythoncom.PumpWaitingMessages()
time.sleep(0.05)
def test_wings_push_switches(self):
"""Testing push switches"""
# init wings
settings = {"pins": {"wings": {"left_switch": 17, "right_switch": 4, "position": 25, "movement": 22}}}
event_queue = Queue()
logging.basicConfig()
logger = logging.getLogger(name="TuxEatPi")
logger.setLevel(logging.DEBUG)
wings = FakeWings(settings, event_queue, logger)
# Test calibrate
self.assertEqual(wings.get_position(), "down")
# test left switch event
wings.push_wing('left')
event = event_queue.get(timeout=5)
self.assertEqual(event.component, 'FakeWings')
self.assertEqual(event.pin_id, wings.pins.get('left_switch'))
self.assertEqual(event.name, 'left_switch')
# test left switch event
wings.push_wing('right')
event = event_queue.get(timeout=5)
self.assertEqual(event.component, 'FakeWings')
self.assertEqual(event.pin_id, wings.pins.get('right_switch'))
self.assertEqual(event.name, 'right_switch')
def __exec_jmptable(self, args):
if self.gctx.dis is None:
error("load a file before")
return
try:
inst_addr = int(args[1], 16)
table_addr = int(args[2], 16)
nb_entries = int(args[3])
entry_size = int(args[4])
except:
error("one parameter is invalid, be sure that addresses start with 0x")
return
if entry_size not in [2, 4, 8]:
error("error the entry size should be in [2, 4, 8]")
return
self.gctx.db.modified = True
self.gctx.dis.add_jmptable(inst_addr, table_addr, entry_size, nb_entries)
queue_wait_analyzer = Queue()
# Re-run the analyzer
func_id = self.gctx.dis.mem.get_func_id(inst_addr)
if func_id == -1:
self.analyzer.msg.put((inst_addr, False, True, queue_wait_analyzer))
else:
ad = self.gctx.dis.func_id[func_id]
self.analyzer.msg.put((ad, True, True, queue_wait_analyzer))
queue_wait_analyzer.get()
class TestCommunicationService(unittest.TestCase):
def setUp(self):
logging.basicConfig(format='%(asctime)s\t%(levelname)-10s\t%(threadName)-32s\t%(name)-32s\t%(message)s', level=logging.DEBUG, datefmt='%Y-%m-%d %H:%M:%S')
self.master = Communication(host='localhost', port=4545, service_name='Master', server=True)
self.worker = Communication(host='localhost', port=4545, service_name='Worker', server=False)
self.masterq = Queue()
self.workerq = Queue()
self.master.register('Q', self.masterq)
self.worker.register('Q', self.workerq)
self.master.startup()
self.worker.startup()
time.sleep(2)
def test_send_string(self):
self.master.send('Worker', 'Q', 'Hello World')
message = self.workerq.get()
self.assertEqual(message, 'Hello World')
self.worker.send('Master', 'Q', 'Hello World')
message = self.masterq.get()
self.assertEqual(message, 'Hello World')
def test_send_numbers(self):
self.master.send('Worker', 'Q', 1234)
message = self.workerq.get()
self.assertEqual(message, 1234)
self.worker.send('Master', 'Q', 12.34)
message = self.masterq.get()
self.assertEqual(message, 12.34)
def tearDown(self):
self.worker.shutdown()
self.master.shutdown()
class FakeConsumer(BrightsideConsumer):
"""The fake consumer is a test double for a consumer wrapping messaging middleware.
To use it, just add BrighsideMessage(s) to the queue and the call receive to pop
then off the stack. Purge, will clean the queue
"""
def __init__(self):
self._queue = Queue()
self._acknowledged_message = None
def acknowledge(self, message):
self._acknowledged_message = message
def has_acknowledged(self, message):
return (self._acknowledged_message is not None) and (self._acknowledged_message.id == message.id)
@property
def queue(self):
return self._queue
def purge(self):
while not self._queue.empty():
self._queue.get(block=False)
assert self._queue.empty()
def receive(self, timeout: int):
return self._queue.get(block=True,timeout=timeout)
def test_multiple_dispatchers(self):
def get_dispatcher_name(q):
q.put(current_thread().name)
sleep(1.2)
d1 = Dispatcher(MockBot('disp1'), Queue())
d2 = Dispatcher(MockBot('disp2'), Queue())
q1 = Queue()
q2 = Queue()
d1._init_async_threads('test_1', workers=1)
d2._init_async_threads('test_2', workers=1)
try:
d1.run_async(get_dispatcher_name, q1)
d2.run_async(get_dispatcher_name, q2)
name1 = q1.get()
name2 = q2.get()
self.assertNotEqual(name1, name2)
finally:
d1.stop()
d2.stop()
# following three lines are for pypy unitests
d1._reset_singleton()
del d1
del d2
def generer_graph(self):
"""Génère le graph des sorties.
Le graph est un dictionnaire comprenant en clé un tuple
(origine, destination) et en valeur la liste des sorties
nécessaires pour s'y rendre.
L'algorithme Dijkstra est utilisé.
"""
graph = {}
aretes = {}
sorties = {}
# On remplit le chemin avec toutes les liaisons
for salle in self.salles.values():
origine = salle.mnemonic
aretes[origine] = []
for sortie in salle.sorties:
destination = sortie.salle_dest.mnemonic
aretes[origine].append(destination)
sorties[origine, destination] = sortie.nom
# Population des chemins dans le graph
for origine in range(1, len(self.salles) + 1):
origine = str(origine)
for destination in range(1, len(self.salles) + 1):
destination = str(destination)
if origine == destination:
continue
frontier = Queue()
frontier.put(origine)
origines = {origine: None}
while not frontier.empty():
actuel = frontier.get()
if actuel == destination:
break
for fils in aretes[actuel]:
if fils not in origines:
frontier.put(fils)
origines[fils] = actuel
# Recherche la liste des sorties
parent = Queue()
parent.put(destination)
chemin = []
while not parent.empty():
actuel = parent.get()
precedent = origines[actuel]
sortie = sorties[precedent, actuel]
chemin.insert(0, sortie)
if precedent != origine:
parent.put(precedent)
graph[origine, destination] = chemin
self.graph = graph
def test_15(self):
for _ in self.singleThreadOnly:
rl = RunLatest(_)
# Start a job, keeping it running.
q1a = Queue()
q1b = Queue()
def f1():
q1b.put(None)
q1a.get()
em1 = Emitter()
future1 = rl.start(em1.g, f1)
q1b.get()
self.assertEqual(future1.state, Future.STATE_RUNNING)
# Start two more. The first should not run; if it does, it raises
# an exception.
def f2():
raise TypeError
rl.start(None, f2)
em3 = Emitter()
rl.start(em3.g, lambda: None)
with WaitForSignal(em3.bing, 1000):
q1a.put(None)
rl.terminate()
def connector_socket(queuein, pipeout, worker_start):
try:
queueout = Queue()
worker_start(queuein, queueout)
while True:
try:
events = queueout.get()
if events is None:
break
pipeout[0].put(events)
while True:
try:
events = queueout.get(False)
pipeout[0].put(events)
except Empty:
break
pipeout[1].send(b"\x00")
except EOFError:
break
except OSError as exc:
if exc.args[0] == errno.EINTR:
continue
else:
break
except:
pass
finally:
pipeout[1].close()
class TextStack(unittest.TestCase):
def setUp(self):
self.queue = Queue()
self.queue.put(4)
self.queue.put(2)
self.queue.put(1)
self.queue.put(6)
self.queueTwo = Queue()
self.queueTwo.put(1)
self.queueTwo.put(2)
def test_put(self):
self.queue.put(9)
self.assertEquals(self.queue.__str__(), [4, 2, 1, 6, 9, None])
with self.assertRaises(IndexError):
self.queue.put(1)
self.queue.put(2)
def test_pop(self):
self.queueTwo.get()
self.assertEquals(self.queueTwo.__str__(), [None, 2, None, None, None , None])
with self.assertRaises(IndexError):
self.queueTwo.get()
self.queueTwo.get()
class S2EBreakpoint(Breakpoint):
def __init__(self, bkpt_num):
from avatar.system import System
super().__init__()
self._bkpt_num = bkpt_num
self._queue = Queue()
System.getInstance().register_event_listener(self._event_receiver)
def wait(self, timeout = None):
if self._handler:
raise Exception("Breakpoint cannot have a handler and be waited on")
if timeout == 0:
return self._queue.get(False)
else:
return self._queue.get(True, timeout)
def delete(self):
System.getInstance().unregister_event_listener(self._event_receiver)
System.getInstance().get_emulator()._gdb_interface.delete_breakpoint(self._bkpt_num)
def _event_receiver(self, evt):
if Event.EVENT_BREAKPOINT in evt["tags"] and \
evt["source"] == "emulator" and \
evt["properties"]["bkpt_number"] == self._bkpt_num:
if self._handler:
self._handler(System.getInstance(), self)
else:
self._queue.put(evt)
class QueueManager_Test(TestCase):
def setUp(self):
self.qm = QueueManager()
self.q = Queue()
self.qq = Queue()
self.qm.register(self.q, {"foo": "bar", "baz": "quux"})
self.qm.register(self.qq, {"foo": "quux"})
def tearDown(self):
self.qm.deregister(self.q)
self.qm.deregister(self.qq)
def test_fanout(self):
message = object()
self.qm.fanout(message)
assert self.q.get(False) is message
assert self.qq.get(False) is message
def test_rules(self):
message = object()
self.qm.fanout(message, {"foo": "bar"})
assert self.q.get(False) is message
with self.assertRaises(Empty):
self.qq.get(False)
def test_already_registered(self):
with self.assertRaises(AlreadyRegistered):
self.qm.register(self.q)
def test_not_found(self):
with self.assertRaises(NotFound):
self.qm.deregister(Queue())
class CommandQueue():
"""
Holds the raw commands in a queue for later processing.
"""
commandQueue = None
_commandProcessor = None
def __init__(self, commandProcessor=CommandProcessor()):
self.commandQueue = Queue()
self._commandProcessor = commandProcessor;
def addCommand(self, client, commandString):
"""
Adds the next command to the command queue for latter processing. We add
the client and commandString instead of wrapping them because
the commandString has not been evaluated in any way.
"""
self.commandQueue.put((client, commandString))
def nextCommand(self):
client, rawCommand = self.commandQueue.get(True)
while client.isLinkdead() :
client, rawCommand = self.commandQueue.get(True)
return self._commandProcessor.process(rawCommand, client)
def test_decompression_event(self):
ifile = TestXlog(self.incoming_path, "00000001000000000000000A", "random")
callback_queue = Queue()
local_filepath = os.path.join(self.temp_dir, "00000001000000000000000A")
self.compression_queue.put(
{
"blob": self.compress(ifile.contents),
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": local_filepath,
"metadata": {
"compression-algorithm": self.algorithm,
"compression-level": 0,
"original-file-size": ifile.size,
"pg-version": 90500,
},
"site": self.test_site,
"type": "DECOMPRESSION",
}
)
callback_queue.get(timeout=1.0)
assert os.path.exists(local_filepath) is True
with open(local_filepath, "rb") as fp:
fdata = fp.read()
assert fdata[:100] == ifile.contents[:100]
assert fdata == ifile.contents
class GridMap():
"""
Map class
"""
def __init__(self, width, height, size):
self.width = width
self.height = height
self.cellSize = size
self.__map = np.zeros((self.width,self.height))
self.__drawQueue = Queue()
self.linear = Linear()
def update(self, x, y, theta):
self.__map[x][y] = 1
self.__drawQueue.put((x,y))
#theta = round(theta, 2)
#self.linear.update(x, y, theta)
def getPoints(self):
points = []
while not self.__drawQueue.empty():
points.append(self.__drawQueue.get())
return points
def drawPoints(self, drawFunc):
"""
draw the points by callback function drawFunc
"""
if self.__drawQueue.empty():
return
drawFunc(self.__drawQueue.get())
class CorrectionBasedBearTest(LocalBearTestHelper):
"""
This test only covers corner cases. The basic functionality is tested in
a more intuitive way in the IndentBearTest.
"""
def setUp(self):
self.section = Section('')
self.queue = Queue()
self.uut = IndentBear(self.section, self.queue)
def test_errors(self):
old_binary, self.uut.BINARY = self.uut.BINARY, "invalid_stuff_here"
self.uut.execute(filename='', file=[])
self.queue.get()
self.assertRegex(str(self.queue.get()), r'\[WARNING\] .*')
self.uut.BINARY = old_binary
def test_missing_binary(self):
old_binary = IndentBear.BINARY
IndentBear.BINARY = "fdgskjfdgjdfgnlfdslk"
self.assertEqual(IndentBear.check_prerequisites(),
"'fdgskjfdgjdfgnlfdslk' is not installed.")
# "echo" is existent on nearly all platforms.
IndentBear.BINARY = "echo"
self.assertTrue(IndentBear.check_prerequisites())
del IndentBear.BINARY
self.assertTrue(IndentBear.check_prerequisites())
IndentBear.BINARY = old_binary
def test6_ThreeThreadsTwoConnections(self):
pool = PooledPg(2, 2, 2, True)
from queue import Queue, Empty
queue = Queue(3)
def connection():
try:
queue.put(pool.connection(), 1, 1)
except TypeError:
queue.put(pool.connection(), 1)
from threading import Thread
for i in range(3):
Thread(target=connection).start()
try:
db1 = queue.get(1, 1)
db2 = queue.get(1, 1)
except TypeError:
db1 = queue.get(1)
db2 = queue.get(1)
db1_con = db1._con
db2_con = db2._con
self.assertNotEqual(db1, db2)
self.assertNotEqual(db1_con, db2_con)
try:
self.assertRaises(Empty, queue.get, 1, 0.1)
except TypeError:
self.assertRaises(Empty, queue.get, 0)
del db1
try:
db1 = queue.get(1, 1)
except TypeError:
db1 = queue.get(1)
self.assertNotEqual(db1, db2)
self.assertNotEqual(db1._con, db2._con)
self.assertEqual(db1._con, db1_con)
def test_decompression_decrypt_event(self):
_, blob = self.compressor.compress_filepath_to_memory(
self.random_file_path,
compression_algorithm=self.config["compression"]["algorithm"],
rsa_public_key=CONSTANT_TEST_RSA_PUBLIC_KEY)
callback_queue = Queue()
local_filepath = os.path.join(self.temp_dir, "00000001000000000000000E")
self.compression_queue.put({
"blob": blob,
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": local_filepath,
"metadata": {
"compression-algorithm": self.algorithm,
"encryption-key-id": "testkey",
"original-file-size": self.random_file_size,
"pg-version": 90500,
},
"site": self.test_site,
"type": "DECOMPRESSION",
})
callback_queue.get(timeout=1.0)
assert os.path.exists(local_filepath) is True
with open(local_filepath, "rb") as fp:
assert fp.read() == self.random_file_contents
def monitor(self, core, mem):
startTime = time.time()
avgMH = Queue()
while ((time.time() - startTime) < self.monitorTime):
time.sleep(5)
devInfo = self.api.getGPUInfo(self.device)
if devInfo['Temperature'] >= self.maxTemp:
self.handleBadClocks('Temperature threshold reached.', devInfo)
return True
if devInfo['HWE'] > self.HWE:
self.HWE = devInfo['HWE']
self.handleBadClocks('Hardware errors found.', devInfo)
#Make sure we give the GPU time to set the new clocks so we get the final HW error count
time.sleep(2)
devInfo = self.api.getGPUInfo(self.device)
self.HWE = devInfo['HWE']
return True
avgMH.put(devInfo['MH'])
if (avgMH.qsize() >= 3):
avgMH.get()
#MH added should be averaged
totalMH = 0
numMH = 0
while (not avgMH.empty()):
totalMH += avgMH.get()
numMH += 1
avg = totalMH/numMH
newrec = {'device': self.device, 'core': core, 'mem': mem, 'success': True, 'MH': avg, 'temp': devInfo['Temperature']}
self.results.append(newrec)
self.logger.addRecord(newrec)
return False
def test5_threadsafety_1(self):
dbpool = self.my_dbpool(1, 2)
from queue import Queue, Empty
queue = Queue(3)
def connection():
queue.put(dbpool.connection())
from threading import Thread
thread1 = Thread(target=connection).start()
thread2 = Thread(target=connection).start()
thread3 = Thread(target=connection).start()
try:
db1 = queue.get(1, 1)
db2 = queue.get(1, 1)
except TypeError:
db1 = queue.get(1)
db2 = queue.get(1)
self.assertNotEqual(db1, db2)
self.assertNotEqual(db1._con, db2._con)
try:
self.assertRaises(Empty, queue.get, 1, 0.1)
except TypeError:
self.assertRaises(Empty, queue.get, 0)
db2.close()
try:
db3 = queue.get(1, 1)
except TypeError:
db3 = queue.get(1)
self.assertNotEqual(db1, db3)
self.assertNotEqual(db1._con, db3._con)
class HttpProxyHandler:
def __init__(self, reqid, router, proxyAddr, inq, outq):
self.reqid=reqid
self.router=router
self.proxyAddr=proxyAddr
self.inq=inq
self.outq=outq
self.closed=False
self.respq=Queue()
proxyback=self.router.getService('httpProxyback')
proxyback.setQueue(self.reqid, self.respq)
def start(self):
t=Thread(target=self.processIn)
t.setDaemon(True)
t.start()
t2=Thread(target=self.processOut)
t2.setDaemon(True)
t2.start()
def processIn(self):
data=self.inq.get()
packet=ProxyMessage()
packet.createProxyMessage(self.reqid, data)
router.sendto(packet.msg, proxyAddr, service='httpProxy')
def processOut(self):
data=self.respq.get()
while data!=None:
self.outq.put(data)
data=self.respq.get()
self.outq.put(None)
def test_lock_fails_if_db_already_locked(self):
db_path = self.tmp / 'kv.sqlite'
q1 = Queue()
q2 = Queue()
kv2 = kv.KV(db_path, timeout=0.1)
def locker():
kv1 = kv.KV(db_path)
with kv1.lock():
q1.put(None)
q2.get()
th = Thread(target=locker)
th.start()
try:
q1.get()
with self.assertRaises(sqlite3.OperationalError) as cm1:
with kv2.lock():
pass
self.assertEqual(str(cm1.exception), 'database is locked')
with self.assertRaises(sqlite3.OperationalError) as cm2:
kv2['a'] = 'b'
self.assertEqual(str(cm2.exception), 'database is locked')
finally:
q2.put(None)
th.join()
def check_if_pingable(ip_list):
"""
Check what IP addresses from the list are reachable
:param ip_list: list of IP addresses to ping
:return: list of reachable and unreachable hosts
"""
queue_ip = Queue()
queue_reachable = Queue()
queue_unreachable = Queue()
reachable = []
unreachable = []
for ip in ip_list:
queue_ip.put(ip)
while not queue_ip.empty():
for x in range(50):
worker = Ping(queue_ip, queue_reachable, queue_unreachable)
worker.daemon = True
worker.start()
queue_ip.join()
while not queue_reachable.empty():
reachable.append(queue_reachable.get())
while not queue_unreachable.empty():
unreachable.append(queue_unreachable.get())
return reachable, unreachable
class AvatarBreakpoint(Breakpoint):
def __init__(self, system, address):
super().__init__()
self._system = system
self._address = address
self._queue = Queue()
system.register_event_listener(self._event_receiver)
def wait(self, timeout = None):
if self._handler:
raise Exception("Breakpoint cannot have a handler and be waited on")
if timeout == 0:
return self._queue.get(False)
else:
return self._queue.get(True, timeout)
def delete(self):
self._system.unregister_event_listener(self._event_receiver)
self._system.get_target().clear_breakpoint(self._address)
def _event_receiver(self, evt):
if EVENT_BREAKPOINT in evt["tags"] and \
evt["source"] == "target" and \
evt["properties"]["address"] == self._address:
if self._handler:
self._handler(self._system, self)
else:
self._queue.put(evt)
class CharmQueue:
""" charm queue for handling relations in the background
"""
def __init__(self):
self.charm_relations_q = Queue()
self.charm_setup_q = Queue()
self.charm_post_proc_q = Queue()
self.is_running = False
def add_relation(self, charm):
self.charm_relations_q.put(charm)
def add_setup(self, charm):
self.charm_setup_q.put(charm)
def add_post_proc(self, charm):
self.charm_post_proc_q.put(charm)
@utils.async
def watch_setup(self):
log.debug("Starting charm setup watcher.")
while True:
try:
charm = self.charm_setup_q.get()
err = charm.setup()
if err:
self.charm_setup_q.put(charm)
self.charm_setup_q.task_done()
except:
log.exception("ignoring exception in setup watcher.")
time.sleep(1)
@utils.async
def watch_relations(self):
log.debug("Starting charm relations watcher.")
while True:
try:
charm = self.charm_relations_q.get()
err = charm.set_relations()
if err:
self.charm_relations_q.put(charm)
self.charm_relations_q.task_done()
except:
log.exception("ignoring exception in relations watcher.")
time.sleep(1)
@utils.async
def watch_post_proc(self):
log.debug("Starting charm post processing watcher.")
while True:
try:
charm = self.charm_post_proc_q.get()
err = charm.post_proc()
if err:
self.charm_post_proc_q.put(charm)
self.charm_post_proc_q.task_done()
except:
log.exception("ignoring exception in post-processing watcher.")
time.sleep(10)
from queue import Queue
start = datetime.datetime.now()
# 创建队列
q = Queue(5)
# 判空
# print(q.empty())
# 判满
# print(q.full())
# 获取队列长度
# print(q.qsize())
# 存数据
q.put('kobe')
q.put('qwe')
q.put('123')
q.put("zxc")
q.put("asd")
# print(q.empty())
# print(q.full())
# print(q.qsize())
# 取数据
print(q.get())
print(q.get())
print(q.get())
print(q.get())
print(q.get())
end = datetime.datetime.now()
print("[ Finished in", (end - start), "]")
class EventEngine(object):
"""
事件驱动引擎
事件驱动引擎中所有的变量都设置为了私有，这是为了防止不小心
从外部修改了这些变量的值或状态，导致bug。
变量说明
__queue：私有变量，事件队列
__active：私有变量，事件引擎开关
__thread：私有变量，事件处理线程
__timer：私有变量，计时器
__handlers：私有变量，事件处理函数字典
方法说明
__run: 私有方法，事件处理线程连续运行用
__process: 私有方法，处理事件，调用注册在引擎中的监听函数
__onTimer：私有方法，计时器固定事件间隔触发后，向事件队列中存入计时器事件
start: 公共方法，启动引擎
stop：公共方法，停止引擎
register：公共方法，向引擎中注册监听函数
unregister：公共方法，向引擎中注销监听函数
put：公共方法，向事件队列中存入新的事件
事件监听函数必须定义为输入参数仅为一个event对象，即：
函数
def func(event)
...
对象方法
def method(self, event)
...
"""
# ----------------------------------------------------------------------
def __init__(self):
"""初始化事件引擎"""
# 事件队列
self.__queue = Queue()
# 事件引擎开关
self.__active = False
# 事件处理线程
self.__thread = Thread(target=self.__run)
# 计时器，用于触发计时器事件
self.__timer = QTimer()
self.__timer.timeout.connect(self.__onTimer)
# 这里的__handlers是一个字典，用来保存对应的事件调用关系
# 其中每个键对应的值是一个列表，列表中保存了对该事件进行监听的函数功能
self.__handlers = defaultdict(list)
# __generalHandlers是一个列表，用来保存通用回调函数（所有事件均调用）
self.__generalHandlers = []
#----------------------------------------------------------------------
def __run(self):
"""引擎运行"""
while self.__active == True:
try:
event = self.__queue.get(block=True,
timeout=1) # 获取事件的阻塞时间设为1秒
self.__process(event)
except Empty:
pass
# ----------------------------------------------------------------------
def __process(self, event):
"""处理事件"""
# 检查是否存在对该事件进行监听的处理函数
if event.type_ in self.__handlers:
# print("recv event")
# 若存在，则按顺序将事件传递给处理函数执行
[handler(event) for handler in self.__handlers[event.type_]]
# 以上语句为Python列表解析方式的写法，对应的常规循环写法为：
# for handler in self.__handlers[event.type_]:
# handler(event)
# print(event.type_)
# 调用通用处理函数进行处理
if self.__generalHandlers:
[handler(event) for handler in self.__generalHandlers]
# ----------------------------------------------------------------------
def __onTimer(self):
"""向事件队列中存入计时器事件"""
# 创建计时器事件
event = Event(type_=EVENT_TIMER)
# 向队列中存入计时器事件
self.put(event)
# ----------------------------------------------------------------------
def start(self, timer=True):
"""
引擎启动
timer：是否要启动计时器
"""
# 将引擎设为启动
self.__active = True
# 启动事件处理线程
self.__thread.start()
# 启动计时器，计时器事件间隔默认设定为1秒
if timer:
self.__timer.start(5000)
def stop(self):
"""停止引擎"""
# 将引擎设为停止
self.__active = False
# 停止计时器
self.__timer.stop()
# 等待事件处理线程退出
self.__thread.join()
# ----------------------------------------------------------------------
def register(self, type_, handler):
"""注册事件处理函数监听"""
# 尝试获取该事件类型对应的处理函数列表，若无defaultDict会自动创建新的list
handlerList = self.__handlers[type_]
# 若要注册的处理器不在该事件的处理器列表中，则注册该事件
if handler not in handlerList:
handlerList.append(handler)
# ----------------------------------------------------------------------
def unregister(self, type_, handler):
"""注销事件处理函数监听"""
# 尝试获取该事件类型对应的处理函数列表，若无则忽略该次注销请求
handlerList = self.__handlers[type_]
# 如果该函数存在于列表中，则移除
if handler in handlerList:
handlerList.remove(handler)
# 如果函数列表为空，则从引擎中移除该事件类型
if not handlerList:
del self.__handlers[type_]
# ----------------------------------------------------------------------
def put(self, event):
"""向事件队列中存入事件"""
self.__queue.put(event)
# ----------------------------------------------------------------------
def registerGeneralHandler(self, handler):
"""注册通用事件处理函数监听"""
if handler not in self.__generalHandlers:
self.__generalHandlers.append(handler)
# ----------------------------------------------------------------------
def unregisterGeneralHandler(self, handler):
"""注销通用事件处理函数监听"""
if handler in self.__generalHandlers:
self.__generalHandlers.remove(handler)
def test_hub_authenticated(request):
auth = HubAuth(cookie_name='jubal')
mock_model = {'name': 'jubalearly', 'groups': ['lions']}
cookie_url = url_path_join(auth.api_url, "authorizations/cookie",
auth.cookie_name)
good_url = url_path_join(cookie_url, "early")
bad_url = url_path_join(cookie_url, "late")
class TestHandler(HubAuthenticated, RequestHandler):
hub_auth = auth
@authenticated
def get(self):
self.finish(self.get_current_user())
# start hub-authenticated service in a thread:
port = 50505
q = Queue()
def run():
asyncio.set_event_loop(asyncio.new_event_loop())
app = Application([('/*', TestHandler)], login_url=auth.login_url)
http_server = HTTPServer(app)
http_server.listen(port)
loop = IOLoop.current()
loop.add_callback(lambda: q.put(loop))
loop.start()
t = Thread(target=run)
t.start()
def finish_thread():
loop.add_callback(loop.stop)
t.join(timeout=30)
assert not t.is_alive()
request.addfinalizer(finish_thread)
# wait for thread to start
loop = q.get(timeout=10)
with requests_mock.Mocker(real_http=True) as m:
# no cookie
r = requests.get('http://127.0.0.1:%i' % port, allow_redirects=False)
r.raise_for_status()
assert r.status_code == 302
assert auth.login_url in r.headers['Location']
# wrong cookie
m.get(bad_url, status_code=404)
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'late'},
allow_redirects=False,
)
r.raise_for_status()
assert r.status_code == 302
assert auth.login_url in r.headers['Location']
# clear the cache because we are going to request
# the same url again with a different result
auth.cache.clear()
# upstream 403
m.get(bad_url, status_code=403)
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'late'},
allow_redirects=False,
)
assert r.status_code == 500
m.get(good_url, text=json.dumps(mock_model))
# no specific allowed user
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'early'},
allow_redirects=False,
)
r.raise_for_status()
assert r.status_code == 200
# pass allowed user
TestHandler.hub_users = {'jubalearly'}
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'early'},
allow_redirects=False,
)
r.raise_for_status()
assert r.status_code == 200
# no pass allowed ser
TestHandler.hub_users = {'kaylee'}
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'early'},
allow_redirects=False,
)
assert r.status_code == 403
# pass allowed group
TestHandler.hub_groups = {'lions'}
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'early'},
allow_redirects=False,
)
r.raise_for_status()
assert r.status_code == 200
# no pass allowed group
TestHandler.hub_groups = {'tigers'}
r = requests.get(
'http://127.0.0.1:%i' % port,
cookies={'jubal': 'early'},
allow_redirects=False,
)
assert r.status_code == 403
class QueueObserver(RunObserver):
def __init__(self, covered_observer, interval=20, retry_interval=10):
self._covered_observer = covered_observer
self._retry_interval = retry_interval
self._interval = interval
self._queue = None
self._worker = None
self._stop_worker_event = None
def queued_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("queued_event", args, kwargs))
def started_event(self, *args, **kwargs):
self._queue = Queue()
self._stop_worker_event, self._worker = IntervalTimer.create(
self._run,
interval=self._interval,
)
self._worker.start()
# Putting the started event on the queue makes no sense
# as it is required for initialization of the covered observer.
return self._covered_observer.started_event(*args, **kwargs)
def heartbeat_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("heartbeat_event", args, kwargs))
def completed_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("completed_event", args, kwargs))
self.join()
def interrupted_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("interrupted_event", args, kwargs))
self.join()
def failed_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("failed_event", args, kwargs))
self.join()
def resource_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("resource_event", args, kwargs))
def artifact_event(self, *args, **kwargs):
self._queue.put(WrappedEvent("artifact_event", args, kwargs))
def log_metrics(self, metrics_by_name, info):
for metric_name, metric_values in metrics_by_name.items():
self._queue.put(
WrappedEvent(
"log_metrics",
[metric_name, metric_values, info],
{},
)
)
def _run(self):
while not self._queue.empty():
try:
event = self._queue.get()
except IndexError:
# Currently there is no event on the queue so
# just go back to sleep.
pass
else:
try:
# method = getattr(self._covered_observer, event.name)
method = getattr(self._covered_observer, event.name)
except NameError:
# covered observer does not implement event handler
# for the event, so just
# discard the message.
self._queue.task_done()
else:
while True:
try:
method(*event.args, **event.kwargs)
except:
# Something went wrong during the processing of
# the event so wait for some time and
# then try again.
self._stop_worker_event.wait(self._retry_interval)
continue
else:
self._queue.task_done()
break
def join(self):
if self._queue is not None:
self._queue.join()
self._stop_worker_event.set()
self._worker.join(timeout=10)
def __getattr__(self, item):
return getattr(self._covered_observer, item)
def __eq__(self, other):
return self._covered_observer == other
def launch(self, session, entities, event):
self.show_message(event, "Processing...", True)
values = event["data"]["values"]
action_id = values.get("action_id")
spec_data = self.action_data_by_id.get(action_id)
if not spec_data:
# it is a bug if this happens!
return {
"success": False,
"message": "Something bad has happened. Please try again."
}
report_messages = collections.defaultdict(list)
project_name = spec_data["project_name"]
to_delete = spec_data["to_delete"]
self.dbcon.Session["AVALON_PROJECT"] = project_name
assets_to_delete = to_delete.get("assets") or []
subsets_to_delete = to_delete.get("subsets") or []
# Convert asset ids to ObjectId obj
assets_to_delete = [ObjectId(id) for id in assets_to_delete if id]
subset_ids_by_parent = spec_data["subset_ids_by_parent"]
subset_ids_by_name = spec_data["subset_ids_by_name"]
subset_ids_to_archive = []
asset_ids_to_archive = []
ftrack_ids_to_delete = []
if len(assets_to_delete) > 0:
map_av_ftrack_id = spec_data["without_ftrack_id"]
# Prepare data when deleting whole avalon asset
avalon_assets = self.dbcon.find({"type": "asset"})
avalon_assets_by_parent = collections.defaultdict(list)
for asset in avalon_assets:
asset_id = asset["_id"]
parent_id = asset["data"]["visualParent"]
avalon_assets_by_parent[parent_id].append(asset)
if asset_id in assets_to_delete:
ftrack_id = map_av_ftrack_id.get(str(asset_id))
if not ftrack_id:
ftrack_id = asset["data"].get("ftrackId")
if not ftrack_id:
continue
ftrack_ids_to_delete.append(ftrack_id)
children_queue = Queue()
for mongo_id in assets_to_delete:
children_queue.put(mongo_id)
while not children_queue.empty():
mongo_id = children_queue.get()
if mongo_id in asset_ids_to_archive:
continue
asset_ids_to_archive.append(mongo_id)
for subset_id in subset_ids_by_parent.get(mongo_id, []):
if subset_id not in subset_ids_to_archive:
subset_ids_to_archive.append(subset_id)
children = avalon_assets_by_parent.get(mongo_id)
if not children:
continue
for child in children:
child_id = child["_id"]
if child_id not in asset_ids_to_archive:
children_queue.put(child_id)
# Prepare names of assets in ftrack and ids of subsets in mongo
asset_names_to_delete = []
if len(subsets_to_delete) > 0:
for name in subsets_to_delete:
asset_names_to_delete.append(name)
for subset_id in subset_ids_by_name[name]:
if subset_id in subset_ids_to_archive:
continue
subset_ids_to_archive.append(subset_id)
# Get ftrack ids of entities where will be delete only asset
not_deleted_entities_id = []
ftrack_id_name_map = {}
if asset_names_to_delete:
for entity in entities:
ftrack_id = entity["id"]
ftrack_id_name_map[ftrack_id] = entity["name"]
if ftrack_id not in ftrack_ids_to_delete:
not_deleted_entities_id.append(ftrack_id)
mongo_proc_txt = "MongoProcessing: "
ftrack_proc_txt = "Ftrack processing: "
if asset_ids_to_archive:
self.log.debug("{}Archivation of assets <{}>".format(
mongo_proc_txt,
", ".join([str(id) for id in asset_ids_to_archive])))
self.dbcon.update_many(
{
"_id": {
"$in": asset_ids_to_archive
},
"type": "asset"
}, {"$set": {
"type": "archived_asset"
}})
if subset_ids_to_archive:
self.log.debug("{}Archivation of subsets <{}>".format(
mongo_proc_txt,
", ".join([str(id) for id in subset_ids_to_archive])))
self.dbcon.update_many(
{
"_id": {
"$in": subset_ids_to_archive
},
"type": "subset"
}, {"$set": {
"type": "archived_subset"
}})
if ftrack_ids_to_delete:
self.log.debug("{}Deleting Ftrack Entities <{}>".format(
ftrack_proc_txt, ", ".join(ftrack_ids_to_delete)))
ftrack_ents_to_delete = (self._filter_entities_to_delete(
ftrack_ids_to_delete, session))
for entity in ftrack_ents_to_delete:
session.delete(entity)
try:
session.commit()
except Exception:
ent_path = "/".join(
[ent["name"] for ent in entity["link"]])
msg = "Failed to delete entity"
report_messages[msg].append(ent_path)
session.rollback()
self.log.warning("{} <{}>".format(msg, ent_path),
exc_info=True)
if not_deleted_entities_id:
joined_not_deleted = ", ".join([
"\"{}\"".format(ftrack_id)
for ftrack_id in not_deleted_entities_id
])
joined_asset_names = ", ".join(
["\"{}\"".format(name) for name in asset_names_to_delete])
# Find assets of selected entities with names of checked subsets
assets = session.query(
("select id from Asset where"
" context_id in ({}) and name in ({})").format(
joined_not_deleted, joined_asset_names)).all()
self.log.debug("{}Deleting Ftrack Assets <{}>".format(
ftrack_proc_txt, ", ".join([asset["id"] for asset in assets])))
for asset in assets:
session.delete(asset)
try:
session.commit()
except Exception:
session.rollback()
msg = "Failed to delete asset"
report_messages[msg].append(asset["id"])
self.log.warning("Asset: {} <{}>".format(
asset["name"], asset["id"]),
exc_info=True)
return self.report_handle(report_messages, project_name, event)
class PreviousCH(ExactMethodForDGP):
m_QueueForWindows = None
#A Queue object
m_QueueForPseudoSources = None
#A Queue object
m_InfoAtAngles = None
#A simple list object
def __init__(self, *, inputModel=None, indexOfSourceVerts=None):
super().__init__(inputModel=inputModel,
indexOfSourceVerts=indexOfSourceVerts)
self.nameOfAlgorithm = "CH"
def InitContainers(self):
# self.m_QueueForPseudoSources = deque();
# self.m_QueueForWindows = deque();
self.m_QueueForPseudoSources = Queue()
self.m_QueueForWindows = Queue()
self.m_InfoAtAngles = [
InfoAtAngle() for i in range(self.model.GetNumOfEdges())
]
self.memory += float(self.model.GetNumOfEdges()) * sys.getsizeof(
InfoAtAngle) / 1024 / 1024
def BuildSequenceTree(self):
self.ComputeChildrenOfSource()
fFromQueueOfPseudoSources = self.UpdateTreeDepthBackWithChoice()
while (self.depthOfResultingTree < self.model.GetNumOfFaces()
and not (self.m_QueueForPseudoSources.empty()
and self.m_QueueForWindows.empty())):
# print('------------------------------------------------------------');
# print('DEPTHOFRESULTINGTREE, QUEUEFORPSEUDOSOURCES, QUEUEFORWINDOWS');
# print(self.depthOfResultingTree, len(self.m_QueueForPseudoSources), len(self.m_QueueForWindows));
# print('BEFORE UPDATING FROMQUEUOFPSEUDOSOURCES', int(fFromQueueOfPseudoSources));
if (self.m_QueueForWindows.qsize() > self.nMaxLenOfWindowQueue):
self.nMaxLenOfWindowQueue = self.m_QueueForWindows.qsize()
if (self.m_QueueForPseudoSources.qsize() >
self.nMaxLenOfPseudoSources):
self.nMaxLenOfPseudoSources = self.m_QueueForPseudoSources.qsize(
)
if (fFromQueueOfPseudoSources):
item = self.m_QueueForPseudoSources.get()
indexOfVert = item.indexOfVert
self.ComputeChildrenOfPseudoSource(indexOfVert)
else:
quoteW = self.m_QueueForWindows.get()
self.ComputeChildrenOfWindow(quoteW)
quoteW.pWindow = None
fFromQueueOfPseudoSources = self.UpdateTreeDepthBackWithChoice()
# print('AFTER UPDATING FROMQUEUOFPSEUDOSOURCES', int(fFromQueueOfPseudoSources));
# print('------------------------------------------------------------');
def FillExperimentalResults(self):
self.NPE = 1
def ClearContainers(self):
self.m_QueueForWindows = Queue()
self.m_QueueForPseudoSources = Queue()
def AddIntoQueueOfPseudoSources(self, quoteOfPseudoSource):
# self.m_QueueForPseudoSources.append(quoteOfPseudoSource);
self.m_QueueForPseudoSources.put(quoteOfPseudoSource)
def AddIntoQueueOfWindows(self, quoteW):
self.m_QueueForWindows.put(quoteW)
self.nCountOfWindows += 1
def UpdateTreeDepthBackWithChoice(self):
while (not self.m_QueueForPseudoSources.empty()
and (self.m_QueueForPseudoSources.queue[0].birthTime !=
self.m_InfoAtVertices[self.m_QueueForPseudoSources.
queue[0].indexOfVert].birthTime)):
self.m_QueueForPseudoSources.get()
while (not self.m_QueueForWindows.empty()):
quoteW = self.m_QueueForWindows.queue[0]
if (quoteW.pWindow.fParentIsPseudoSource):
if (quoteW.pWindow.birthTimeOfParent != self.m_InfoAtVertices[
quoteW.pWindow.indexOfParent].birthTime):
quoteW.pWindow = None
self.m_QueueForWindows.get()
else:
break
else:
if (quoteW.pWindow.birthTimeOfParent == self.m_InfoAtAngles[
quoteW.pWindow.indexOfParent].birthTime):
break
elif (quoteW.pWindow.fIsOnLeftSubtree == (
quoteW.pWindow.entryPropOfParent < self.m_InfoAtAngles[
quoteW.pWindow.indexOfParent].entryProp)):
break
else:
quoteW.pWindow = None
self.m_QueueForWindows.get()
fFromQueueOfPseudoSources = False
if (self.m_QueueForWindows.empty()):
if (not self.m_QueueForPseudoSources.empty()):
infoOfHeadElemOfPseudoSources = self.m_InfoAtVertices[
self.m_QueueForPseudoSources.queue[0].indexOfVert]
self.depthOfResultingTree = max(
self.depthOfResultingTree,
infoOfHeadElemOfPseudoSources.level)
fFromQueueOfPseudoSources = True
else:
if (self.m_QueueForPseudoSources.empty()):
infoOfHeadElemOfWindows = self.m_QueueForWindows.queue[
0].pWindow
self.depthOfResultingTree = max(self.depthOfResultingTree,
infoOfHeadElemOfWindows.level)
fFromQueueOfPseudoSources = False
else:
infoOfHeadElemOfPseudoSources = self.m_InfoAtVertices[
self.m_QueueForPseudoSources.queue[0].indexOfVert]
infoOfHeadElemOfWindows = self.m_QueueForWindows.queue[
0].pWindow
if (infoOfHeadElemOfPseudoSources.level <=
infoOfHeadElemOfWindows.level):
self.depthOfResultingTree = max(
self.depthOfResultingTree,
infoOfHeadElemOfPseudoSources.level)
fFromQueueOfPseudoSources = True
else:
self.depthOfResultingTree = max(
self.depthOfResultingTree,
infoOfHeadElemOfWindows.level)
fFromQueueOfPseudoSources = False
return fFromQueueOfPseudoSources
def CheckValidityOfWindow(self, w):
return True
def IsTooNarrowWindow(self, w):
return ((w.proportions[1] - w.proportions[0]) <
Constants.LENGTH_EPSILON_CONTROL)
def ComputeChildrenOfSource(self, indexOfSourceVert=-1):
if (indexOfSourceVert != -1):
self.m_InfoAtVertices[indexOfSourceVert].birthTime += 1
self.m_InfoAtVertices[indexOfSourceVert].level = 0
self.m_InfoAtVertices[indexOfSourceVert].disUptodate = 0.0
degree = len(self.model.Neigh(indexOfSourceVert))
for i in range(degree):
self.FillVertChildOfPseudoSource(indexOfSourceVert, i)
for i in range(degree):
self.CreateIntervalChildOfPseudoSource(indexOfSourceVert, i)
else:
for it in self.indexOfSourceVerts:
if (it >= self.model.GetNumOfVerts()):
continue
self.ComputeChildrenOfSource(indexOfSourceVert=it)
def ComputeChildrenOfPseudoSourceFromPseudoSource(self,
indexOfParentVertex):
degree = len(self.model.Neigh(indexOfParentVertex))
neighs = self.model.Neigh(indexOfParentVertex)
indexOfParentOfParent = self.m_InfoAtVertices[
indexOfParentVertex].indexOfParent
subIndex = self.model.GetSubindexToVert(indexOfParentVertex,
indexOfParentOfParent)
angleSum = 0.0
indexPlus = subIndex
while (indexPlus != ((subIndex - 1 + degree) % degree)):
angleSum += neighs[indexPlus].second
if (angleSum >
(Constants.M_PI - Constants.ToleranceOfConvexAngle)):
break
indexPlus = (indexPlus + 1) % degree
angleSum = 0.0
indexMinus = (subIndex - 1 + degree) % degree
while ((indexMinus == (subIndex - 1 + degree) % degree)
or (indexMinus != (indexPlus - 1 + degree) % degree)):
angleSum += neighs[indexMinus].second
if (angleSum >
(Constants.M_PI - Constants.ToleranceOfConvexAngle)):
break
indexMinus = (indexMinus - 1 + degree) % degree
if (indexMinus == ((indexPlus - 1 + degree) % degree)):
return
#vertices;
i = (indexPlus + 1) % degree
while (i != (indexMinus + 1) % degree):
self.FillVertChildOfPseudoSource(indexOfParentVertex, i)
i = (i + 1) % degree
#windows
i = indexPlus
while (i != (indexMinus + 1) % degree):
self.CreateIntervalChildOfPseudoSource(indexOfParentVertex, i)
i = (i + 1) % degree
def ComputeChildrenOfPseudoSourceFromWindow(self, indexOfParentVertex):
degree = len(self.model.Neigh(indexOfParentVertex))
neighs = self.model.Neigh(indexOfParentVertex)
indexOfParentOfParent = self.m_InfoAtVertices[
indexOfParentVertex].indexOfParent
leftVert = self.model.m_Edges[indexOfParentOfParent].indexOfLeftVert
rightVert = self.model.m_Edges[indexOfParentOfParent].indexOfRightVert
subIndexLeft = self.model.GetSubindexToVert(indexOfParentVertex,
leftVert)
subIndexRight = (subIndexLeft + 1) % degree
x1 = self.m_InfoAtVertices[
indexOfParentVertex].entryProp * self.model.m_Edges[
indexOfParentOfParent].edge_length
y1 = 0.0
x2 = self.model.m_Edges[indexOfParentOfParent].edge_length
y2 = 0.0
x1 -= self.model.m_Edges[
indexOfParentOfParent].coordOfOppositeVert.first
y1 -= self.model.m_Edges[
indexOfParentOfParent].coordOfOppositeVert.second
x2 -= self.model.m_Edges[
indexOfParentOfParent].coordOfOppositeVert.first
y2 -= self.model.m_Edges[
indexOfParentOfParent].coordOfOppositeVert.second
try:
anglePlus = ((x1 * x2) + (y1 * y2)) / ((x1**2 + y1**2) *
(x2**2 + y2**2))**0.5
except ZeroDivisionError:
anglePlus = 0.0
anglePlus = math.acos(max(-1.0, min(1.0, anglePlus)))
angleSum = anglePlus
indexPlus = subIndexRight
while (indexPlus != subIndexLeft):
angleSum += neighs[indexPlus].second
if (angleSum >
(Constants.M_PI - Constants.ToleranceOfConvexAngle)):
break
indexPlus = (indexPlus + 1) % degree
angleSum = neighs[subIndexLeft].second - anglePlus
indexMinus = (subIndexLeft - 1 + degree) % degree
while (indexMinus != (indexPlus - 1 + degree) % degree):
angleSum += neighs[indexMinus].second
if (angleSum >
(Constants.M_PI - Constants.ToleranceOfConvexAngle)):
break
indexMinus = (indexMinus - 1 + degree) % degree
if (indexMinus == (indexPlus - 1 + degree) % degree):
return
for i in range(degree):
self.FillVertChildOfPseudoSource(indexOfParentVertex, i)
#windows
i = indexPlus
while (i != (indexMinus + 1) % degree):
self.CreateIntervalChildOfPseudoSource(indexOfParentVertex, i)
i = (i + 1) % degree
def ComputeChildrenOfWindow(self, quoteParentWindow):
w = quoteParentWindow.pWindow
edge = self.model.m_Edges[w.indexOfCurEdge]
entryProp = self.model.ProportionOnEdgeByImage(w.indexOfCurEdge,
w.coordOfPseudoSource)
# print("ENTRYPROP, PROPORTION 0, PROPOTION 1 ", entryProp, w.proportions[0], w.proportions[1]);
if (entryProp >= w.proportions[1]):
self.ComputeTheOnlyLeftChild(w)
return
if (entryProp <= w.proportions[0]):
self.ComputeTheOnlyRightChild(w)
return
disToAngle = self.model.DistanceToIncidentAngle(
w.indexOfCurEdge, w.coordOfPseudoSource)
incidentVertex = edge.indexOfOppositeVert
fLeftChildToCompute = False
fRightChildToCompute = False
fWIsWinning = False
totalDis = w.disToRoot + disToAngle
if (self.m_InfoAtAngles[w.indexOfCurEdge].birthTime == -1):
fLeftChildToCompute = True
fRightChildToCompute = True
fWIsWinning = True
else:
if (totalDis < (self.m_InfoAtAngles[w.indexOfCurEdge].disUptodate -
Constants.LENGTH_EPSILON_CONTROL)):
fLeftChildToCompute = True
fRightChildToCompute = True
fWIsWinning = True
else:
fLeftChildToCompute = (
entryProp <
self.m_InfoAtAngles[w.indexOfCurEdge].entryProp)
fRightChildToCompute = not fLeftChildToCompute
fWIsWinning = False
if (not fWIsWinning):
if (fLeftChildToCompute):
self.ComputeTheOnlyLeftTrimmedChild(w)
if (fRightChildToCompute):
self.ComputeTheOnlyRightTrimmedChild(w)
return
self.m_InfoAtAngles[w.indexOfCurEdge].disUptodate = totalDis
self.m_InfoAtAngles[w.indexOfCurEdge].entryProp = entryProp
self.m_InfoAtAngles[w.indexOfCurEdge].birthTime += 1
self.ComputeLeftTrimmedChildWithParent(w)
self.ComputeRightTrimmedChildWithParent(w)
if (totalDis < (self.m_InfoAtVertices[incidentVertex].disUptodate -
Constants.LENGTH_EPSILON_CONTROL)):
self.m_InfoAtVertices[incidentVertex].fParentIsPseudoSource = False
self.m_InfoAtVertices[incidentVertex].birthTime += 1
self.m_InfoAtVertices[
incidentVertex].indexOfParent = w.indexOfCurEdge
self.m_InfoAtVertices[
incidentVertex].indexOfRootVertOfParent = w.indexOfRoot
self.m_InfoAtVertices[incidentVertex].level = w.level + 1
self.m_InfoAtVertices[incidentVertex].disUptodate = totalDis
self.m_InfoAtVertices[incidentVertex].entryProp = entryProp
if (not self.model.IsConvexVert(incidentVertex)):
self.AddIntoQueueOfPseudoSources(
QuoteInfoAtVertex(
birthTime=self.m_InfoAtVertices[incidentVertex].
birthTime,
indexOfVert=incidentVertex,
disUptodate=totalDis))
def ComputeChildrenOfPseudoSource(self, indexOfParentVertex):
if (self.m_InfoAtVertices[indexOfParentVertex].fParentIsPseudoSource):
self.ComputeChildrenOfPseudoSourceFromPseudoSource(
indexOfParentVertex)
else:
self.ComputeChildrenOfPseudoSourceFromWindow(indexOfParentVertex)
def CreateIntervalChildOfPseudoSource(self, source,
subIndexOfIncidentEdge):
indexOfIncidentEdge = self.model.Neigh(
source)[subIndexOfIncidentEdge].first
if (self.model.IsExtremeEdge(indexOfIncidentEdge)):
return
edge = self.model.m_Edges[indexOfIncidentEdge]
edgeIndex = edge.indexOfRightEdge
if (self.model.IsExtremeEdge(edgeIndex)):
return
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.fParentIsPseudoSource = True
quoteW.pWindow.fDirectParenIsPseudoSource = True
quoteW.pWindow.birthTimeOfParent = self.m_InfoAtVertices[
source].birthTime
quoteW.pWindow.indexOfParent = source
quoteW.pWindow.indexOfRoot = source
quoteW.pWindow.indexOfCurEdge = edgeIndex
quoteW.pWindow.level = self.m_InfoAtVertices[source].level + 1
quoteW.pWindow.disToRoot = self.m_InfoAtVertices[source].disUptodate
quoteW.pWindow.proportions[0] = 0.0
quoteW.pWindow.proportions[1] = 1.0
quoteW.pWindow.entryPropOfParent
reverseEdge = self.model.m_Edges[edgeIndex].indexOfReverseEdge
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByReversingCurrentEdge(
reverseEdge, self.model.m_Edges[reverseEdge].coordOfOppositeVert)
self.AddIntoQueueOfWindows(quoteW)
def FillVertChildOfPseudoSource(self, source, subIndexOfVert):
edge = self.model.m_Edges[self.model.Neigh(source)
[subIndexOfVert].first]
index = edge.indexOfRightVert
dis = (self.m_InfoAtVertices[source].disUptodate + edge.edge_length)
if (dis >= (self.m_InfoAtVertices[index].disUptodate -
Constants.LENGTH_EPSILON_CONTROL)):
return
self.m_InfoAtVertices[index].fParentIsPseudoSource = True
self.m_InfoAtVertices[index].birthTime += 1
self.m_InfoAtVertices[index].indexOfParent = source
self.m_InfoAtVertices[
index].level = self.m_InfoAtVertices[source].level + 1
self.m_InfoAtVertices[index].disUptodate = dis
if (not self.model.IsConvexVert(index)):
self.AddIntoQueueOfPseudoSources(
QuoteInfoAtVertex(
birthTime=self.m_InfoAtVertices[index].birthTime,
indexOfVert=index,
disUptodate=dis))
def ComputeTheOnlyLeftChild(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfLeftEdge)):
return
# print('COMPUTE THE ONLY LEFT CHILD');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[0] = self.model.ProportionOnLeftEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[0])
quoteW.pWindow.proportions[0] = max(0.0, quoteW.pWindow.proportions[0])
quoteW.pWindow.proportions[1] = self.model.ProportionOnLeftEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[1])
quoteW.pWindow.proportions[1] = min(1.0, quoteW.pWindow.proportions[1])
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = w.fParentIsPseudoSource
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = True
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfLeftEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundLeftChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fIsOnLeftSubtree = w.fIsOnLeftSubtree
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.entryPropOfParent = w.entryPropOfParent
quoteW.pWindow.birthTimeOfParent = w.birthTimeOfParent
quoteW.pWindow.indexOfParent = w.indexOfParent
quoteW.pWindow.indexOfRoot = w.indexOfRoot
self.AddIntoQueueOfWindows(quoteW)
def ComputeTheOnlyRightChild(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfRightEdge)):
return
# print('COMPUTE THE ONLY RIGHT CHILD');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[
0] = self.model.ProportionOnRightEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[0])
quoteW.pWindow.proportions[0] = max(0.0, quoteW.pWindow.proportions[0])
quoteW.pWindow.proportions[
1] = self.model.ProportionOnRightEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[1])
quoteW.pWindow.proportions[1] = min(1.0, quoteW.pWindow.proportions[1])
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = w.fParentIsPseudoSource
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = False
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfRightEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundRightChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.birthTimeOfParent = w.birthTimeOfParent
quoteW.pWindow.indexOfParent = w.indexOfParent
quoteW.pWindow.indexOfRoot = w.indexOfRoot
quoteW.pWindow.fIsOnLeftSubtree = w.fIsOnLeftSubtree
quoteW.pWindow.entryPropOfParent = w.entryPropOfParent
self.AddIntoQueueOfWindows(quoteW)
def ComputeTheOnlyLeftTrimmedChild(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfLeftEdge)):
return
# print('COMPUTE THE ONLY LEFT TRIMMED CHILD');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[0] = self.model.ProportionOnLeftEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[0])
quoteW.pWindow.proportions[0] = max(0.0, quoteW.pWindow.proportions[0])
quoteW.pWindow.proportions[1] = 1
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = w.fParentIsPseudoSource
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = True
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfLeftEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundLeftChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.birthTimeOfParent = w.birthTimeOfParent
quoteW.pWindow.indexOfParent = w.indexOfParent
quoteW.pWindow.indexOfRoot = w.indexOfRoot
quoteW.pWindow.fIsOnLeftSubtree = w.fIsOnLeftSubtree
quoteW.pWindow.entryPropOfParent = w.entryPropOfParent
self.AddIntoQueueOfWindows(quoteW)
def ComputeTheOnlyRightTrimmedChild(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfRightEdge)):
return
# print('COMPUTE THE ONLY RIGHT TRIMMED CHILD');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[0] = 0.0
quoteW.pWindow.proportions[
1] = self.model.ProportionOnRightEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[1])
quoteW.pWindow.proportions[1] = min(1.0, quoteW.pWindow.proportions[1])
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = w.fParentIsPseudoSource
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = False
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfRightEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundRightChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.birthTimeOfParent = w.birthTimeOfParent
quoteW.pWindow.indexOfParent = w.indexOfParent
quoteW.pWindow.indexOfRoot = w.indexOfRoot
quoteW.pWindow.fIsOnLeftSubtree = w.fIsOnLeftSubtree
quoteW.pWindow.entryPropOfParent = w.entryPropOfParent
self.AddIntoQueueOfWindows(quoteW)
def ComputeLeftTrimmedChildWithParent(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfLeftEdge)):
return
# print('COMPUTE THE ONLY LEFT TRIMMED CHILD WITH PARENT');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[0] = self.model.ProportionOnLeftEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[0])
quoteW.pWindow.proportions[0] = max(0.0, quoteW.pWindow.proportions[0])
quoteW.pWindow.proportions[1] = 1
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = False
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = True
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfLeftEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundLeftChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.birthTimeOfParent = self.m_InfoAtAngles[
w.indexOfCurEdge].birthTime
quoteW.pWindow.indexOfParent = w.indexOfCurEdge
quoteW.pWindow.indexOfRoot = w.indexOfRoot
quoteW.pWindow.fIsOnLeftSubtree = True
quoteW.pWindow.entryPropOfParent = self.m_InfoAtAngles[
w.indexOfCurEdge].entryProp
self.AddIntoQueueOfWindows(quoteW)
def ComputeRightTrimmedChildWithParent(self, w):
if (self.model.IsExtremeEdge(
self.model.m_Edges[w.indexOfCurEdge].indexOfRightEdge)):
return
# print('COMPUTE THE ONLY RIGHT TRIMMED CHILD WITH PARENT');
quoteW = QuoteWindow()
quoteW.pWindow = Window()
quoteW.pWindow.proportions[0] = 0.0
quoteW.pWindow.proportions[
1] = self.model.ProportionOnRightEdgeByImage(
w.indexOfCurEdge, w.coordOfPseudoSource, w.proportions[1])
quoteW.pWindow.proportions[1] = min(1.0, quoteW.pWindow.proportions[1])
if (self.IsTooNarrowWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fParentIsPseudoSource = False
quoteW.pWindow.fDirectParenIsPseudoSource = False
quoteW.pWindow.fDirectParentEdgeOnLeft = False
quoteW.pWindow.indexOfCurEdge = self.model.m_Edges[
w.indexOfCurEdge].indexOfRightEdge
quoteW.pWindow.disToRoot = w.disToRoot
quoteW.pWindow.coordOfPseudoSource = self.model.GetNew2DCoordinatesByRotatingAroundRightChildEdge(
w.indexOfCurEdge, w.coordOfPseudoSource)
if (not self.CheckValidityOfWindow(quoteW.pWindow)):
quoteW.pWindow = None
return
quoteW.pWindow.fIsOnLeftSubtree = False
quoteW.pWindow.birthTimeOfParent = self.m_InfoAtAngles[
w.indexOfCurEdge].birthTime
quoteW.pWindow.indexOfParent = w.indexOfCurEdge
quoteW.pWindow.indexOfRoot = w.indexOfRoot
quoteW.pWindow.level = w.level + 1
quoteW.pWindow.entryPropOfParent = self.m_InfoAtAngles[
w.indexOfCurEdge].entryProp
self.AddIntoQueueOfWindows(quoteW)
model.eval()
model.device()
print ('Trained model loaded: %s' % args.model)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
torch.set_grad_enabled(False)
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
# print ('Loading initial frames...')
lastframe = first_image
I1 = torch.from_numpy(np.transpose(lastframe, (2,0,1))).to(device, non_blocking=True).unsqueeze(0)
I1 = F.pad(I1, padding)
frame = read_buffer.get()
for nn in range(1, input_duration+1):
frame = read_buffer.get()
if frame is None:
break
I0 = I1
I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0)
I1 = F.pad(I1, padding)
output = make_inference(model, I0, I1, args.exp, args.UHD)
write_buffer.put(lastframe)
for mid in output:
if sys.platform == 'darwin':
class RetrieveIndexThread(threading.Thread):
"""
A thread to run retrieval of the Saleforce Documentation index, and access the reference_cache
"""
def __init__(self,
window,
doc_type,
open_when_done=True,
sublime_opening_cache_refresh=False):
"""
:param window:
An instance of :class:`sublime.Window` that represents the Sublime
Text window to show the available package list in.
:param doc_type:
Either:
- a salesforce_reference.retrieve.DocType (usually obtained from
salesforce_reference.retrieve.DocTypeEnum) indicating the
docType to retrieve
- the string "*" - indicating that all documentation types should
be retrieved and displayed at the same time
:param open_when_done:
Whether this thread is being run solely for caching, or should open
the documentation list for user selection when done. Defaults to
true - should open the documentaton list when done
:param sublime_opening_cache_refresh:
flag indicating whether this is a cache refresh happening while
sublime is opening. Setting this to True causes special behaviour:
- open_when_done param will be set to False as this is not a user
initiated action, so opening the doc searcher would be jarring
- doc_type will be ignored - we will cache all the doc types we can,
however...
- if refreshCacheOnLoad is set to False in the settings for a
particular doc type, this doc type will not be cached
"""
self.window = window
if not isinstance(doc_type, DocType) and doc_type != "*":
raise ValueError(
"doc_type parameter to RetrieveIndexThread must be either "
"an instance of DocType (or a subclass), or the string '*' "
"indicating you want to retrieve and display all available "
"documentation types ")
self.doc_type = doc_type
self.open_when_done = open_when_done
self.sublime_opening_cache_refresh = sublime_opening_cache_refresh
if sublime_opening_cache_refresh:
self.doc_type = "*"
self.open_when_done = False
self.queue = Queue()
global reference_cache
threading.Thread.__init__(self)
def run(self):
if self.doc_type == "*":
all_doc_type_settings = settings.get("docTypes")
for doc_type in DocTypeEnum.get_all():
if all_doc_type_settings is None:
specific_doc_type_settings = None
else:
specific_doc_type_settings = all_doc_type_settings.get(
doc_type.name.lower())
if specific_doc_type_settings is None:
exclude = False
refresh_on_load = False
else:
exclude = specific_doc_type_settings.get(
"excludeFromAllDocumentationCommand")
refresh_on_load = specific_doc_type_settings.get(
"refreshCacheOnLoad")
if (not exclude
and not reference_cache.entries_by_doc_type.get(
doc_type.name)
and not (self.sublime_opening_cache_refresh
and not refresh_on_load)):
self.queue.put(
doc_type.preferred_strategy(self.window,
reference_cache,
cache_lock,
self.queue.task_done))
else:
if not reference_cache.entries_by_doc_type.get(self.doc_type.name):
self.queue.put(
self.doc_type.preferred_strategy(self.window,
reference_cache,
cache_lock,
self.queue.task_done))
while not self.queue.empty():
self.queue.get().start()
if (self.open_when_done):
self.queue.join()
if self.doc_type == "*":
self.window.show_quick_panel(reference_cache.titles,
self.open_documentation)
else:
self.window.show_quick_panel(
reference_cache.titles_by_doc_type.get(self.doc_type.name),
self.open_documentation)
def open_documentation(self, reference_index):
url = ""
if (reference_index != -1):
if self.doc_type == "*":
entry = reference_cache.entries[reference_index]
else:
entry = reference_cache.entries_by_doc_type.get(
self.doc_type.name)[reference_index]
if entry:
if self.doc_type == "*":
url = DocTypeEnum.get_by_name(
entry.doc_type).doc_base_url + entry.url
else:
url = self.doc_type.doc_base_url + entry.url
if url:
webbrowser.open_new_tab(url)
def get_cpu_fun(self):
device = self.check_device_fun()
self.interval = self.interval_time.currentText()
self.durtime = self.during_time.currentText()
self.package = self.test_package.currentText()
if device == 1:
self.create_excel()
self.get_cpu.setEnabled(False)
self.show_msg("采集性能数据")
self.getData = 1
path = "D:\\PerfReport"
self.com.mkdir(path)
self.cpu_thread = CpuThread(self.excel, self.writeSheet, self.wb,
self.interval, self.durtime,
self.package, self.lock)
self.mem_thread = MemThread(self.excel, self.writeSheet, self.wb,
self.interval, self.durtime,
self.package, self.lock)
self.fps_thread = FpsThread(self.excel, self.writeSheet, self.wb,
self.interval, self.durtime,
self.package, self.lock)
self.net_thread = NetThread(self.excel, self.writeSheet, self.wb,
self.interval, self.durtime,
self.package, self.lock)
self.tempe_thread = TempeThread(self.excel, self.writeSheet,
self.wb, self.interval,
self.durtime, self.package,
self.lock)
self.battery_thread = BatteryThread(self.excel, self.writeSheet,
self.wb, self.interval,
self.durtime, self.package,
self.lock)
self.drawcall_thread = DrawcallThread(self.excel, self.writeSheet,
self.wb, self.interval,
self.durtime, self.package,
self.lock)
self.time_thread = TimeThread(self.excel, self.writeSheet, self.wb,
self.interval, self.durtime,
self.package, self.lock)
self.mem_thread.trigger.connect(self.stop_get_mem)
self.cpu_thread.trigger.connect(self.stop_get_cpu)
self.fps_thread.trigger.connect(self.stop_get_fps)
self.net_thread.trigger.connect(self.stop_get_net)
self.tempe_thread.trigger.connect(self.stop_get_tempe)
self.battery_thread.trigger.connect(self.stop_get_battery)
self.drawcall_thread.trigger.connect(self.stop_get_drawcall)
self.time_thread.trigger.connect(self.stop_get_time)
QueThread = Queue()
QueThread.put(self.cpu_thread)
QueThread.put(self.fps_thread)
QueThread.put(self.battery_thread)
QueThread.put(self.mem_thread)
QueThread.put(self.tempe_thread)
QueThread.put(self.net_thread)
QueThread.put(self.drawcall_thread)
QueThread.put(self.time_thread)
while not QueThread.empty():
QueThread.get().start()
self.isCollect = 1
self.stop_collect.setEnabled(True)
class Service(object):
"""Minimal GenServer like actor for handling uploads"""
def __init__(self, parent=None):
"""Initialize service"""
self.mailbox = Queue()
self.runner = None
self.parent = parent
def spawn(self, *args, **kwargs):
"""Start thread"""
if self.runner:
return
self.runner = threading.Thread(target=self.main)
self.runner.start()
# Invoke setup callback
self.init(*args, **kwargs)
return self
def terminate(self):
"""Shutdown"""
self.send('end')
self.runner.join()
return self
def main(self):
"""Gen server like default behaviour"""
for message in self.receive():
if type(message) is not tuple:
raise Exception("Unexpected input")
if message[0] == 'cast':
self.handle_cast(message[1])
elif message[0] == 'call':
replies = message[1]
result = self.handle_call(message[2])
replies.put(result)
def send(self, message):
"""Add a message to the queue"""
self.mailbox.put(message)
return self
def receive(self):
"""Read from queue"""
while True:
message = self.mailbox.get()
if message == 'end':
return
yield message
def call(self, action):
"""Sync interaction"""
replies = Queue()
self.send(('call', replies, action))
# Get response
return replies.get()
def cast(self, action):
"""Async interaction"""
return self.send(('cast', action))
def init(self):
"""Initialize callback"""
pass
def handle_call(self, action):
"""Handle sync action"""
return False
def hadle_cast(self, action):
"""Handle async action"""
pass
def find_shortest_path(self, from_vertex, to_vertex):
"""Search for the shortest path from vertex a to b using Breadth first search
Args:
from_vertex (str) : starting point on the graph
to_vertex (str) : the distanation or end of the path
Returns:
shortest path (tuple): List of vertices in the path and len
Empty list if path does not exist
"""
if from_vertex not in self.vert_dict or to_vertex not in self.vert_dict:
raise KeyError(
"One of the given vertices does not exist in graph!")
# check if you are at the location
if from_vertex == to_vertex:
vert_obj = self.vert_dict[from_vertex]
return ([vert_obj.data], 0)
# grab the start location from graph
current_vertex = self.vert_dict[from_vertex]
# initialize the queue, visited nodes set, a dictionary
# to keep track of parent
queue = Queue(maxsize=len(self.get_vertices()))
seen_vertex = set()
parent_pointers = {}
# start the traversal
queue.put(current_vertex)
seen_vertex.add(current_vertex.data)
path = []
path_found = False
parent = None
current_vertex.parent = parent
# alternative way of storing the references to parent pointers
parent_pointers[current_vertex.data] = None
while not queue.empty():
# dequeue the front element
current_vertex = queue.get()
path.append(current_vertex)
# check if we are at destination
if current_vertex.data == to_vertex:
path_found = True # found the goal
break
# otherwise
for neighbor in current_vertex.neighbors:
if neighbor.data not in seen_vertex:
queue.put(neighbor)
seen_vertex.add(neighbor.data)
neighbor.parent = current_vertex
parent_pointers[neighbor.data] = current_vertex.data
if path_found:
path = []
while current_vertex is not None:
path.append(current_vertex.data)
current_vertex = current_vertex.parent
return (path[::-1], len(path) - 1)
# if there is no path from source to destination return -1
return ([], -1)
queue = Queue()
#we have to traverse trough all nodes in the list
for node in adj_list.keys():
visited[node] = False
parent[node] = None
level[node] = -1
s = 'A'
visited[s] = True
level[s] = 0
queue.put(s)
while not queue.empty():
u = queue.get()
bfs_traversal_output.append(u)
for v in adj_list[u]:
if not visited[v]:
visited[v] = True
parent[v] = u
level[v] = level[u] + 1
queue.put(v)
print(bfs_traversal_output)
print(level['G'])
v = 'G'
path = []
while v is not None:
class DBMySQL(object):
"""
数据库操作基类
"""
def __init__(self, host, port, user, password, db, pool_size=5):
self._pool = Queue(maxsize=pool_size)
self._host = host
self._port = int(port)
self._user = user
self._password = password
self._db = db
self._connection = None
self.init_pool(pool_size)
def init_pool(self, pool_size):
for x in range(pool_size):
self.put_to_pool(self.create_connection())
def put_to_pool(self, conn):
try:
self._pool.put_nowait(conn)
except Exception as e:
raise Exception('put_to_pool error')
def create_connection(self):
connection = pymysql.connect(host=self._host,
port=self._port,
user=self._user,
password=self._password,
db=self._db,
autocommit=True,
charset='utf8',
cursorclass=pymysql.cursors.DictCursor)
return connection
@property
def connection(self):
self._connection = self._pool.get(True)
if not self.ping():
self._connection.connect()
return self._connection
def ping(self):
try:
return self._connection.query('SELECT 1')
except Exception:
return None
def query(self, sql, params_dict=None, is_fetchone=False):
"""
执行sql查询语句。如：select * from user where name = '{name}'
@param sql sql语句
@param params_dict 参数字典
@param is_fetchone 是否只返回一条记录
@return dict列表或dict
"""
result = None
connection = self.connection
with connection.cursor() as cursor:
try:
cursor.execute(sql, params_dict)
if is_fetchone:
result = cursor.fetchone()
else:
result = cursor.fetchall()
except Exception as e:
raise e
self.put_to_pool(connection)
return result
def execute(self, sql, params_dict=None):
"""
执行sql语句
@param sql:
@param params_dict:
@return:
"""
is_success = False
connection = self.connection
with connection.cursor() as cursor:
try:
# sql 格式INSERT applicant (from_type, name) VALUES (%(from_type)s, %(name)s)
# params_dict 格式 {'from_type': 1, 'name': 'yuri'}
cursor.execute(sql, params_dict)
is_success = True
connection.commit()
except Exception as e:
raise e
self.put_to_pool(connection)
return is_success
def close(self):
while not self._pool.empty():
self._pool.get(True).close()
# Ejercicio 602: Recorrer todos los elementos de una cola por medio de un ciclo while.
from queue import Queue
q = Queue()
for i in range(5):
q.put(i)
print(q.qsize())
print()
while not q.empty():
print(q.get(), end=' ')
class BpsControl99(Settings):
def __init__(self, userName, password):
Settings.__init__(self)
self.userName = userName
self.password = password
self.session = requests.session() # 实例化session，维持会话,可以让我们在跨请求时保存某些参数
self.__load()
self.q = Queue() # 多线程调用的函数不能用return返回值，用来保存返回值
self.engine1 = create_engine(
'mysql+mysqlconnector://{}:{}@{}:{}/{}'.format(
self.mysql1['user'], self.mysql1['password'],
self.mysql1['host'], self.mysql1['port'],
self.mysql1['datebase']))
self.engine2 = create_engine(
'mysql+mysqlconnector://{}:{}@{}:{}/{}'.format(
self.mysql2['user'], self.mysql2['password'],
self.mysql2['host'], self.mysql2['port'],
self.mysql2['datebase']))
self.engine3 = create_engine(
'mysql+mysqlconnector://{}:{}@{}:{}/{}'.format(
self.mysql3['user'], self.mysql3['password'],
self.mysql3['host'], self.mysql3['port'],
self.mysql3['datebase']))
def __load(self): # 登录系统保持会话状态
url = r'https://goms.giikin.com/admin/login/index.html'
data = {
'username': self.userName,
'password': self.password,
'remember': '1'
}
r_header = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36'
}
req = self.session.post(url=url, headers=r_header, data=data)
print('------ 成功登陆系统后台 -------')
def tgetOrderInfo(self, orderId, searchType): # 进入查询界面
url = 'https://goms.giikin.com/admin/order/orderquery.html'
data = {'phone': None, 'ship_email': None, 'ip': None}
if searchType == '订单号':
data.update({'order_number': orderId, 'waybill_number': None})
elif searchType == '运单号':
data.update({'order_number': None, 'waybill_number': orderId})
req = self.session.post(url=url, data=data)
print('-------已成功发送请求++++++')
orderInfo = self.t_parseDate(req) # 获取订单简单信息
# print(orderInfo)
def t_parseDate(self, req): # 对返回的response 进行处理； 处理订单简单信息
soup = BeautifulSoup(req.text, 'lxml') # 创建 beautifulsoup 对象
orderInfo = {}
labels = soup.find_all('th') # 获取行标签的th值
vals = soup.find_all('td') # 获取表格的td的值
if len(labels) > len(vals) or len(labels) < len(vals):
print('查询失败！！！')
else:
for i in range(len(labels)):
orderInfo[str(labels[i]).replace("<th>", "").replace(
"</th>",
"").strip()] = str(vals[i]).replace("<td>", "").replace(
"</td>", "").strip()
print('-------已处理订单简单信息---------')
try:
self.q.put(orderInfo)
except Exception as e:
print('放入失败---：', str(Exception) + str(e))
return orderInfo
def getNumberT(self, team, searchType, last_month): # ----主线程的执行（多线程函数）
match = {
'slgat': '港台',
'sltg': '泰国',
'slxmt': '新马',
'slzb': '直播团队',
'slyn': '越南',
'slrb': '日本'
}
print("======== 开始订单产品详情查询 ======")
month_begin = (datetime.datetime.now() -
relativedelta(months=4)).strftime('%Y-%m-%d')
start = datetime.datetime.now()
sql = '''SELECT id,`订单编号` FROM slgat_order_list sl
where sl.日期 ='{0}' and sl.是否改派 ='直发' and sl.币种 ='台湾';'''.format(last_month)
ordersDict = pd.read_sql_query(sql=sql, con=self.engine1)
print(ordersDict)
ordersDict = ordersDict['订单编号'].values.tolist()
print('获取耗时：', datetime.datetime.now() - start)
print('------正在查询单个订单的详情++++++')
print('主线程开始执行……………………')
threads = [] # 多线程用线程池--
for order in ordersDict: # 注意前后数组的取值长度一致
threads.append(
Thread(target=self.tgetOrderInfo,
args=(order, searchType))) # -----也即是子线程
print('子线程分配完成++++++')
if threads: # 当所有的线程都分配完成之后，通过调用每个线程的start()方法再让他们开始。
print(len(threads))
for th in threads:
th.start() # print ("开启子线程…………")
for th in threads:
th.join() # print ("退出子线程")
else:
print("没有需要运行子线程！！！")
print('主线程执行结束---------')
results = []
for i in range(len(ordersDict)): # print(i)
try:
results.append(self.q.get())
except Exception as e:
print('取出失败---：', str(Exception) + str(e))
print('-----执行结束---------')
print(' V ')
pf = pd.DataFrame(list(results)) # 将字典列表转换为DataFrame
pf = pf[[
'订单号', '订单状态', '物流单号', '商品名称', '是否二次改派', '数量', '币种', '下单时间', '电话',
'物流状态'
]]
print('正在写入缓存中......')
try:
pf.to_sql('备用', con=self.engine1, index=False, if_exists='replace')
sql = '''update slgat_order_list a, 备用 b
set a.`币种`=b.`币种`,
a.`数量`=b.`数量`,
a.`电话号码`=b.`电话` ,
a.`运单编号`=b.`物流单号`,
a.`系统订单状态`=b.`订单状态`,
a.`系统物流状态`=b.`物流状态`,
a.`是否改派`=b.`是否二次改派`
where a.`订单编号`=b.`订单号`;'''
pd.read_sql_query(sql=sql, con=self.engine1, chunksize=1000)
except Exception as e:
print('更新失败：', str(Exception) + str(e))
print('更新成功…………')
return ordersDict
class Takeover(Module):
"""
OneForAll多线程子域接管风险检查模块
Example:
python3 takeover.py --target www.example.com --format csv run
python3 takeover.py --target ./subdomains.txt --thread 10 run
Note:
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数dpath为None默认使用OneForAll结果目录
:param any target: 单个子域或者每行一个子域的文件路径(必需参数)
:param int thread: 线程数(默认100)
:param str format: 导出格式(默认csv)
:param str dpath: 导出目录(默认None)
"""
def __init__(self, target, thread=100, dpath=None, format='csv'):
Module.__init__(self)
self.subdomains = set()
self.module = 'Check'
self.source = 'Takeover'
self.target = target
self.thread = thread
self.dpath = dpath
self.format = format
self.fingerprints = None
self.subdomainq = Queue()
self.cnames = list()
self.results = Dataset()
def save(self):
logger.log('INFOR', '正在保存检查结果')
if self.format == 'txt':
data = str(self.results)
else:
data = self.results.export(self.format)
fpath = self.dpath.joinpath(f'takeover.{self.format}')
utils.save_data(fpath, data)
def compare(self, subdomain, cname, responses):
domain_resp = self.get('http://' + subdomain, check=False)
cname_resp = self.get('http://'+cname, check=False)
if domain_resp is None or cname_resp is None:
return
for resp in responses:
if resp in domain_resp.text and resp in cname_resp.text:
logger.log('ALERT', f'{subdomain}存在子域接管风险')
self.results.append([subdomain, cname])
break
def worker(self, subdomain):
cname = get_cname(subdomain)
if cname is None:
return
maindomain = get_maindomain(cname)
for fingerprint in self.fingerprints:
cnames = fingerprint.get('cname')
if maindomain not in cnames:
continue
responses = fingerprint.get('response')
self.compare(subdomain, cname, responses)
def check(self):
while not self.subdomainq.empty(): # 保证域名队列遍历结束后能退出线程
subdomain = self.subdomainq.get() # 从队列中获取域名
self.worker(subdomain)
self.subdomainq.task_done()
def progress(self):
# 设置进度
bar = tqdm()
bar.total = len(self.subdomains)
bar.desc = 'Progress'
bar.ncols = True
while True:
done = bar.total - self.subdomainq.qsize()
bar.n = done
bar.update()
if done == bar.total: # 完成队列中所有子域的检查退出
break
bar.close()
def run(self):
start = time.time()
logger.log('INFOR', f'开始执行{self.source}模块')
self.format = utils.check_format(self.format)
self.dpath = utils.check_dpath(self.dpath)
self.subdomains = utils.get_domains(self.target)
if self.subdomains:
logger.log('INFOR', f'正在检查子域接管风险')
self.fingerprints = get_fingerprint()
self.results.headers = ['subdomain', 'cname']
# 创建待检查的子域队列
for domain in self.subdomains:
self.subdomainq.put(domain)
# 检查线程
for _ in range(self.thread):
check_thread = Thread(target=self.check, daemon=True)
check_thread.start()
# 进度线程
progress_thread = Thread(target=self.progress, daemon=True)
progress_thread.start()
self.subdomainq.join()
self.save()
else:
logger.log('FATAL', f'获取域名失败')
end = time.time()
elapsed = round(end - start, 1)
logger.log('INFOR', f'{self.source}模块耗时{elapsed}秒'
f'发现{len(self.results)}个子域存在接管风险')
logger.log('DEBUG', f'结束执行{self.source}模块')
class NodeClient:
def __init__(self,
node_pem_cert: bytes,
node_pem_key: bytes,
host: str,
network_name: str,
receive_timeout: int,
network_id: str = DEFAULT_NETWORK_ID):
self.node_pem_cert = node_pem_cert
self.node_pem_key = node_pem_key
self.ec_key = load_pem_private_key(self.node_pem_key,
password=None,
backend=default_backend())
self.network_id = network_id
self.host = host
self.tcp_port = 0
self.udp_port = get_free_tcp_port()
self.network_name = network_name # docker network name
self._receive_timeout = receive_timeout
self.return_queue = Queue() # type: ignore
self.server = self._start_transport_server()
@property
def node_pb(self) -> Node:
node_id = get_node_id_raw(self.ec_key)
return Node(id=node_id,
host=self.host.encode('utf8'),
tcp_port=self.tcp_port,
udp_port=self.udp_port)
@property
def header_pb(self) -> Header:
return Header(sender=self.node_pb, networkId=self.network_id)
def get_peer_node_ip(self, rnode: RNode) -> str:
return rnode.get_peer_node_ip(self.network_name)
def _start_transport_server(self) -> grpc.Server:
server_credential = grpc.ssl_server_credentials([(self.node_pem_key,
self.node_pem_cert)])
server = grpc.server(futures.ThreadPoolExecutor())
add_TransportLayerServicer_to_server(
TransportServer(self.node_pb, self.network_id, self.return_queue),
server)
self.tcp_port = server.add_secure_port("{}:0".format(self.host),
server_credential)
server.start()
return server
def block_request(self, block_hash: str, rnode: RNode) -> BlockMessage:
block_request = BlockRequest(hash=bytes.fromhex(block_hash))
request_msg_packet = Packet(typeId="BlockRequest",
content=block_request.SerializeToString())
protocol = Protocol(header=self.header_pb, packet=request_msg_packet)
request = TLRequest(protocol=protocol)
self.send_request(request, rnode)
try:
return self.return_queue.get(timeout=self._receive_timeout)
except Empty as e:
raise BlockNotFound(block_hash, rnode) from e
def send_request(self, request: TLRequest, rnode: RNode) -> None:
credential = grpc.ssl_channel_credentials(rnode.get_node_pem_cert(),
self.node_pem_key,
self.node_pem_cert)
# only linux system can connect to the docker container through the container name
rnode_ip = self.get_peer_node_ip(rnode)
channel = grpc.secure_channel(
"{}:{}".format(rnode_ip, DEFAULT_TRANSPORT_SERVER_PORT),
credential,
options=(('grpc.ssl_target_name_override',
get_node_id_str(
load_pem_private_key(rnode.get_node_pem_key(), None,
default_backend()))), ))
try:
stub = TransportLayerStub(channel)
stub.Send(request)
finally:
channel.close()
def send_block(self, block: BlockMessage, rnode: RNode) -> None:
block_msg_packet = Packet(typeId="BlockMessage",
content=block.SerializeToString())
protocol = Protocol(header=self.header_pb, packet=block_msg_packet)
request = TLRequest(protocol=protocol)
self.send_request(request, rnode)
def stop(self) -> None:
self.server.stop(0)
class USBCreator(object):
def __init__(self):
# Load window and widgets
self.scriptName = basename(__file__)
self.scriptDir = abspath(dirname(__file__))
self.mediaDir = join(self.scriptDir, '../../share/usb-creator')
self.builder = Gtk.Builder()
self.builder.add_from_file(join(self.mediaDir, 'usb-creator.glade'))
# Main window objects
go = self.builder.get_object
self.window = go("usb-creator")
self.lblDevice = go("lblDevice")
self.lblIso = go("lblIso")
self.lblAvailable = go("lblAvailable")
self.lblRequired = go("lblRequired")
self.cmbDevice = go("cmbDevice")
self.cmbDeviceHandler = ComboBoxHandler(self.cmbDevice)
self.txtIso = go("txtIso")
self.btnRefresh = go("btnRefresh")
self.btnUnmount = go("btnUnmount")
self.btnBrowseIso = go("btnBrowseIso")
self.btnClear = go("btnClear")
self.chkFormatDevice = go("chkFormatDevice")
self.chkRepairDevice = go("chkRepairDevice")
self.btnExecute = go("btnExecute")
self.lblUsb = go("lblUsb")
self.tvUsbIsos = go("tvUsbIsos")
self.btnDelete = go("btnDelete")
self.pbUsbCreator = go("pbUsbCreator")
self.statusbar = go("statusbar")
# Translations
self.window.set_title(_("USB Creator"))
self.lblDevice.set_label(_("Device"))
self.lblUsb.set_label(_("USB"))
self.available_text = _("Available")
self.required_text = _("Required")
self.chkFormatDevice.set_label(_("Format device"))
self.chkFormatDevice.set_tooltip_text(
_("Warning: all data will be lost"))
self.chkRepairDevice.set_label(_("Repair device"))
self.chkRepairDevice.set_tooltip_text(
_("Tries to repair an unbootable USB"))
self.btnExecute.set_label("_{}".format(_("Execute")))
self.lblIso.set_label(_("ISO"))
self.btnDelete.set_label("_{}".format(_("Delete")))
self.btnRefresh.set_tooltip_text(_("Refresh device list"))
self.btnUnmount.set_tooltip_text(_("Unmount device"))
self.btnBrowseIso.set_tooltip_text(_("Browse for ISO file"))
self.btnClear.set_tooltip_text(_("Clear the ISO field"))
# Log lines to show: check string, percent done (0=pulse, appends last word in log line), show line (translatable)
self.log_lines = []
self.log_lines.append(
["partitioning usb", 5,
_("Partitioning USB...")])
self.log_lines.append(
["searching for bad blocks", 0,
_("Searching for bad block")])
self.log_lines.append(["installing", 15, _("Installing Grub...")])
self.log_lines.append(["rsync", 25, _("Start copying ISO...")])
self.log_lines.append(["left to copy", 0, _("kB left to copy:")])
self.log_lines.append(["check hash", 85, _("Check hash of ISO...")])
# Initiate variables
self.devices = []
self.device = {}
self.device['path'] = ''
self.device['size'] = 0
self.device['has_partition'] = False
self.device['mount'] = ''
self.device['available'] = 0
self.device["new_iso"] = ''
self.device["new_iso_required"] = 0
self.logos = self.get_logos()
self.queue = Queue(-1)
self.threads = {}
self.htmlDir = join(self.mediaDir, "html")
self.helpFile = join(self.get_language_dir(), "help.html")
log = getoutput(
"cat /usr/bin/usb-creator | grep 'LOG=' | cut -d'=' -f 2")
self.log_file = log[0]
self.log = Logger(self.log_file, addLogTime=False, maxSizeKB=5120)
self.tvUsbIsosHandler = TreeViewHandler(self.tvUsbIsos)
self.lblAvailable.set_label('')
self.lblRequired.set_label('')
# Connect builder signals and show window
self.builder.connect_signals(self)
self.window.show_all()
# Get attached devices
self.on_btnRefresh_clicked()
# Init log
init_log = ">>> Start USB Creator: {} <<<".format(datetime.now())
self.log.write(init_log)
# Version information
self.version_text = _("Version")
self.pck_version = getPackageVersion('usb-creator')
self.set_statusbar_message("{}: {}".format(self.version_text,
self.pck_version))
# ===============================================
# Main window functions
# ===============================================
def on_btnExecute_clicked(self, widget):
if exists(self.device["path"]):
arguments = []
arguments.append("-d {}".format(self.device["path"]))
clear = self.chkFormatDevice.get_active()
repair = self.chkRepairDevice.get_active()
iso = self.device["new_iso"]
iso_path = self.txtIso.get_text().strip()
# ISO path does not exist
if iso != iso_path:
msg = _(
"Cannot add ISO from path: {}.\n"
"Please, remove the ISO path or browse for an existing ISO."
)
WarningDialog(self.btnExecute.get_label(),
msg.format(iso_path))
return True
# Check if there is enough space
available = self.device["available"]
if self.chkFormatDevice.get_active():
available = self.device["size"]
if available - self.device["new_iso_required"] < 0:
msg = _(
"There is not enough space available on the pen drive.\n"
"Please, remove unneeded files before continuing.")
WarningDialog(self.btnExecute.get_label(), msg)
return True
if clear:
arguments.append("-f")
arguments.append("-b")
if repair:
arguments.append("-r")
arguments.append("-b")
arguments.append("-g")
# This should use the history file to get the original hash
arguments.append("-s")
if exists(iso):
arguments.append("-i \"{}\"".format(iso))
arguments.append("-s")
cmd = "usb-creator {}".format(" ".join(arguments))
self.log.write("Execute command: {}".format(cmd))
self.exec_command(cmd)
def on_btnDelete_clicked(self, widget):
selected_isos = self.tvUsbIsosHandler.getToggledValues(toggleColNr=0,
valueColNr=2)
if selected_isos:
msg = _(
"Are you sure you want to remove the selected ISO from the device?"
)
answer = QuestionDialog(self.btnDelete.get_label(), msg)
if answer:
for iso in selected_isos:
iso_path = join(self.device["mount"], iso)
if exists(iso_path):
os.remove(iso_path)
self.log.write("Remove ISO: {}".format(iso_path))
shell_exec("usb-creator -d {} -g".format(self.device["path"]))
self.on_cmbDevice_changed()
self.fill_treeview_usbcreator(self.device["mount"])
def on_btnBrowseIso_clicked(self, widget):
file_filter = Gtk.FileFilter()
file_filter.set_name("ISO")
file_filter.add_mime_type("application/x-cd-image")
file_filter.add_pattern("*.iso")
start_dir = dirname(self.txtIso.get_text().strip())
if not exists(start_dir):
start_dir = expanduser("~")
iso = SelectFileDialog(title=_('Select ISO'),
start_directory=start_dir,
gtkFileFilter=file_filter).show()
if iso is not None:
self.log.write("Add ISO: {}".format(iso))
self.txtIso.set_text(iso)
def on_btnClear_clicked(self, widget):
self.txtIso.set_text('')
def on_txtIso_changed(self, widget=None):
iso_path = self.txtIso.get_text().strip()
if exists(iso_path):
if isdir(iso_path):
isos = glob(join(iso_path, '*.iso'))
if isos:
required = 0
for iso in isos:
# Check if these ISOs overwrite current USB ISOs
check_usb_iso_size = 0
if not self.chkFormatDevice.get_active():
check_usb_iso = join(self.device["mount"],
basename(iso))
if exists(check_usb_iso):
check_usb_iso_size = self.get_iso_size(
check_usb_iso)
required += (self.get_iso_size(iso) -
check_usb_iso_size)
if required < 0:
required = 0
self.lblRequired.set_label("{}: {} MB".format(
self.required_text, int(required / 1024)))
# Save the info
self.device["new_iso"] = iso_path
self.device["new_iso_required"] = required
self.log.write("New ISO directory: {}, {}".format(
iso_path, required))
else:
self.device["new_iso"] = ''
self.device["new_iso_required"] = 0
self.log.write(
"New ISO directory does not contain ISOs: {}".format(
iso_path))
else:
# Check if this ISO overwrites current USB ISO
check_usb_iso_size = 0
if not self.chkFormatDevice.get_active():
check_usb_iso = join(self.device["mount"],
basename(iso_path))
if exists(check_usb_iso):
check_usb_iso_size = self.get_iso_size(check_usb_iso)
required = (self.get_iso_size(iso_path) - check_usb_iso_size)
self.lblRequired.set_label("{}: {} MB".format(
self.required_text, int(required / 1024)))
# Save the info
self.device["new_iso"] = iso_path
self.device["new_iso_required"] = required
self.log.write("New ISO: {}, {}".format(iso_path, required))
else:
self.device["new_iso"] = ''
self.device["new_iso_required"] = 0
self.lblRequired.set_text('')
def on_btnRefresh_clicked(self, widget=None):
self.devices = self.get_devices()
self.cmbDeviceHandler.fillComboBox(self.devices, 0)
def on_btnUnmount_clicked(self, widget):
unmount_text = _("Unmount")
device = self.device["path"]
self.unmount_device(device)
self.on_btnRefresh_clicked()
if device in self.devices:
msg = _("Could not unmount the device.\n"
"Please unmount the device manually.")
else:
msg = _("You can now safely remove the device.")
MessageDialog(unmount_text, msg)
def on_cmbDevice_changed(self, widget=None):
device = self.cmbDeviceHandler.getValue()
if device is not None:
mount = ''
size = 0
available = 0
# Get the size of the USB
usb_size = getoutput(
"env LANG=C udisks --show-info {} | grep size".format(device))
if usb_size:
# udisks returns bytes, while df returns kbs
size = int(int(usb_size[0].split(":")[1].strip()) / 1024)
# Assume that the USB is empty (will check later)
available = size
# Get free size on USB
has_partition = self.device_has_partition(device)
if has_partition:
mount = self.get_device_mount(device)
# This function can be called from on_chkFormatDevice_toggled
if widget != self.chkFormatDevice:
self.chkFormatDevice.set_sensitive(True)
self.chkFormatDevice.set_active(False)
free_size = getoutput(
"df --output=avail {}1 | awk 'NR==2'".format(device))
if free_size:
available = int(free_size[0])
else:
self.chkFormatDevice.set_active(True)
self.chkFormatDevice.set_sensitive(False)
self.chkRepairDevice.set_active(False)
self.fill_treeview_usbcreator(mount)
self.lblAvailable.set_label("{}: {} MB".format(
self.available_text, int(available / 1024)))
# Save the info
self.device['path'] = device
self.device['size'] = size
self.device['has_partition'] = has_partition
self.device['mount'] = mount
self.device['available'] = available
self.log.write("Selected device info: {}".format(self.device))
# Update info
iso_path = self.txtIso.get_text().strip()
if iso_path != "" and exists(iso_path):
self.on_txtIso_changed()
else:
self.fill_treeview_usbcreator()
self.lblAvailable.set_label('')
self.lblRequired.set_label('')
self.txtIso.set_text('')
self.device['path'] = ''
self.device['size'] = 0
self.device['has_partition'] = False
self.device['mount'] = ''
self.device['available'] = 0
self.device["new_iso"] = ''
self.device["new_iso_required"] = 0
def on_chkFormatDevice_toggled(self, widget):
# Recalculate available space and requied space
self.on_cmbDevice_changed(widget)
def on_btnHelp_clicked(self, widget):
# Open the help file as the real user (not root)
shell_exec("%s/open-as-user \"%s\"" % (self.scriptDir, self.helpFile))
def fill_treeview_usbcreator(self, mount=''):
isos_list = []
# columns: checkbox, image (logo), device, driver
column_types = ['bool', 'GdkPixbuf.Pixbuf', 'str', 'str']
if exists(mount):
isos = glob(join(mount, '*.iso'))
for iso in isos:
iso_name = basename(iso)
iso_name_lower = iso_name.lower()
iso_size = "{} MB".format(int(self.get_iso_size(iso) / 1024))
iso_logo = ""
for key, logo in list(self.logos.items()):
if key != "iso":
if key in iso_name_lower:
if len(logo) > len(iso_logo):
iso_logo = logo
if iso_logo == "":
iso_logo = self.logos["iso"]
self.log.write("ISO on {}: {}, {}, {}".format(
mount, iso_name, iso_size, iso_logo))
isos_list.append([False, iso_logo, iso_name, iso_size])
# Fill treeview
self.tvUsbIsosHandler.fillTreeview(contentList=isos_list,
columnTypesList=column_types)
def exec_command(self, command):
try:
# Run the command in a separate thread
self.set_buttons_state(False)
name = 'cmd'
t = ExecuteThreadedCommands([command], self.queue)
self.threads[name] = t
t.daemon = True
t.start()
self.queue.join()
GObject.timeout_add(1000, self.check_thread, name)
except Exception as detail:
ErrorDialog(self.btnExecute.get_label(), detail)
def check_thread(self, name):
if self.threads[name].is_alive():
self.set_progress()
if not self.queue.empty():
ret = self.queue.get()
self.log.write("Queue returns: {}".format(ret), 'check_thread')
self.queue.task_done()
self.show_message(ret)
return True
# Thread is done
self.log.write(">> Thread is done", 'check_thread')
if not self.queue.empty():
ret = self.queue.get()
self.queue.task_done()
self.show_message(ret)
del self.threads[name]
self.set_buttons_state(True)
self.on_cmbDevice_changed()
self.fill_treeview_usbcreator(self.device["mount"])
self.set_statusbar_message("{}: {}".format(self.version_text,
self.pck_version))
return False
def set_buttons_state(self, enable):
if not enable:
# Disable buttons
self.btnExecute.set_sensitive(False)
self.btnDelete.set_sensitive(False)
self.btnBrowseIso.set_sensitive(False)
self.btnRefresh.set_sensitive(False)
self.btnUnmount.set_sensitive(False)
self.btnClear.set_sensitive(False)
self.chkFormatDevice.set_sensitive(False)
self.chkRepairDevice.set_sensitive(False)
self.cmbDevice.set_sensitive(False)
self.txtIso.set_sensitive(False)
else:
# Enable buttons and reset progress bar
self.btnExecute.set_sensitive(True)
self.btnDelete.set_sensitive(True)
self.btnBrowseIso.set_sensitive(True)
self.btnRefresh.set_sensitive(True)
self.btnUnmount.set_sensitive(True)
self.btnClear.set_sensitive(True)
self.chkFormatDevice.set_sensitive(True)
self.chkRepairDevice.set_sensitive(True)
self.cmbDevice.set_sensitive(True)
self.txtIso.set_sensitive(True)
self.pbUsbCreator.set_fraction(0)
def get_logos(self):
logos_dict = {}
logos_path = join(self.mediaDir, 'logos')
logos = glob(join(logos_path, '*.png'))
for logo in logos:
key = splitext(basename(logo))[0]
logos_dict[key] = logo
return logos_dict
def set_progress(self):
if exists(self.log_file):
msg = ''
last_line = getoutput(
"tail -50 {} | grep -v DEBUG | grep -v ==".format(
self.log_file))
for line in reversed(last_line):
# Check for session start line: that is the last line to check
if ">>>>>" in line and "<<<<<" in line:
break
for chk_line in self.log_lines:
if chk_line[0] in line.lower():
#print((line))
word = ''
if chk_line[1] == 0:
self.pbUsbCreator.pulse()
words = line.split(' ')
for word in reversed(words):
if word.strip() != '':
break
else:
self.pbUsbCreator.set_fraction(
float(chk_line[1] / 100))
msg = "{} {}".format(chk_line[2], word)
break
if msg != '':
break
self.set_statusbar_message(msg)
def set_statusbar_message(self, message):
if message is not None:
context = self.statusbar.get_context_id('message')
self.statusbar.push(context, message)
def get_devices(self):
devices = []
my_devices = getoutput(
"udisks --enumerate-device-files | egrep '/dev/sd[a-z]$'")
for device in my_devices:
info = getoutput("env LANG=C udisks --show-info {}".format(device))
detachable = False
has_partition = False
for line in info:
if "detachable" in line and "1" in line:
detachable = True
elif "partition" in line:
has_partition = True
if detachable and has_partition:
devices.append(device)
break
devices.sort()
return devices
def device_has_partition(self, device):
part_count = getoutput(
"udisks --show-info {} | grep count | grep -v block".format(
device))
if part_count:
if "1" in part_count[0]:
return True
return False
def get_device_mount(self, device):
shell_exec("udisks --mount {}1".format(device))
mount = getoutput(
"grep %s1 /etc/mtab | awk '{print $2}' | sed 's/\\040/ /g'" %
device)
if mount:
return mount[0]
return ''
def get_iso_size(self, iso):
iso_size = getoutput("du -Lk \"%s\" | awk '{print $1}'" % iso)
if iso_size:
return int(iso_size[0])
return 0
def unmount_device(self, device):
shell_exec("udisks --unmount {}1".format(device))
shell_exec("udisks --detach {}".format(device))
# Close the gui
def on_usbcreator_destroy(self, widget):
# Unmount devices
for device in self.devices:
if self.get_device_mount(device) != "":
self.unmount_device(device)
# Close the app
Gtk.main_quit()
def show_message(self, cmdOutput):
try:
self.log.write("Command output: {}".format(cmdOutput),
'show_message')
ret = int(cmdOutput)
if ret > 1 and ret != 255:
if ret == 1:
ErrorDialog(
self.btnExecute.get_label(),
_("Run this application with root permission."))
elif ret == 2:
ErrorDialog(
self.btnExecute.get_label(),
_("Wrong arguments were passed to usb-creator."))
elif ret == 3:
ErrorDialog(
self.btnExecute.get_label(),
_("The device was not found or no device was given."))
elif ret == 4:
ErrorDialog(self.btnExecute.get_label(),
_("Given ISO path was not found."))
elif ret == 5:
ErrorDialog(self.btnExecute.get_label(),
_("Device is in use by another application."))
elif ret == 6:
ErrorDialog(self.btnExecute.get_label(),
_("Unable to mount the device."))
elif ret == 7:
ErrorDialog(self.btnExecute.get_label(),
_("Hash mismatch."))
elif ret == 8:
ErrorDialog(self.btnExecute.get_label(),
_("The device has no fat32 partition."))
elif ret == 9:
ErrorDialog(self.btnExecute.get_label(),
_("The device has no bootloader installed."))
elif ret == 10:
ErrorDialog(
self.btnExecute.get_label(),
_("There is not enough space available on the device.")
)
elif ret == 11:
ErrorDialog(
self.btnExecute.get_label(),
_("Unable to guess distribution from ISO name.\n"
"Make sure you have the distribution name in the ISO name."
))
else:
msg = _(
"An unknown error accured.\n"
"Please, visit our forum for support: http://forums.solydxk.com"
)
ErrorDialog(self.window.get_title(), msg)
else:
msg = _("The USB was successfully written.")
MessageDialog(self.window.get_title(), msg)
except:
ErrorDialog(self.btnExecute.get_label(), cmdOutput)
# ===============================================
# Language specific functions
# ===============================================
def get_language_dir(self):
# First test if full locale directory exists, e.g. html/pt_BR,
# otherwise perhaps at least the language is there, e.g. html/pt
# and if that doesn't work, try html/pt_PT
lang = self.get_current_language()
path = join(self.htmlDir, lang)
if not isdir(path):
base_lang = lang.split('_')[0].lower()
path = join(self.htmlDir, base_lang)
if not isdir(path):
path = join(self.htmlDir,
"{}_{}".format(base_lang, base_lang.upper()))
if not isdir(path):
path = join(self.htmlDir, 'en')
return path
def get_current_language(self):
lang = os.environ.get('LANG', 'US').split('.')[0]
if lang == '':
lang = 'en'
return lang
class LoopClosing(object):
def __init__(self, system, params):
self.system = system
self.params = params
self.loop_detector = LoopDetection(params)
self.optimizer = PoseGraphOptimization()
self.loops = []
self.stopped = False
self._queue = Queue()
self.maintenance_thread = Thread(target=self.maintenance)
self.maintenance_thread.start()
def stop(self):
self.stopped = True
self._queue.put(None)
self.maintenance_thread.join()
print('loop closing stopped')
def add_keyframe(self, keyframe):
self._queue.put(keyframe)
self.loop_detector.add_keyframe(keyframe)
def add_keyframes(self, keyframes):
for kf in keyframes:
self.add_keyframe(kf)
def maintenance(self):
last_query_keyframe = None
while not self.stopped:
keyframe = self._queue.get()
if keyframe is None or self.stopped:
return
# check if this keyframe share many mappoints with a loop keyframe
covisible = sorted(keyframe.covisibility_keyframes().items(),
key=lambda _: _[1],
reverse=True)
if any([(keyframe.id - _[0].id) > 5 for _ in covisible[:2]]):
continue
if (last_query_keyframe is not None
and abs(last_query_keyframe.id - keyframe.id) < 3):
continue
candidate = self.loop_detector.detect(keyframe)
if candidate is None:
continue
query_keyframe = keyframe
match_keyframe = candidate
result = match_and_estimate(query_keyframe, match_keyframe,
self.params)
if result is None:
continue
if (result.n_inliers < max(
self.params.lc_inliers_threshold,
result.n_matches * self.params.lc_inliers_ratio)):
continue
if (np.abs(result.correction.translation()).max() >
self.params.lc_distance_threshold):
continue
self.loops.append(
(match_keyframe, query_keyframe, result.constraint))
query_keyframe.set_loop(match_keyframe, result.constraint)
# We have to ensure that the mapping thread is on a safe part of code,
# before the selection of KFs to optimize
safe_window = self.system.mapping.lock_window() # set
safe_window.add(self.system.reference)
for kf in self.system.reference.covisibility_keyframes():
safe_window.add(kf)
# The safe window established between the Local Mapping must be
# inside the considered KFs.
considered_keyframes = self.system.graph.keyframes()
self.optimizer.set_data(considered_keyframes, self.loops)
before_lc = [
g2o.Isometry3d(kf.orientation, kf.position)
for kf in safe_window
]
# Propagate initial estimate through 10% of total keyframes
# (or at least 20 keyframes)
d = max(20, len(considered_keyframes) * 0.1)
propagator = SmoothEstimatePropagator(self.optimizer, d)
propagator.propagate(self.optimizer.vertex(match_keyframe.id))
# self.optimizer.set_verbose(True)
self.optimizer.optimize(20)
# Exclude KFs that may being use by the local BA.
self.optimizer.update_poses_and_points(considered_keyframes,
exclude=safe_window)
self.system.stop_adding_keyframes()
# Wait until mapper flushes everything to the map
self.system.mapping.wait_until_empty_queue()
while self.system.mapping.is_processing():
time.sleep(1e-4)
# Calculating optimization introduced by local mapping while loop was been closed
for i, kf in enumerate(safe_window):
after_lc = g2o.Isometry3d(kf.orientation, kf.position)
corr = before_lc[i].inverse() * after_lc
vertex = self.optimizer.vertex(kf.id)
vertex.set_estimate(vertex.estimate() * corr)
self.system.pause()
for keyframe in considered_keyframes[::-1]:
if keyframe in safe_window:
reference = keyframe
break
uncorrected = g2o.Isometry3d(reference.orientation,
reference.position)
corrected = self.optimizer.vertex(reference.id).estimate()
T = uncorrected.inverse() * corrected # close to result.correction
# We need to wait for the end of the current frame tracking and ensure that we
# won't interfere with the tracker.
while self.system.is_tracking():
time.sleep(1e-4)
self.system.set_loop_correction(T)
# Updating keyframes and map points on the lba zone
self.optimizer.update_poses_and_points(safe_window)
# keyframes after loop closing
keyframes = self.system.graph.keyframes()
if len(keyframes) > len(considered_keyframes):
self.optimizer.update_poses_and_points(
keyframes[len(considered_keyframes) - len(keyframes):],
correction=T)
for query_meas, match_meas in result.shared_measurements:
new_query_meas = Measurement(query_meas.type,
Measurement.Source.REFIND,
query_meas.get_keypoints(),
query_meas.get_descriptors())
self.system.graph.add_measurement(query_keyframe,
match_meas.mappoint,
new_query_meas)
new_match_meas = Measurement(match_meas.type,
Measurement.Source.REFIND,
match_meas.get_keypoints(),
match_meas.get_descriptors())
self.system.graph.add_measurement(match_keyframe,
query_meas.mappoint,
new_match_meas)
self.system.mapping.free_window()
self.system.resume_adding_keyframes()
self.system.unpause()
while not self._queue.empty():
keyframe = self._queue.get()
if keyframe is None:
return
last_query_keyframe = query_keyframe
class KeyClipWriter:
def __init__(self, bufSize=64, timeout=1.0):
# store the maximum buffer size of frames to be kept
# in memory along with the sleep timeout during threading
self.bufSize = bufSize
self.timeout = timeout
# initialize the buffer of frames, queue of frames that
# need to be written to file, video writer, writer thread,
# and boolean indicating whether recording has started or not
self.frames = deque(maxlen=bufSize)
self.Q = None
self.writer = None
self.thread = None
self.recording = False
def update(self, frame):
# update the frames buffer
self.frames.appendleft(frame)
# if we are recording, update the queue as well
if self.recording:
self.Q.put(frame)
def start(self, outputPath, fourcc, fps):
# indicate that we are recording, start the video writer,
# and initialize the queue of frames that need to be written to the video file
self.recording = True
self.writer = cv2.VideoWriter(
outputPath, fourcc, fps,
(self.frames[0].shape[1], self.frames[0].shape[0]), True)
self.Q = Queue()
# loop over the frames in the deque structure and add them to the queue
for i in range(len(self.frames), 0, -1):
self.Q.put(self.frames[i - 1])
# start a thread write frames to the video file
self.thread = Thread(target=self.write, args=())
self.thread.daemon = True
self.thread.start()
def write(self):
# keep looping
while True:
# if we are done recording, exit the thread
if not self.recording:
return
# check to see if there are entries in the queue
if not self.Q.empty():
# grab the next frame in the queue and write it to the video file
frame = self.Q.get()
self.writer.write(frame)
# otherwise, the queue is empty, so sleep for a bit so we don't waste CPU cycles
else:
time.sleep(self.timeout)
def flush(self):
# empty the queue by flushing all remaining frames to file
while not self.Q.empty():
frame = self.Q.get()
self.writer.write(frame)
def finish(self):
# indicate that we are done recording, join the thread,
# flush all remaining frames in the queue to file, and release the writer pointer
self.recording = False
self.thread.join()
self.flush()
self.writer.release()
class Agent:
def __init__(self, ip_address, port_no, view_size):
self.TCP_Socket = TCPSocketManager(ip_address, port_no, view_size)
self.view_size = view_size
self.view_window = [[' '] * view_size for i in range(view_size)]
self.global_map_size = 160
self.global_map_obstacles = [['?'] * self.global_map_size
for i in range(self.global_map_size)]
self.global_map_distance = [[0] * self.global_map_size
for i in range(self.global_map_size)]
self.top_global_map_scores = [[0] * self.global_map_size
for i in range(self.global_map_size)]
self.deep_global_map_scores = [[{}] * self.global_map_size
for i in range(self.global_map_size)]
self.AstarGlobalMap = AstarMap(self.global_map_obstacles)
self.items = {'k': 0, 'o': 0, 'a': 0, 'r': 0, '$': 0}
self.visibleItemsPts = {
'k': 100,
'o': 100,
'a': 100,
'T': 100,
'$': 10000
}
self.direction = 0
self.position = [80, 80]
self.home = [80, 80]
self.onRaft = 0
self.onLand = 1
self.actionQueue = Queue()
self.rotationAngleTbl = {
(-1, 0): 0,
(0, -1): 90,
(1, 0): 180,
(0, 1): 270
}
self.placedStonesCoordinates = []
self.visibleCoordinates = []
self.visibleItems = []
self.exploredCoordinates = [[80, 80]]
self.deep_search_limit = 20
self.max_layer_scores = [0] * (self.deep_search_limit + 1)
self.update_global_values_flag = 1
self.theFinalPath = []
#------------------------------------------------------------------------------------#
# checkDeepMapValues() #
# This function checks if future global map state Scores have already been #
# calculated and saved in self.deep_global_map_scores #
#------------------------------------------------------------------------------------#
def checkDeepMapValues(self, start, goal, stoneLocation, currentItems):
x = start[0]
y = start[1]
goal = str(goal)
stoneLocation = str(stoneLocation)
currentItems = str(currentItems)
if goal in self.deep_global_map_scores[x][y]:
if stoneLocation in self.deep_global_map_scores[x][y][goal]:
if currentItems in self.deep_global_map_scores[x][y][goal][
stoneLocation]:
deepMapValue = self.deep_global_map_scores[x][y][goal][
stoneLocation][currentItems]
return deepMapValue
return 0
#------------------------------------------------------------------------------------#
# addDeepMapValues() #
# This function adds future global map state Scores to self.deep_global_map_scores #
# in order to avoid repeitive calculations. #
#------------------------------------------------------------------------------------#
def addDeepMapValues(self, start, goal, stoneLocation, currentItems,
deepMapValue):
x = start[0]
y = start[1]
goal = str(goal)
stoneLocation = str(stoneLocation)
currentItems = str(currentItems)
if deepMapValue == 0:
deepMapValue = 1
if goal not in self.deep_global_map_scores[x][y]:
self.deep_global_map_scores[x][y][goal] = {}
if stoneLocation not in self.deep_global_map_scores[x][y][goal]:
self.deep_global_map_scores[x][y][goal][stoneLocation] = {}
self.deep_global_map_scores[x][y][goal][stoneLocation][
currentItems] = deepMapValue
#------------------------------------------------------------------------------------#
# decideNextPosition() #
# This function returns the next position for the agent by finding visible #
# coordinate with the highest Scores stored.I f the agent holds the treasure and a #
# route homeis feasible, then it immediately goes to [80,80] #
#------------------------------------------------------------------------------------#
def decideNextPosition(self):
nextPositions = [[80, 80]]
maxscores = -1000
maxcoordScore = -1000
goal_list = []
# If agent holds the treasure, then it searches for a path home.
if self.items['$'] > 0:
if self.decideRouteHomefromTreasure(self.position):
return nextPositions
# Loops through all possible coordinates and choose the coordinate with the highest Scores.
for coordinate in self.visibleCoordinates:
i = coordinate[0]
j = coordinate[1]
coordScores = self.top_global_map_scores[i][j]
distanceScores = self.global_map_distance[i][j] * 0.1
totalScores = coordScores - distanceScores
obstacle = self.global_map_obstacles[i][j]
if totalScores > maxscores:
maxscores = totalScores
nextPositions = [[i, j]]
nextObstacle = obstacle
maxcoordScore = coordScores
elif maxcoordScore == coordScores and nextObstacle in self.visibleItemsPts and nextObstacle == obstacle:
nextPositions.append([i, j])
return nextPositions
#------------------------------------------------------------------------------------#
# decideRouteHomefromTreasure() #
# This function checks if there is a path home from the current starting position #
#------------------------------------------------------------------------------------#
def decideRouteHomefromTreasure(self, starting_coordinate):
astarresult = self.astarMinimum(self.AstarGlobalMap,
starting_coordinate, self.home,
self.items, 0)
if len(astarresult[0]) > 0:
return 1
return 0
#------------------------------------------------------------------------------------#
# checkRouteHome() #
# This function checks if there is a path home from the current starting position #
#------------------------------------------------------------------------------------#
def checkRouteHome(self, starting_coordinate, treasure_coordinate):
# Calculates the route to the treasure using the starting and treasure coordinates
routeToTreasure = self.astarMinimum(self.AstarGlobalMap,
starting_coordinate,
treasure_coordinate, self.items,
self.onRaft)
# Generates list of remaining items and stone used during the routetoTreasure
remaining_items, stone_coordinates = routeToTreasure[
2], routeToTreasure[4]
# Updates a temporary globalmap with the stones used during the routetoTreasure
temp_globalmap_2 = copy.deepcopy(self.global_map_obstacles)
temp_AstarMap = AstarMap(temp_globalmap_2)
for coordinates in stone_coordinates:
temp_globalmap_2[coordinates[0]][coordinates[1]] = ' '
# If route to treasure exists then the agent determines if there is a route from the treasure to home.
if len(routeToTreasure[0]) > 0:
routeHome = self.astarMinimum(temp_AstarMap, treasure_coordinate,
self.home, remaining_items, 0)
else:
return 0
# If route home from treasure exists then execute this path
if len(routeHome[0]) > 0:
return 1
#------------------------------------------------------------------------------------#
# astarMinimum() #
# Generates a path using AStar Search with the least number of stones possible. #
# This function prioritizes usage of the stones over the raft. #
#------------------------------------------------------------------------------------#
def astarMinimum(self, AstarMap, position, goal, items, onRaft):
totalStoneCount = items['o']
starting_item_list = items.copy()
starting_stones = 0
# Priortize the raft
if items['r'] > 0:
starting_stones = totalStoneCount
for count in range(starting_stones, totalStoneCount + 1):
items = starting_item_list.copy()
items['o'] = count
testRoute = astarItems(AstarMap, position, goal, items, onRaft, [])
if (len(testRoute[0])) > 0:
testRoute[2]['o'] = totalStoneCount - count
return testRoute
return astarItems(AstarMap, position, goal, starting_item_list, onRaft,
[])
#------------------------------------------------------------------------------------#
# decideBestRoute() #
# This function creates the best route to a goal(coordinate) using a modified #
# yen algorithm/astar heuristic search and a decision tree. #
#------------------------------------------------------------------------------------#
def decideBestRoute(self, goal_list):
#self.max_layer_scores = [0] * (self.deep_search_limit + 1)
best_goal_scores = 0
best_goal_route = []
for goal in goal_list:
#Generates a simple path using Astar Search
astar_memory = self.astarMinimum(self.AstarGlobalMap,
self.position, goal, self.items,
self.onRaft)
bestRoute, bestRouteStones = astar_memory[0], astar_memory[4]
obstacle = self.global_map_obstacles[goal[0]][goal[1]]
# If the goal is water or land, then immediately return the simple astar path.
if not ((obstacle in self.items and len(bestRouteStones) > 0) or
(obstacle == 'o' and len(bestRoute) > 0
and self.items['o'] > 0)):
return bestRoute
print(
"Agent is calculating the best route to {} ({}). Please wait..."
.format(goal, obstacle))
#Generates multiple paths using a hybrid Yen-Astar Search Algorithm.
aStarMultiPath_List = YenAstarMultiPath(
self.global_map_obstacles, self.position, goal, self.items,
self.onRaft) #Generate Multiple Paths
#Setup copies of the global maps to use in generating the decision tree
starting_globalmap = copy.deepcopy(self.global_map_obstacles)
starting_visibleItems = self.visibleItems.copy()
max_deepMapScores = 0
# Loop through all routes to decide the best route to take using future global map states
for routeNb, someRoute in enumerate(aStarMultiPath_List):
print(
"Agent is exploring future global map states of this route. Please wait... \n {}"
.format(someRoute[0]))
temp_globalmap = copy.deepcopy(starting_globalmap)
temp_visibleItems = starting_visibleItems.copy()
temp_visibleItems.remove(goal)
search_level = 0
new_items = someRoute[2]
new_raftstate = someRoute[3]
new_stone_coordinates = self.placedStonesCoordinates + someRoute[
4]
# Agent checks if this route's future global map state has been assessed previously.
globalMapScores = self.checkDeepMapValues(
self.position, goal, new_stone_coordinates, self.items)
#If the route's global map state hasn't been found, then generate a temporary map with the new stone locations created on the new route.
if globalMapScores == 0:
for i in new_stone_coordinates:
temp_globalmap[i[0]][i[1]] = ' '
temp_AstarMap = AstarMap(temp_globalmap)
#Calculate the value of the route's future global map state if the route was executed
globalMapScores = self.deepSearch_globalMapScores(
temp_AstarMap, temp_globalmap, temp_visibleItems, goal,
new_items, new_raftstate, search_level,
new_stone_coordinates)
if globalMapScores < 0:
continue
#Store the value of the route's future global map state
self.addDeepMapValues(self.position, goal,
new_stone_coordinates, self.items,
globalMapScores)
#The best route will have the highest global map Scores. This path is saved.
if globalMapScores > max_deepMapScores:
max_deepMapScores = globalMapScores
bestRoute = someRoute[0].copy()
bestRouteStones = new_stone_coordinates.copy()
#The winning path was found so the entire search is stopped and this path is prioritized
if max_deepMapScores > 100000:
self.theFinalPath = bestRoute + self.theFinalPath
break
if len(self.theFinalPath) > 0:
return self.theFinalPath
if globalMapScores >= best_goal_scores:
best_goal_scores = globalMapScores
best_goal_route = bestRoute.copy()
return best_goal_route
#------------------------------------------------------------------------------------#
# deepSearch_globalMapScores #
# This function calculates the total global map Scores of a position using future #
# states based on all possible routes takeen to all visible coordinates #
#------------------------------------------------------------------------------------#
def deepSearch_globalMapScores(self, DS_AstarMap, globalmap, visibleItems,
start, items, raftstate, search_level,
new_stone_coordinates):
# Limits the level of deep search to avoid long calculation times.
search_level = search_level + 1
if search_level > self.deep_search_limit: return 0
# Setup Variables
current_globalMapScores = 0
bestRouteStones = []
bestRoute = []
best_astar_path = []
new_stone_coordinates_prev = new_stone_coordinates.copy()
# Loops through all visible items and adds the total coordinate Scores for each.
for goal in visibleItems:
max_deepMapScores = 0
obstacle = globalmap[goal[0]][goal[1]]
#Build shortest path
curr_astar_path = self.astarMinimum(DS_AstarMap, start, goal,
items, raftstate)
currRoute, currRouteStones = curr_astar_path[0], curr_astar_path[4]
#If no path possible then ignore
if len(currRoute) == 0: continue
#If stones used, then it requires multiple path assessment to determine best possible path
if (obstacle in self.items and len(currRouteStones) > 0) or (
obstacle == 'o' and len(currRoute) > 0
): #if stones are used then ai needs to begin planning by generating multiple options with YenAstarMultiPath
#Multiple Paths will be generated using YenAStar
aStarMultiPath_List = YenAstarMultiPath(
globalmap, start, goal, items, raftstate)
#Save globalmap and visible items. These change each route based on stone coordinates.
starting_globalmap = copy.deepcopy(globalmap)
starting_visibleItems = visibleItems.copy()
#Remove destimation from visible_items temporarily
if goal in starting_visibleItems:
starting_visibleItems.remove(goal)
#Begin loop through each new path generated by the YenAStar
for routeNb, someRoute in enumerate(aStarMultiPath_List):
#Setup Variables for someRoute (Someroute is the current route being assessed using a Score system)
new_items, new_raftstate, new_stone_coordinates = someRoute[
2], someRoute[3], someRoute[4]
globalMapScores = 0
#create a new map and visible items list
temp_globalmap = copy.deepcopy(starting_globalmap)
temp_visibleItems = starting_visibleItems.copy()
#update stone coordinates of the new map based on new stones generated by this route
new_stone_coordinates = new_stone_coordinates_prev + new_stone_coordinates
# Agent checks if this route's future global map state has been assessed previously.
globalMapScores = self.checkDeepMapValues(
start, goal, new_stone_coordinates, items)
if globalMapScores == 0:
#build new map with new updated stone coordinates
for i in new_stone_coordinates:
temp_globalmap[i[0]][i[1]] = ' '
temp_AstarMap = AstarMap(temp_globalmap)
#Calculate map Scores as if this route was excuted by the agent
#Begins deepsearch to determine best route based on the new map states.
globalMapScores = self.deepSearch_globalMapScores(
temp_AstarMap, temp_globalmap,
starting_visibleItems, goal, new_items,
new_raftstate, search_level, new_stone_coordinates)
if globalMapScores < 0:
return -10000
#Store the value of the route's future global map state
self.addDeepMapValues(start, goal,
new_stone_coordinates, items,
globalMapScores)
#Compares map Scores to best route. Saves if new route provides better future state Scores.
if globalMapScores >= max_deepMapScores:
max_deepMapScores = globalMapScores
bestRoute = copy.deepcopy(someRoute[0])
bestRouteStones = new_stone_coordinates.copy()
#End the search if the final path to the treasure and home is found.
if globalMapScores > 100000:
self.theFinalPath = bestRoute[
1:] + self.theFinalPath
return globalMapScores
else:
bestRoute = currRoute.copy()
bestRouteStones = currRouteStones.copy()
#Scores system for Treasure->Home Path Detection.
if obstacle == '$':
temp_DS_map = AstarMap(globalmap)
homesearch = self.astarMinimum(temp_DS_map, goal, [80, 80],
items, 0)
if len(homesearch[0]) > 0:
self.theFinalPath = bestRoute[1:]
return 10000000000000
current_globalMapScores += max_deepMapScores + self.visibleItemsPts[
obstacle]
return current_globalMapScores
def updateDistance(self):
for coordinate in self.visibleCoordinates:
self.global_map_distance[coordinate[0]][coordinate[1]] = int(
manhattan(self.position, coordinate))
#------------------------------------------------------------------------------------#
# update_top_global_scores #
# Updates the top_global_map_scores memory for all visible and unexplored coordinates#
#------------------------------------------------------------------------------------#
def update_top_global_scores(self):
removecoordinatesList = []
# Calculate the Scores for visible coordinates only
for coordinate in self.visibleCoordinates:
x, y = coordinate[0], coordinate[1]
obstacle = self.global_map_obstacles[x][y]
# Only update items periodically to avoid excessive calculations
if self.update_global_values_flag == 0 and obstacle in self.items:
continue
# Water is given negative Scores if the agent has no raft, no stones and is not on a raft
if obstacle == '~' and self.items['r'] < 1 and self.items[
'o'] < 1 and (self.onRaft == 0):
self.top_global_map_scores[x][y] = -500000000
continue
# astar path is generated from the agents current position to the coordinate
astar_memory = self.astarMinimum(self.AstarGlobalMap,
self.position, [x, y], self.items,
self.onRaft)
placed_stone_list = astar_memory[4]
# These are sent to the the calculateTotalCoord Function below.
totalScores, remove_flag, reachScores, coordScores = self.calculateTotalCoordScores(
x, y, astar_memory, self.global_map_obstacles, self.onRaft,
self.items, self.position)
# Save Scores for the coordinate
self.top_global_map_scores[x][y] = totalScores
if remove_flag:
removecoordinatesList.append([x, y])
#remove coordinate from visible list if deemed by the calculateTotalCoordScores function
for coordinate in removecoordinatesList:
self.visibleCoordinates.remove(coordinate)
self.exploredCoordinates.append(coordinate)
self.top_global_map_scores[coordinate[0]][coordinate[1]] = -10003
self.update_global_values_flag = 0
#------------------------------------------------------------------------------------#
# calculateTotalCoordScores: Calculates the TOTAL Scores for a coordinate #
# #
# All coordinates are assigned Scores based on its reachability (reachScores) #
# and importance to the agent (coordScores). #
# Reachability is determine using whether astar search can generate a path #
# with/without any items. #
# Coordinates that do not require items are higher in value/priority #
# Importance to the agent is calculated in the calculateCoordScores() function below.#
#------------------------------------------------------------------------------------#
def calculateTotalCoordScores(self, x, y, astar_memory, global_map, onRaft,
items, position):
remove_flag = 0
obstacle = global_map[x][y]
reachScores, coordScores = 0, 0
#Door, agent and boundaries are ignored
if (obstacle == '*') or (obstacle == '^') or (obstacle == '.'):
return 0, 1, 0, 0
#reachScores depedent on two factors: Reachable with/without items.
if len(astar_memory[0]
) > 0: #Path of length zero means coordinate is not reachable
if astar_memory[
1]: #if list is empty, then it requires no items. This gives it 50 Scores.
reachScores = 10
else:
reachScores = 50
coordScores = self.calculateCoordScores(
[x, y], global_map, onRaft, items,
position) #Calculates coordinate Scores using function
totalScores = reachScores + coordScores
else:
#print('{},{} is not reachable'.format(x,y))
totalScores = -10002
#Land ignored if it doesnt exceed threshold
if (obstacle == ' ') and coordScores < 2.6 and reachScores > 1:
#print('Removed: {} Reason: not enough coordscores ({})'.format([x,y], coordScores))
remove_flag = 1
totalScores = 0
return totalScores, remove_flag, reachScores, coordScores
#------------------------------------------------------------------------------------#
# calculateCoordScores: #
# Calculates the coordinate Scores for a given coordinate #
# Below is the Score setup for each coordinate type (land, water, item) #
#------------------------------------------------------------------------------------#
def calculateCoordScores(
self, coordinate, global_map, onRaft, items,
position): # assigns Scores to each coordinate on the global map
x = coordinate[0]
y = coordinate[1]
obstacle = global_map[x][y]
#Scores setup for key, axe and stone. Varies whether agent is on a raft or on land
if obstacle in [
'k', 'a', 'o'
]: #return 10 pts if new items collectable without losing items
if onRaft:
return 8
else:
return 20
#Scores setup for Tree and Door. Dependant on whether agent has axe or key
elif obstacle in [
'T', '-'
]: # returns 10 pts if agent contains pre-req to collect new items
if obstacle == 'T' and self.items['a'] > 0:
if onRaft:
return 8
else:
return 20
elif obstacle == '-' and self.items['k'] > 0:
if onRaft: # the agent prioritizes obtaining another raft over opening doors
return 8
else:
return 20
else:
return 0
#Scores setup for land. Dependent on surrounding unknown space (?), water and raft state
elif obstacle == ' ': # values exploring land depending on whether agent is currently on a raft
if onRaft:
Scores = -5
else:
Scores = 0
for x_1 in range(-2, 3):
for y_2 in range(-2, 3):
if (x + x_1 in range(
self.global_map_size)) and (y + y_2 in range(
self.global_map_size)):
surrounding_obstacle = self.global_map_obstacles[
x + x_1][y + y_2]
if surrounding_obstacle == '?':
if onRaft:
Scores += 0.2
else:
Scores += 1
if surrounding_obstacle == '~':
Scores += 0.2
return Scores
#Scores setup for water. Dependent on surrounding unknown space (?), water and current items/raft
elif obstacle == '~': # values exploring the water depending on whether agent is currently on a raft
Scores = 0
if (items['o'] < 1 and items['r'] < 1) and onRaft == 0:
return 0
for x_1 in range(-2, 3):
for y_2 in range(-2, 3):
if (x + x_1 in range(
self.global_map_size)) and (y + y_2 in range(
self.global_map_size)):
surrounding_obstacle = global_map[x + x_1][y + y_2]
if surrounding_obstacle == '?':
if onRaft:
Scores += 1.5
else:
Scores += 0.5
return Scores
#Scores setup for treasure. Agent only goes for treasure if it can plot a path home
elif obstacle == '$':
if self.checkRouteHome(position, [x, y]):
return 999999999
return -10000
return 0
#------------------------------------------------------------------------------------#
# update_global_map_obstacles() #
# Updates the global_map_obstacles using the map received from the TCP Socket #
#------------------------------------------------------------------------------------#
def update_global_map_obstacles(self):
rotated_view_window = rotate(self.view_window, 360 - self.direction)
N = 5
M = 160
# current position of a agent in the
# global map.
curr_i = self.position[0]
curr_j = self.position[1]
# looping around the view map to copy into
# GlobalMap
for i in range(N):
for j in range(N):
element = rotated_view_window[i][j]
# indices to copy into global map.
Iidx = curr_i + i - 2
Jidx = curr_j + j - 2
# Conditons for writing to gloabl map
# 1) The copying indices should not
# exceed the global map
# 2) Not copying the current directions of AI.
if (Iidx >= 0 and Iidx < M and Jidx >= 0 and Jidx < M
and not (i == 2 and j == 2)):
if [Iidx, Jidx] in self.placedStonesCoordinates:
self.global_map_obstacles[Iidx][Jidx] = ' '
else:
self.global_map_obstacles[Iidx][Jidx] = element
elif (i == 2 and j == 2):
self.global_map_obstacles[Iidx][Jidx] = '^'
self.top_global_map_scores[Iidx][Jidx] = -10000
if [Iidx, Jidx] not in self.exploredCoordinates:
self.exploredCoordinates.append([Iidx, Jidx])
if [Iidx, Jidx] in self.visibleCoordinates:
self.visibleCoordinates.remove([Iidx, Jidx])
if [Iidx, Jidx] in self.visibleItems:
self.visibleItems.remove([Iidx, Jidx])
if [Iidx, Jidx] not in self.exploredCoordinates and [
Iidx, Jidx
] not in self.visibleCoordinates:
if self.global_map_obstacles[Iidx][
Jidx] in self.visibleItemsPts:
self.visibleItems.append([Iidx, Jidx])
astar_path = self.astarMinimum(self.AstarGlobalMap,
self.position, [Iidx, Jidx],
self.items, self.onRaft)
totalScores, remove_flag, reachScores, coordScores = self.calculateTotalCoordScores(
Iidx, Jidx, astar_path, self.global_map_obstacles,
self.onRaft, self.items, self.position)
if remove_flag:
self.top_global_map_scores[Iidx][Jidx] = 0
self.exploredCoordinates.append([Iidx, Jidx])
remove_flag = 0
else:
self.top_global_map_scores[Iidx][Jidx] = totalScores
self.visibleCoordinates.append([Iidx, Jidx])
self.AstarGlobalMap.updateGrid(self.global_map_obstacles,
self.onRaft)
#------------------------------------------------------------------------------------#
# agentController: #
# Converts a list of positions into keyboard actions for the agent #
# These actions are queued in actionQueue and pushed into the game one at a time #
#------------------------------------------------------------------------------------#
def agentController(
self, positionList): ## Missing implementation for unlock and cut
requiredItems = {'-': 'k', 'T': 'a'}
currentPos = self.position
currentDir = self.direction
if len(positionList) == 0:
return
if currentPos != positionList[0]:
#print("Error: Positions do not match {} -> {}". format(currentPos, positionList[0]))
return
for newPosition in positionList[1:]:
movement = (newPosition[0] - currentPos[0],
newPosition[1] - currentPos[1])
if movement in self.rotationAngleTbl:
newDirection = self.rotationAngleTbl[movement]
else:
#print("Error: Invalid movement {} -> {} = {}".format(currentPos, newPosition,movement))
return
rotationRequired = (newDirection - currentDir) % 360
if rotationRequired == 90:
self.actionQueue.put('L')
elif rotationRequired == 180:
self.actionQueue.put('LL')
elif rotationRequired == 270:
self.actionQueue.put('R')
if self.global_map_obstacles[newPosition[0]][newPosition[
1]] in requiredItems: #if tree or door then cut or unlock
if self.global_map_obstacles[newPosition[0]][
newPosition[1]] == 'T':
self.actionQueue.put('C')
elif self.global_map_obstacles[newPosition[0]][
newPosition[1]] == '-':
self.actionQueue.put('U')
self.actionQueue.put('F')
currentPos = newPosition
currentDir = newDirection
#------------------------------------------------------------------------------------#
# move_agent: #
# Takes the actions and updates the agent's position and direction to ensure it #
# aligns with the current game #
#------------------------------------------------------------------------------------#
def move_agent(self, action):
blocks = ['T', '*', '-', '.']
next_position = self.give_front_position(self.position)
if action == 'L' or action == 'l':
self.direction = (self.direction + 90) % 360
elif action == 'R' or action == 'r':
self.direction = (self.direction - 90) % 360
elif action == 'F' or action == 'f':
#Saves coordinates that have been explored and removes them from visible coordinates to prevent looping
if self.position not in self.exploredCoordinates:
self.exploredCoordinates.append(self.position)
if self.position in self.visibleCoordinates:
self.visibleCoordinates.remove(self.position)
if self.global_map_obstacles[next_position[0]][
next_position[1]] not in blocks:
prevElement = self.global_map_obstacles[self.position[0]][
self.position[1]]
self.position = next_position
nextElement = self.global_map_obstacles[next_position[0]][
next_position[1]]
#Update items list if items found
if nextElement in self.items:
self.items[nextElement] += 1
self.update_global_values_flag = 1
# reduce stones/raft if agent steps on water
if nextElement == '~':
if self.items['o'] > 0:
self.items['o'] -= 1
self.placedStonesCoordinates.append(next_position)
self.update_global_values_flag = 1
elif self.onLand: # Only change this if the previous position was land ' '
if self.items['r'] > 0:
self.items['r'] = self.items['r'] - 1
self.onRaft = 1
self.onLand = 0
#print("on a raft")
self.update_global_values_flag = 1
#confirm agent is on land.
elif nextElement == ' ':
if self.onRaft:
self.update_global_values_flag = 1
self.onRaft = 0
self.onLand = 1
#add raft to items if tree is cut
elif (action == 'C' or action == 'c'):
tree = self.global_map_obstacles[next_position[0]][
next_position[1]]
if tree == 'T' and self.items['a'] > 0:
self.items['r'] += 1
self.update_global_values_flag = 1
#updates position depending on the direction of the agent's movement (front position)
def give_front_position(self, starting_position):
new_position = starting_position.copy()
if self.direction == 0:
new_position[0] -= 1
elif self.direction == 90:
new_position[1] -= 1
elif self.direction == 180:
new_position[0] += 1
elif self.direction == 270:
new_position[1] += 1
return new_position
def print_small_matrix(self, matrix, print_size):
print("\n+{}+".format("-" * print_size))
for i in range(print_size):
print('|', end='')
for j in range(print_size):
if (i == 2) and (j == 2):
print('^', end='')
else:
print(matrix[i][j], end='')
print('|')
print("+{}+\n".format("-" * print_size))
return 0
def print_large_matrix(self, matrix, print_size):
print("\n+{}+".format("-" * print_size))
for i in range(print_size):
print('|', end='')
for j in range(print_size):
print(matrix[i][j], end='')
print('|')
print("+{}+\n".format("-" * print_size))
return 0
def checkWinCondition(self, start_time):
if self.items['$'] > 0 and self.position == self.home:
print("Winner Winner Chicken Dinner!")
end_time = time.time() - start_time
print("Time: {}".format(end_time))
sys.exit()
def main_loop(self):
time.sleep(1)
start_time = time.time()
refreshvalues_flag = 0
action_list = ['z']
while (True):
for i in range(len(action_list)):
self.move_agent(
action_list[i]
) # Moves agent in current program to ensure allignment with the game
self.checkWinCondition(
start_time
) # Ends game if win conditions met (Treasure found and returned home)
self.view_window = self.TCP_Socket.recv_map(
) # Receives map from socket and saves it
if len(self.view_window) > 0:
self.update_global_map_obstacles()
self.print_small_matrix(self.view_window, self.view_size)
if self.actionQueue.empty():
refreshvalues_flag += 1
self.updateDistance()
if self.update_global_values_flag or refreshvalues_flag > 1: # Update coordinate values every 2 moves. Immediate update is required during certain conditions
self.update_top_global_scores()
refreshvalues_flag = 0
nextPosition = self.decideNextPosition() #Picks next positions
bestRoute = self.decideBestRoute(
nextPosition) #Generates thhe best route
self.agentController(
bestRoute
) #Converts route into game actions. Game actions are added into the actionQueue
if not self.actionQueue.empty():
action_list = self.actionQueue.get()
self.TCP_Socket.send_action(action_list)
class Program(object):
def __init__(self, name, command):
if debug:
print("Create Program '%s' with command '%s'" % (name, command))
self.queue = Queue()
self.name = name
self.params = command.split()
self.start()
def start(self):
def target():
print(self.params)
self.process = subprocess.Popen(self.params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr,
universal_newlines=True,
bufsize=1)
if debug: print(self.name, "start reading", sep=" : ")
current_label = None
content = ""
for line in self.process.stdout:
if debug: print("<" + self.name, line, sep=" : ", end="")
if current_label:
if line.startswith("START"):
print(self.name,
": current label not completed (STOP expected)",
file=stderr)
break
elif line.startswith("STOP"):
label = line.strip().split(maxsplit=1)
if len(label) == 1 or label[1] == current_label:
if debug:
print("QUEUE", self.name,
(current_label, content))
self.queue.put((current_label, content))
current_label = None
content = ""
else:
print(self.name,
": 'STOP' label not corresponding to '%s'" %
current_label,
file=stderr)
break
else:
content += line
elif line.startswith("START"):
label = line.strip().split(maxsplit=1)
if len(label) == 2:
current_label = label[1]
else:
print(self.name,
": 'START' must be followed by a label",
file=stderr)
break
else:
print(self.name,
" : no label started (START expected)",
file=stderr)
break
if debug:
print(self.name, "stop reading", sep=" : ")
self.stop()
self.thread = Thread(target=target)
self.thread.start()
if debug: print("'%s' started" % self.name)
time.sleep(timeout / 1000)
def stop(self):
try:
self.process.terminate()
self.thread.join()
except:
pass
self.process = None
self.thread = None
if debug: print("'%s' stoped" % self.name)
def is_running(self):
return self.process != None
def read(self, label, error_intolerant=False):
if debug: print("'%s' : read label '%s'" % (self.name, label))
while not self.queue.empty():
item = self.queue.get()
if debug: print("DEQUEUE", self.name, item)
if item[0] == label:
if debug: print(item[1], end="")
return item[1]
elif error_intolerant:
print(self.name,
" : expected label '%s' instead of '%s'" %
(label, item[0]),
file=stderr)
self.stop()
break
return None
def write(self, label, content):
if not self.is_running():
return
if content == None:
content = ""
if debug: print(">" + self.name, "START %s" % label, sep=" : ")
self.process.stdin.write("START %s\n" % label)
for line in content.strip().split("\n"):
if not line: continue
if debug:
print(">" + self.name, line, sep=" : ")
self.process.stdin.write(line + "\n")
self.process.stdin.write("STOP %s\n" % label)
if debug: print(">" + self.name, "STOP %s" % label, sep=" : ")
class Microvm:
"""Class to represent a Firecracker microvm.
A microvm is described by a unique identifier, a path to all the resources
it needs in order to be able to start and the binaries used to spawn it.
Besides keeping track of microvm resources and exposing microvm API
methods, `spawn()` and `kill()` can be used to start/end the microvm
process.
"""
def __init__(
self,
resource_path,
fc_binary_path,
jailer_binary_path,
microvm_id,
build_feature='',
monitor_memory=True,
aux_bin_paths=None
):
"""Set up microVM attributes, paths, and data structures."""
# Unique identifier for this machine.
self._microvm_id = microvm_id
self.build_feature = build_feature
# Compose the paths to the resources specific to this microvm.
self._path = os.path.join(resource_path, microvm_id)
self._kernel_path = os.path.join(self._path, MICROVM_KERNEL_RELPATH)
self._fsfiles_path = os.path.join(self._path, MICROVM_FSFILES_RELPATH)
self._kernel_file = ''
self._rootfs_file = ''
# The binaries this microvm will use to start.
self._fc_binary_path = fc_binary_path
assert os.path.exists(self._fc_binary_path)
self._jailer_binary_path = jailer_binary_path
assert os.path.exists(self._jailer_binary_path)
# Create the jailer context associated with this microvm.
self._jailer = JailerContext(
jailer_id=self._microvm_id,
exec_file=self._fc_binary_path
)
self.jailer_clone_pid = None
# Now deal with the things specific to the api session used to
# communicate with this machine.
self._api_session = None
self._api_socket = None
# Session name is composed of the last part of the temporary path
# allocated by the current test session and the unique id of this
# microVM. It should be unique.
self._session_name = os.path.basename(os.path.normpath(
resource_path
)) + self._microvm_id
# nice-to-have: Put these in a dictionary.
self.actions = None
self.boot = None
self.drive = None
self.logger = None
self.mmds = None
self.network = None
self.machine_cfg = None
self.vsock = None
# The ssh config dictionary is populated with information about how
# to connect to a microVM that has ssh capability. The path of the
# private key is populated by microvms with ssh capabilities and the
# hostname is set from the MAC address used to configure the microVM.
self._ssh_config = {
'username': '******',
'netns_file_path': self._jailer.netns_file_path()
}
# Deal with memory monitoring.
if monitor_memory:
self._memory_events_queue = Queue()
else:
self._memory_events_queue = None
# External clone/exec tool, because Python can't into clone
self.aux_bin_paths = aux_bin_paths
def kill(self):
"""All clean up associated with this microVM should go here."""
if self._jailer.daemonize:
if self.jailer_clone_pid:
run('kill -9 {}'.format(self.jailer_clone_pid), shell=True)
else:
run(
'screen -XS {} kill'.format(self._session_name),
shell=True
)
if self._memory_events_queue and not self._memory_events_queue.empty():
raise mem_tools.MemoryUsageExceededException(
self._memory_events_queue.get())
@property
def api_session(self):
"""Return the api session associated with this microVM."""
return self._api_session
@property
def api_socket(self):
"""Return the socket used by this api session."""
# TODO: this methods is only used as a workaround for getting
# firecracker PID. We should not be forced to make this public.
return self._api_socket
@property
def path(self):
"""Return the path on disk used that represents this microVM."""
return self._path
@property
def id(self):
"""Return the unique identifier of this microVM."""
return self._microvm_id
@property
def jailer(self):
"""Return the jailer context associated with this microVM."""
return self._jailer
@jailer.setter
def jailer(self, jailer):
"""Setter for associating a different jailer to the default one."""
self._jailer = jailer
@property
def kernel_file(self):
"""Return the name of the kernel file used by this microVM to boot."""
return self._kernel_file
@kernel_file.setter
def kernel_file(self, path):
"""Set the path to the kernel file."""
self._kernel_file = path
@property
def rootfs_file(self):
"""Return the path to the image this microVM can boot into."""
return self._rootfs_file
@rootfs_file.setter
def rootfs_file(self, path):
"""Set the path to the image associated."""
self._rootfs_file = path
@property
def fsfiles(self):
"""Path to filesystem used by this microvm to attach new drives."""
return self._fsfiles_path
@property
def ssh_config(self):
"""Get the ssh configuration used to ssh into some microVMs."""
return self._ssh_config
@ssh_config.setter
def ssh_config(self, key, value):
"""Set the dict values inside this configuration."""
self._ssh_config.__setattr__(key, value)
@property
def memory_events_queue(self):
"""Get the memory usage events queue."""
return self._memory_events_queue
@memory_events_queue.setter
def memory_events_queue(self, queue):
"""Set the memory usage events queue."""
self._memory_events_queue = queue
def create_jailed_resource(self, path):
"""Create a hard link to some resource inside this microvm."""
return self.jailer.jailed_path(path, create=True)
def get_jailed_resource(self, path):
"""Get the jailed path to a resource."""
return self.jailer.jailed_path(path, create=False)
def setup(self):
"""Create a microvm associated folder on the host.
The root path of some microvm is `self._path`.
Also creates the where essential resources (i.e. kernel and root
filesystem) will reside.
# Microvm Folder Layout
There is a fixed tree layout for a microvm related folder:
``` file_tree
<microvm_uuid>/
kernel/
<kernel_file_n>
....
fsfiles/
<fsfile_n>
<ssh_key_n>
<other fsfiles>
...
...
```
"""
os.makedirs(self._path, exist_ok=True)
os.makedirs(self._kernel_path, exist_ok=True)
os.makedirs(self._fsfiles_path, exist_ok=True)
def spawn(self):
"""Start a microVM as a daemon or in a screen session."""
self._jailer.setup()
self._api_socket = self._jailer.api_socket_path()
self._api_session = Session()
self.actions = Actions(self._api_socket, self._api_session)
self.boot = BootSource(self._api_socket, self._api_session)
self.drive = Drive(self._api_socket, self._api_session)
self.logger = Logger(self._api_socket, self._api_session)
self.machine_cfg = MachineConfigure(
self._api_socket,
self._api_session
)
self.mmds = MMDS(self._api_socket, self._api_session)
self.network = Network(self._api_socket, self._api_session)
self.vsock = Vsock(self._api_socket, self._api_session)
jailer_param_list = self._jailer.construct_param_list()
# When the daemonize flag is on, we want to clone-exec into the
# jailer rather than executing it via spawning a shell. Going
# forward, we'll probably switch to this method for running
# Firecracker in general, because it represents the way it's meant
# to be run by customers (together with CLONE_NEWPID flag).
#
# We have to use an external tool for CLONE_NEWPID, because
# 1) Python doesn't provide a os.clone() interface, and
# 2) Python's ctypes libc interface appears to be broken, causing
# our clone / exec to deadlock at some point.
if self._jailer.daemonize:
if self.aux_bin_paths:
cmd = [self.aux_bin_paths['cloner']] + \
[self._jailer_binary_path] + \
jailer_param_list
_p = run(cmd, stdout=PIPE, stderr=PIPE, check=True)
# Terrible hack to make the tests fail when starting the
# jailer fails with a panic. This is needed because we can't
# get the exit code of the jailer. In newpid_clone.c we are
# not waiting for the process and we always return 0 if the
# clone was successful (which in most cases will be) and we
# don't do anything if the jailer was not started
# successfully.
if _p.stderr.decode().strip():
raise Exception(_p.stderr.decode())
self.jailer_clone_pid = int(_p.stdout.decode().rstrip())
else:
# This code path is not used at the moment, but I just feel
# it's nice to have a fallback mechanism in place, in case
# we decide to offload PID namespacing to the jailer.
_pid = os.fork()
if _pid == 0:
os.execv(
self._jailer_binary_path,
[self._jailer_binary_path] + jailer_param_list
)
self.jailer_clone_pid = _pid
else:
start_cmd = 'screen -dmS {session} {binary} {params}'
start_cmd = start_cmd.format(
session=self._session_name,
binary=self._jailer_binary_path,
params=' '.join(jailer_param_list)
)
run(start_cmd, shell=True, check=True)
out = run('screen -ls', shell=True, stdout=PIPE)\
.stdout.decode('utf-8')
screen_pid = re.search(r'([0-9]+)\.{}'.format(self._session_name),
out).group(1)
self.jailer_clone_pid = open('/proc/{0}/task/{0}/children'
.format(screen_pid)
).read().strip()
# Wait for the jailer to create resources needed, and Firecracker to
# create its API socket.
# We expect the jailer to start within 80 ms. However, we wait for
# 1 sec since we are rechecking the existence of the socket 5 times
# and leave 0.2 delay between them.
self._wait_create()
@retry(delay=0.2, tries=5)
def _wait_create(self):
"""Wait until the API socket and chroot folder are available."""
os.stat(self._jailer.api_socket_path())
def basic_config(
self,
vcpu_count: int = 2,
ht_enabled: bool = False,
mem_size_mib: int = 256,
add_root_device: bool = True
):
"""Shortcut for quickly configuring a microVM.
It handles:
- CPU and memory.
- Kernel image (will load the one in the microVM allocated path).
- Root File System (will use the one in the microVM allocated path).
- Does not start the microvm.
The function checks the response status code and asserts that
the response is within the interval [200, 300).
"""
response = self.machine_cfg.put(
vcpu_count=vcpu_count,
ht_enabled=ht_enabled,
mem_size_mib=mem_size_mib
)
assert self._api_session.is_status_no_content(response.status_code)
if self.memory_events_queue:
mem_tools.threaded_memory_monitor(
mem_size_mib,
self.jailer_clone_pid,
self._memory_events_queue
)
# Add a kernel to start booting from.
response = self.boot.put(
kernel_image_path=self.create_jailed_resource(self.kernel_file)
)
assert self._api_session.is_status_no_content(response.status_code)
if add_root_device:
# Add the root file system with rw permissions.
response = self.drive.put(
drive_id='rootfs',
path_on_host=self.create_jailed_resource(self.rootfs_file),
is_root_device=True,
is_read_only=False
)
assert self._api_session.is_status_no_content(response.status_code)
def ssh_network_config(
self,
network_config,
iface_id,
allow_mmds_requests=False,
tx_rate_limiter=None,
rx_rate_limiter=None
):
"""Create a host tap device and a guest network interface.
'network_config' is used to generate 2 IPs: one for the tap device
and one for the microvm. Adds the hostname of the microvm to the
ssh_config dictionary.
:param network_config: UniqueIPv4Generator instance
:param iface_id: the interface id for the API request
:param allow_mmds_requests: specifies whether requests sent from
the guest on this interface towards the MMDS address are
intercepted and processed by the device model.
:param tx_rate_limiter: limit the tx rate
:param rx_rate_limiter: limit the rx rate
:return: an instance of the tap which needs to be kept around until
cleanup is desired, the configured guest and host ips, respectively.
"""
# Create tap before configuring interface.
tapname = self.id[:8] + 'tap' + iface_id
(host_ip, guest_ip) = network_config.get_next_available_ips(2)
tap = net_tools.Tap(
tapname,
self._jailer.netns,
ip="{}/{}".format(
host_ip,
network_config.get_netmask_len()
)
)
guest_mac = net_tools.mac_from_ip(guest_ip)
response = self.network.put(
iface_id=iface_id,
host_dev_name=tapname,
guest_mac=guest_mac,
allow_mmds_requests=allow_mmds_requests,
tx_rate_limiter=tx_rate_limiter,
rx_rate_limiter=rx_rate_limiter
)
assert self._api_session.is_status_no_content(response.status_code)
self.ssh_config['hostname'] = guest_ip
return tap, host_ip, guest_ip
def start(self):
"""Start the microvm.
This function has asserts to validate that the microvm boot success.
"""
response = self.actions.put(action_type='InstanceStart')
assert self._api_session.is_status_no_content(response.status_code)
class QNotificationArea(QtGui.QWidget): # QtWidgets.QWidget
"""
Notification area to show notifications in. Will be projected on top of
another QWidget which should be passed as an argument to this class.
:inherits: QtWidgets.QWidget
"""
default_notification_styles = u"""
QNotification {
font-size: 16px;
padding: 0px;
margin: 0px;
border-radius: 6px;
}
QNotification #message{
color: #FFFFFF;
padding: 0px;
margin: 0px;
width: 100%;
}
QNotification #closeButton{
color: #FFFFFF;
padding: 0px;
margin: 0px;
}
QNotification#primary {
background-color: #337ab7;
border-color: #2e6da4;
}
QNotification#success {
background-color: #5cb85c;
border-color: #4cae4c;
}
QNotification#info {
background-color: #5bc0de;
border-color: #46b8da;
}
QNotification#warning {
background-color: #f0ad4e;
border-color: #eea236;
}
QNotification#danger {
background-color: #d9534f;
border-color: #d43f3a;
}
QNotification#space-grey {
background-color: #343d46;
border-color: c0c5ce;
}
"""
def __init__(self, target_widget, *args, **kwargs):
""" Constructor
:param target_widget: QtWidgets.QWidget The widget to project the notifications on
:param useGlobalCSS: bool (default: False) Flag which indicates whether global style sheets should be used
(which have been set at app-level). If False, the default style sheets stored at
self.default_notification_styles will be loaded.
:param useQueue: bool (default: True) Indicates whether a message queue should be implemented.
This will only show *maxMessages* at the same time and will put all other messages in a queue.
Once a message disappears, the next one in the queue will be shown
(up to maxMessages at the same time)
:param: maxMessages: int (default: 2) The number of messages to display at the same time.
:raises: TypeError if targetWidget is not an object that inherits QWidget.
"""
if not isinstance(target_widget, QtGui.QWidget): # QtWidgets.QWidget
raise TypeError('targetWidget is not a QWidget (or child of it')
# Get some variables from kwargs.
useGlobalCSS = kwargs.get(u'useGlobalCSS', False)
self.useQueue = kwargs.get(u'useQueue', True)
self.maxMessages = kwargs.get(u'maxMessages', 2)
super(QNotificationArea, self).__init__(*args, **kwargs)
if not useGlobalCSS:
self.setStyleSheet(self.default_notification_styles)
if self.useQueue:
self.queue = Queue()
self.setParent(target_widget)
self.targetWidget = target_widget
self.setContentsMargins(0, 0, 0, 0)
notification_area_layout = QtGui.QVBoxLayout() # QtWidgets.QVBoxLayout()
self.setLayout(notification_area_layout)
# Init effects to None
self.entryEffect = None
self.entryEffectDuration = None
self.exitEffect = None
self.exitEffectDuration = None
# Notifications area geometry animation.
self.slideAnimation = QtCore.QPropertyAnimation(self, safe_encode("geometry"))
self._slide_right_limit = None
# Store original target classes resizeEvent to be called in our own function.
self.target_resize_event = target_widget.resizeEvent
# Overwrite resizeEvent function of targetWidget to capture it our-self
# (parent's resizeEvent will be called in our function too).
self.targetWidget.resizeEvent = self.resizeEvent
self.hide()
# Private functions:
def __delete_notification(self, notification=None):
""" Closes and destroys the supplied notification.
:param notification: (default: None)
"""
notification.close()
self.layout().removeWidget(notification)
self.adjustSize()
# Hide notification area if it doesn't contain any items
if self.layout().count() == 0:
self.hide()
if self.useQueue:
try:
notification = self.queue.get(False)
self._show_notification(notification)
except Empty:
pass
# Public functions:
def setEntryEffect(self, effect, duration=250):
""" Sets the effect with which the notifications are to appear.
:param effect: dict{'fadeIn', None} The effect which should be used (for now only 'fadeIn' is available)
if None is passed for this argument, no effect will be used and the notifications will
just appear directly.
:param duration: int (default: 250 ms) The duration of the effect in milliseconds.
:raises: TypeError If the object provided for duration is not an integer.
:raises: ValueError When duration is less than 0, or effect has an invalid value
"""
if effect not in [None, u'fadeIn']:
raise ValueError(u'Invalid entry effect')
if not isinstance(duration, int):
raise TypeError(u'Duration should be an int')
if duration < 0:
raise ValueError(u'Duration should be larger than 0')
# Set the entry effect and its duration.
self.entryEffect = effect
self.entryEffectDuration = duration
def setExitEffect(self, effect, duration=500):
""" Sets the effect with which the notifications are to disappear.
:param effect: dict{'fadeOut', None} the effect which should be used (for now only 'fadeOut' is available)
if None is passed for this argument, no effect will be used and the notifications will
just appear directly.
:param duration: int (default: 1000 ms) The duration of the effect in milliseconds
:raises: TypeError If the object passed for duration is not an int.
:raises: ValueError When duration is less than 0, or effect has an invalid value.
"""
if effect not in [None, u'fadeOut']:
raise ValueError(u'Invalid exit effect')
if not isinstance(duration, int):
raise TypeError(u'Duration should be an int')
if duration < 0:
raise ValueError(u'Duration should be larger than 0')
# Set the exit effect and its duration.
self.exitEffect = effect
self.exitEffectDuration = duration
# Events:
# @QtCore.Slot('QString', 'QString', int)
# @QtCore.Slot('QString', 'QString', int, 'QString')
@QtCore.pyqtSlot('QString', 'QString', int)
@QtCore.pyqtSlot('QString', 'QString', int, 'QString')
def display(self, message, category, timeout=5000, button_text=None):
""" Displays a notification.
If a queue is used, then the notification will only be shown directly
if the number of notifications shown is smaller than maxMessages.
:param message: str The message to display.
:param category : dict{'primary', 'success', 'info', 'warning', 'danger'}
The type of notification that should be shown. Adheres to bootstrap standards which are
primary, success, info, warning and danger.
:param timeout : int (optional) The duration for which the notification should be shown.
If None then the notification will be shown indefinitely.
:param button_text: str (optional) The text to display on the closing button.
If not provided a cross will be shown.
:raises: ValueError if the category is other than one of the expected values.
"""
notification = QNotification(message, category, timeout, button_text, self)
notification.closeClicked.connect(self.remove)
# Queue if max amount of notifications is shown.
if self.useQueue and self.layout().count() >= self.maxMessages:
self.queue.put(notification)
else:
self._show_notification(notification)
def _show_notification(self, notification):
"""
:param notification:
:return:
"""
if not self.isVisible():
self.show()
self.raise_()
self.layout().addWidget(notification)
# Check for entry effects
if self.entryEffect is not None:
if self.entryEffect == u"fadeIn":
notification.fadeIn(self.entryEffectDuration)
else:
notification.display()
self.adjustSize()
if notification.timeout is not None and notification.timeout > 0:
QtCore.QTimer.singleShot(notification.timeout, lambda: self.remove(notification))
# def slideIn(self, duration):
# """
# Moves the QNotification Area from
#
# :param duration:
# :return:
# """
# print(self.x(), self.y())
# self.setGeometry(QtCore.QRect(100, 0, self.width(), self.height()))
# @QtCore.Slot()
@QtCore.pyqtSlot()
def remove(self, notification=None):
""" Removes a notification.
:param notification: QNotification (default: None)
The notification to remove. This function also serves as a PyQt slot
for signals emitted from a QNotification. In this case, the QNotification
object is retrieved by using self.sender()
:raises: ValueError If notification is not None or a QNotification.
"""
# This function also functions as a pyqt slot. In that case, no
# notification argument is passed, but this is set as self.sender()
if notification is None:
try:
notification = self.sender()
except:
raise ValueError(u'QNotification object needs to be passed or '
u'this function should be used as a slot for a signal '
u'emitted by a QNotification')
if notification.isBeingRemoved:
return
else:
notification.isBeingRemoved = True
# Check if notification is still present (and has not manually been
# closed before this function is called by a timeout)
if self.layout().indexOf(notification) < 0:
return
# Implement animation here
if self.exitEffect == u'fadeOut':
notification.fadeOut(self.__delete_notification, self.exitEffectDuration)
else:
self.__delete_notification(notification)
# Internal Qt functions:
def resizeEvent(self, event):
""" Internal QT function (do not call directly).
:param event:
"""
self.target_resize_event(event)
newsize = event.size()
self.setFixedWidth(newsize.width())
self.adjustSize()
def paintEvent(self, pe):
""" Redefinition of paintEvent.
Makes class QNotificationArea available in style sheets. Internal QT function (do not call directly).
:param pe:
"""
# o = QtWidgets.QStyleOption()
option = QtGui.QStyleOption()
# o.initFrom(self)
# p = QtGui.QPainter(self)
painter = QtGui.QPainter(self)
option.initFrom(self)
# self.style().drawPrimitive(QtWidgets.QStyle.PE_Widget, o, p, self)
self.style().drawPrimitive(QtGui.QStyle.PE_Widget, option, painter, self)
class SomaticTrainerHomeWindow(Frame):
model: keras.Model
port: serial.Serial
serial_sniffing_thread: threading.Thread
training_set: GestureTrainingSet
pointer_gesture = [True, True, True, False]
class State(Enum):
quitting = -1
disconnected = 0
connecting = 1
connected = 2
recording = 3
processing = 4
def __init__(self, master=None):
Frame.__init__(self, master)
self.master = master
self.logger = logging.getLogger('HomeWindow')
self.logger.setLevel(logging.INFO)
self._log_parsing = False
self._log_angular_velocity = False
self.gesture_cone_angle = 2 / 3 * np.pi # 120 degrees
hand_icon_directory = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'Hands')
unknown_hand_icon_bitmap = Image.open(os.path.join(hand_icon_directory, 'Unknown.png'))
unknown_hand_icon_bitmap.thumbnail((250, 250))
self.unknown_hand_icon = ImageTk.PhotoImage(image=unknown_hand_icon_bitmap)
hand_bitmaps = []
self.hand_icons = {}
for i in range(16):
hand_bitmap = Image.open(os.path.join(hand_icon_directory, '{:04b}.png'.format(i)))
hand_bitmap.thumbnail((250, 250))
hand_bitmaps.append(hand_bitmap)
self.hand_icons[i] = ImageTk.PhotoImage(image=hand_bitmap)
self.example_thumbnails = {}
self.state = self.State.disconnected
self.serial_sniffing_thread = None
self.sample_handling_thread = None
self.queue = Queue()
self.port = None
self.receiving = False
self.last_hand_id = -1
self._config_file_version = 1
self.open_file_pathspec = ''
self.open_file_has_been_modified = False
self.change_count_since_last_save = 0
self.autosave_change_threshold = 25
self.training_set = GestureTrainingSet()
self.gesture_buffer = []
self.raw_data_buffer = []
self.current_gesture_duration = 0
self.samples_to_handle = []
self.bearing_zero = None
self.last_unprocessed_bearing_received = None
self.angular_velocity_window = deque(maxlen=10)
self.starting_velocity_estimation_buffer = deque(maxlen=10)
self.last_coordinate_visualized = None
self.master.title('Somatic Trainer')
self.pack(fill=BOTH, expand=1)
self.menu_bar = Menu(master)
self.file_menu = Menu(self.menu_bar, tearoff=0)
self.file_menu.add_command(label='New', command=self.new_file)
self.file_menu.add_command(label='Open', command=self.open_file)
self.save_entry_index = 2
self.file_menu.add_command(label='Save', command=self.save_file, state=DISABLED)
self.save_as_entry_index = 3
self.file_menu.add_command(label='Save as...', command=self.save_as, state=DISABLED)
self.menu_bar.add_cascade(label='File', menu=self.file_menu)
self.port_menu = Menu(self.menu_bar, tearoff=0, postcommand=self.populate_serial_port_menu)
self.menu_bar.add_cascade(label='Serial Port', menu=self.port_menu)
# This baloney is needed to get the selector menu checkboxes to play nice
self._serial_port_active_var = BooleanVar()
self._serial_port_active_var.set(True)
self._serial_port_inactive_var = BooleanVar()
self._serial_port_inactive_var.set(False)
master.config(menu=self.menu_bar)
left_column = Frame(self, width=250)
right_column = Frame(self)
Label(left_column, text='Hand').pack(fill=X)
self.hand_display = Canvas(left_column, width=250, height=250)
self.hand_display.create_image((0, 0), image=self.unknown_hand_icon, anchor=N + W)
self.hand_display.pack(fill=X)
Label(left_column, text='Path').pack(fill=X)
self.path_display = Canvas(left_column, width=250, height=250, bg='white')
self.path_display.pack(fill=X)
label_width_locking_frame = Frame(right_column, height=20, width=250)
label_width_locking_frame.pack_propagate(0) # This prevents a long filename from stretching the window
label_width_locking_frame.pack(fill=X, anchor=N + E)
self.file_name_label = Label(label_width_locking_frame, text='No training file open', bg='white', relief=SUNKEN)
self.file_name_label.pack(fill=BOTH, expand=1)
self.glyph_picker = ttk.Treeview(right_column, column='count')
self.glyph_picker.column("#0", width=100, stretch=False)
self.glyph_picker.column('count', stretch=True)
self.glyph_picker.heading('count', text='Count')
self.glyph_picker.pack(fill=BOTH, expand=1, anchor=S + E)
self.reload_glyph_picker()
self.glyph_picker.bind('<<TreeviewSelect>>', lambda x: self.reload_example_list())
picker_frame = Frame(right_column, bd=2, relief=SUNKEN)
picker_frame.grid_rowconfigure(0, weight=1)
picker_frame.grid_columnconfigure(0, weight=1)
self.thumbnail_canvas = Canvas(picker_frame, bd=0, width=250, height=150, bg='white')
self.thumbnail_canvas.grid(row=0, column=0, sticky=N + S + E + W)
self.thumbnail_scrollbar = Scrollbar(picker_frame)
self.thumbnail_scrollbar.grid(row=0, column=1, sticky=N + S)
self.thumbnail_canvas.configure(yscrollcommand=self.thumbnail_scrollbar.set)
self.thumbnail_scrollbar.config(command=self.thumbnail_canvas.yview)
self.thumbnail_canvas.config(yscrollcommand=self.thumbnail_scrollbar.set)
def bind_wheel_to_thumbnails(event):
self.thumbnail_canvas.bind_all('<MouseWheel>', on_wheel_scroll)
def unbind_wheel_from_thumbnails(event):
self.thumbnail_canvas.unbind_all('<MouseWheel>')
def on_wheel_scroll(event):
self.thumbnail_canvas.yview_scroll(int(event.delta / -120), 'units')
self.thumbnail_frame = Frame(self.thumbnail_canvas, bg='white')
self.thumbnail_frame.bind('<Enter>', bind_wheel_to_thumbnails)
self.thumbnail_frame.bind('<Leave>', unbind_wheel_from_thumbnails)
self.thumbnail_frame.bind('<Configure>', lambda x: self.thumbnail_canvas.configure(
scrollregion=self.thumbnail_canvas.bbox(ALL)))
self.thumbnail_canvas.create_window((0, 0), window=self.thumbnail_frame, anchor=NW)
for i in range(5):
self.thumbnail_frame.grid_columnconfigure(i, weight=1)
self.thumbnail_buttons = []
picker_frame.pack(fill=X, anchor=S + E)
self.status_line = Label(self, text='Not connected', bd=1, relief=SUNKEN, anchor=W, bg='light goldenrod')
left_column.grid(row=0, column=0, sticky=N)
right_column.grid(row=0, column=1, sticky=N + S + E + W)
self.status_line.grid(row=1, column=0, columnspan=2, sticky=S + E + W)
self.grid_columnconfigure(0, weight=0)
self.grid_columnconfigure(1, weight=1)
self.grid_rowconfigure(0, weight=1)
# Debug code!
self.model = keras.models.load_model(
'E:\\Dropbox\\Projects\\Source-Controlled Projects\\Somatic\Training Utility\\simple_model.h5')
def save_state(self):
datastore = {'port': self.port.port if self.port is not None and self.port.isOpen() else None,
'wip': self.open_file_pathspec,
'selected-glyph': self.get_selected_glyph(),
'version': self._config_file_version}
with open(os.getcwd() + 'config.json', 'w') as f:
json.dump(datastore, f)
def restore_state(self):
if not os.path.isfile(os.getcwd() + 'config.json'):
self.logger.warning('No saved configuration, starting fresh')
return
with open(os.getcwd() + 'config.json', 'r') as f:
datastore = json.load(f)
if 'version' not in datastore or datastore['version'] != self._config_file_version:
self.logger.warning('Saved config is outdated, not loading')
return
if 'wip' in datastore:
self.logger.info('Re-opening work-in-progress file {}'.format(datastore['wip']))
if self.open_file(datastore['wip']):
try:
if 'selected-glyph' in datastore and datastore['selected-glyph']:
selected_item = datastore['selected-glyph']
if selected_item is not None:
self.glyph_picker.focus(selected_item)
self.glyph_picker.selection_set(selected_item)
self.glyph_picker.see(selected_item)
self.logger.info('Resuming with glyph {} selected'.format(datastore['selected-glyph']))
except TclError:
self.logger.warning("Couldn't select glyph {} - it doesn't exist?".format(datastore['selected-glyph']))
else:
self.logger.warning("Couldn't reopen file - doesn't exist?")
if 'port' in datastore and datastore['port']:
port_names = [port.device for port in comports()]
if datastore['port'] in port_names:
self.disconnect()
self.connect_to(datastore['port'])
self.logger.info('Reconnected to glove on {}'.format(datastore['port']))
else:
self.logger.info("Couldn't reconnect to port {}".format(datastore['port']))
def reload_glyph_picker(self):
scroll_position = self.glyph_picker.yview()[0]
selected_item = self.get_selected_glyph()
self.glyph_picker.delete(*self.glyph_picker.get_children())
for glyph in GestureTrainingSet.big_ole_list_o_glyphs:
self.glyph_picker.insert('', 'end', text="Glyph '{}'".format(glyph),
value=str(self.training_set.count(glyph)) if self.training_set else '0',
iid=glyph)
if selected_item is not None:
self.glyph_picker.focus(selected_item)
self.glyph_picker.selection_set(selected_item)
self.glyph_picker.yview_moveto(scroll_position)
def get_selected_glyph(self):
selected_item = self.glyph_picker.selection()[0] if len(self.glyph_picker.selection()) else None
return selected_item
def reload_example_list(self):
for button in self.thumbnail_buttons:
button.grid_forget()
button.destroy()
del self.thumbnail_buttons[:]
selected_glyph = self.get_selected_glyph()
if selected_glyph is None or not self.training_set.count(selected_glyph):
return
for example in self.training_set.get_examples_for(selected_glyph):
self.insert_thumbnail_button_for(example)
self.thumbnail_canvas.yview_moveto(0)
def insert_thumbnail_button_for(self, example):
for button in self.thumbnail_buttons:
if button.gesture == example:
self.logger.debug('Already placed button for example w/ UUID {}'.format(example.uuid))
return
if example in self.example_thumbnails:
thumbnail = self.example_thumbnails[example]
else:
thumbnail = ImageTk.PhotoImage(
image=_gesture_to_image(
example.bearings, 50, 50, 2, 2, 2))
self.example_thumbnails[example] = thumbnail
button = Button(self.thumbnail_frame, image=thumbnail)
button.image = thumbnail # Button doesn't have an image field - this monkey patch retains a reference
button.gesture = example # Monkey-patch a reference to the gesture into the button to associate them
button.configure(command=lambda: self.visualize(example.bearings))
# Right click on OSX
button.bind('<Button-2>', lambda x: self.delete_thumbnail_button(button))
# Right click on Windows
button.bind('<Button-3>', lambda x: self.delete_thumbnail_button(button))
self.thumbnail_buttons.append(button)
position = len(self.thumbnail_buttons) - 1
button.grid(row=int(position / 5), column=int(position % 5))
if len(self.thumbnail_buttons) == 1:
# This is the first thumbnail in the list - the scrollbar won't readjust automatically.
self.thumbnail_canvas.yview_moveto(0)
def delete_thumbnail_button(self, button):
self.logger.info('Removing example for {}, UUID {}'.format(
button.gesture.glyph, str(button.gesture.uuid)))
self.training_set.remove(button.gesture)
# self.reload_example_list()
position = self.thumbnail_buttons.index(button)
button.grid_forget()
button.destroy()
self.thumbnail_buttons.remove(button)
for i in range(position, len(self.thumbnail_buttons)):
self.thumbnail_buttons[i].grid(row=int(i / 5), column=int(i % 5))
self.glyph_picker.item(button.gesture.glyph, value=self.training_set.count(button.gesture.glyph))
# We can save now! Yay!
self.open_file_has_been_modified = True
self.file_menu.entryconfigure(self.save_entry_index, state=NORMAL)
self.file_menu.entryconfigure(self.save_as_entry_index, state=NORMAL)
if self.open_file_pathspec:
self.change_count_since_last_save += 1 # Autosave. Make Murphy proud.
if self.change_count_since_last_save >= self.autosave_change_threshold:
self.save_file()
self.change_count_since_last_save = 0
def start(self):
self.master.after(10, self.queue_handler)
self.restore_state()
self.logger.debug('Gesture cone angle: {:.5f}'.format(self.gesture_cone_angle))# TODO cut this
def stop(self):
if self.open_file_has_been_modified:
response = messagebox.askyesnocancel('Unsaved changes',
'Save before quitting?')
if response:
self.save_file()
elif response is None:
return
self.state = self.State.quitting
self.receiving = False
self.save_state()
self.queue.put({'type': 'quit'})
def queue_handler(self):
try:
command = self.queue.get(block=False)
if command['type'] is 'ack':
if self.state is self.State.connecting:
self.state = self.State.connected
self.logger.info('Got ack - connected')
if command['type'] is 'viz':
path = command['path']
self.path_display.delete(ALL)
last_point = None
x_center = (max(path[:, 0] - min(path[:, 0]))) / 2
y_center = (max(path[:, 1] - min(path[:, 1]))) / 2
padding = 10
for i, coords in enumerate(path):
x_coord = (coords[0] + 0.5 - x_center) * (250 - padding * 2) + padding
y_coord = (coords[1] + 0.5 - y_center) * (250 - padding * 2) + padding
if last_point:
green = int(_scale(i, 0, len(path), 255, 0))
blue = int(_scale(i, 0, len(path), 0, 255))
self.path_display.create_line(last_point[0],
last_point[1],
x_coord, y_coord,
width=2,
fill='#00{:02X}{:02X}'.format(green, blue))
last_point = [x_coord, y_coord]
self.path_display.create_oval((x_coord - 2, y_coord - 2, x_coord + 2, y_coord + 2),
fill='SeaGreen1')
if command['type'] is 'rx':
bearing = command['bearing']
if self.state is not self.State.disconnected and self.state is not self.State.quitting:
self.update_status(command['fingers'], bearing, command['freq'])
fingers = command['fingers']
hand_id = fingers[0] * 0b1000 + fingers[1] * 0b0100 + fingers[2] * 0b0010 + fingers[3] * 0b0001
if hand_id != self.last_hand_id:
self.hand_display.create_image((0, 0), image=self.hand_icons[hand_id], anchor=N + W)
self.last_hand_id = hand_id
if self.state is self.State.recording:
x_coord = np.tan(bearing[0]) * 250
y_coord = np.tan(bearing[1]) * 250
if 0 <= x_coord <= 250 and 0 <= y_coord <= 250:
# self.logger.debug('Rendering ({}, {})'.format(x_coord, y_coord))
if self.last_coordinate_visualized:
self.path_display.create_line(self.last_coordinate_visualized[0],
self.last_coordinate_visualized[1],
x_coord, y_coord,
width=2, fill='blue')
self.last_coordinate_visualized = [x_coord, y_coord]
self.path_display.create_oval((x_coord - 2, y_coord - 2, x_coord + 2, y_coord + 2),
fill='SeaGreen1')
else:
self.logger.debug('Coordinate ({}, {}) invalid - leftover from before gesture?'
.format(x_coord, y_coord))
if command['type'] is 'quit':
self.master.destroy()
return
except Empty:
pass
self.master.after(10, self.queue_handler)
def new_file(self):
if self.state is self.State.recording:
self.state = self.State.connected
self.cancel_gesture()
if self.open_file_has_been_modified:
outcome = messagebox.askyesnocancel('Unsaved changes',
'Training data has been changed.\nSave before creating new file?')
if outcome:
self.save_file()
elif outcome is None:
return
new_file_pathspec = filedialog.asksaveasfilename(
title='Create training file',
filetypes=(('JSON dictionary', '*.json'), ('All files', '*')))
if new_file_pathspec:
new_file_pathspec += '.json'
self.open_file_pathspec = new_file_pathspec
open(new_file_pathspec, 'w') # Save empty file
self.training_set = GestureTrainingSet()
self.file_menu.entryconfigure(self.save_entry_index, state=NORMAL)
self.file_menu.entryconfigure(self.save_as_entry_index, state=NORMAL)
self.file_name_label.configure(text=self.open_file_pathspec, anchor=E)
self.open_file_has_been_modified = True
self.reload_glyph_picker()
def open_file(self, file_to_open=None):
if file_to_open is None:
file_to_open = filedialog.askopenfilename(title='Select training data',
filetypes=(('JSON dictionary', '*.json'), ('All files', '*')))
if not os.path.isfile(file_to_open):
self.logger.warning("Can't open training file because it doesn't exist - {}".format(file_to_open))
return
if file_to_open:
try:
training_set = GestureTrainingSet.load(file_to_open)
self.training_set = training_set
self.reload_glyph_picker()
self.open_file_pathspec = file_to_open
self.file_menu.entryconfigure('Save', state=NORMAL)
self.file_menu.entryconfigure('Save as...', state=NORMAL)
self.file_name_label.configure(text=self.open_file_pathspec, anchor=E)
return True
except (KeyError, AttributeError, json.decoder.JSONDecodeError) as e:
self.logger.exception('Oopsie')
messagebox.showerror("Can't load file",
"This file can't be loaded.\nError: {}".format(repr(e)))
return False
def save_file(self):
if self.state is self.State.recording:
self.state = self.State.connected
self.cancel_gesture()
if not self.training_set:
self.logger.warning('Tried to save empty training set?')
return
if not self.open_file_pathspec:
file_to_create = filedialog.asksaveasfilename(title='Save training data',
filetypes=(('JSON dictionary', '*.json'), ('All files', '*')))
if file_to_create:
self.open_file_pathspec = file_to_create + '.json'
else:
return
self.training_set.save(self.open_file_pathspec)
self.logger.info('Saved {}'.format(self.open_file_pathspec))
self.open_file_has_been_modified = False
self.change_count_since_last_save = 0
def save_as(self):
if self.state is self.State.recording:
self.state = self.State.connected
self.cancel_gesture()
if not self.training_set:
self.logger.warning('Tried to save empty training set?')
return
file_to_create = filedialog.asksaveasfilename(title='Save training data',
filetypes=(('JSON dictionary', '*.json'), ('All files', '*')))
if file_to_create:
self.open_file_pathspec = file_to_create
else:
return
self.training_set.save(self.open_file_pathspec)
self.open_file_has_been_modified = False
def populate_serial_port_menu(self):
self.port_menu.delete(0, 999)
ports = comports()
self._serial_port_active_var.set(True)
self._serial_port_inactive_var.set(False)
if ports:
for path, _, __ in ports:
if self.port and self.port.isOpen() and path == self.port.port:
self.port_menu.add_checkbutton(label=path, command=self.disconnect,
variable=self._serial_port_active_var, onvalue=True, offvalue=False)
else:
self.port_menu.add_checkbutton(label=path, command=lambda pathspec=path: self.connect_to(pathspec),
variable=self._serial_port_inactive_var, onvalue=True, offvalue=False)
else:
self.port_menu.add_command(label='No serial ports available', state=DISABLED)
# def handle_connection_button(self):
def connect_to(self, portspec):
self.disconnect()
try:
self.port = Serial(port=portspec, baudrate=115200, timeout=0.2)
if not self.port.isOpen():
self.port.open()
# self.serial_connect_button.configure(text='Disconnect')
self.state = self.State.connecting
self.status_line.configure(text='Waiting for response...', bg='DarkGoldenrod2')
self.port.write(bytes('>AT\r\n'.encode('utf-8')))
self.start_receiving()
except SerialException as e:
self.logger.exception('dafuq')
error_tokens = [x.strip("' ") for x in e.args[0].split('(')[1].split(',')]
messagebox.showinfo("Can't open {}".format(portspec), 'Error {}: {}'.format(error_tokens[0], error_tokens[1]))
self.state = self.State.disconnected
self.status_line.configure(text="Can't connect to {}".format(portspec), bg='firebrick1')
self.port = None
def disconnect(self):
if self.port:
self.logger.info('Now disconnecting')
if self.receiving:
self.receiving = False
if threading.current_thread() is not self.serial_sniffing_thread \
and self.serial_sniffing_thread \
and self.serial_sniffing_thread.is_alive():
self.logger.info('Quitting receive')
self.serial_sniffing_thread.join(timeout=1)
self.logger.info('No longer receiving')
if self.port.isOpen():
self.logger.info('Port closed')
self.port.close()
self.port = None
# self.serial_connect_button.configure(text='Connect')
self.last_hand_id = -1
self.cancel_gesture()
self.hand_display.create_image((0, 0), image=self.unknown_hand_icon, anchor=N + W)
self.state = self.State.disconnected
self.status_line.configure(text='Not connected', bg='light goldenrod')
def start_receiving(self):
if self.receiving:
self.logger.warning('Already receiving, not doing it')
return
self.receiving = True
self.logger.info('Now receiving')
self.serial_sniffing_thread = threading.Thread(target=self.handle_packets, daemon=True)
self.serial_sniffing_thread.start()
def handle_packets(self):
while self.receiving and self.port and self.port.isOpen():
try:
incoming = self.port.readline().decode()
except SerialException:
self.logger.exception('Lost serial connection, bailing out')
self.disconnect()
return
if incoming:
if self._log_parsing:
self.logger.debug('Received packet {}'.format(incoming))
# Packet format:
# >[./|],[./|],[./|],[./|],
# [float o.h],[float o.p],[float o.r],
# [float a.x], [float a.y], [float a.z], [us since last sample]
if incoming.count('>') is not 1:
if self._log_parsing:
self.logger.debug('Packet corrupt - missing delimeter(s)')
continue
# Strip crap that arrived before the delimeter, and also the delimiter itself
incoming = incoming[incoming.index('>') + 1:].rstrip()
if incoming == 'OK':
self.logger.info('Received ack')
self.queue.put({'type': 'ack'})
continue
if incoming.count(',') is not 10:
if self._log_parsing:
self.logger.debug('Packet corrupt - insufficient fields')
continue
tokens = incoming.split(',')
if self._log_parsing:
self.logger.debug('Tokens: {0}'.format(tokens))
fingers = list(map(lambda i: i is '.', tokens[:4]))
if self._log_parsing:
self.logger.debug('Fingers: {0}'.format(fingers))
bearing = np.array([float(t) for t in tokens[4:7]])
acceleration = np.array([float(t) for t in tokens[7:10]])
microseconds = float(tokens[-1])
if self._log_parsing:
self.logger.debug('Sample parsed')
self.accept_sample(fingers, bearing, acceleration, microseconds)
else:
time.sleep(0.05)
def accept_sample(self, *args):
self.samples_to_handle.append(args)
if not self.sample_handling_thread or not self.sample_handling_thread.is_alive():
self.sample_handling_thread = threading.Thread(target=self.sample_handling_loop, daemon=True)
self.sample_handling_thread.start()
def sample_handling_loop(self):
while len(self.samples_to_handle):
fingers, bearing, acceleration, microseconds = self.samples_to_handle[0]
del self.samples_to_handle[0]
self.handle_sample(fingers, bearing, acceleration, microseconds)
time.sleep(0)
def handle_sample(self, fingers, bearing, acceleration, microseconds):
"""
:type fingers: list of bool
:type bearing: np.array
:type acceleration: np.array
:type microseconds: float
"""
if microseconds == 0:
frequency = 0
else:
frequency = 1 / (microseconds / 1000000)
raw_bearing = bearing
bearing = np.array([bearing[0], bearing[1]]) # We don't care about roll
if self.last_unprocessed_bearing_received is not None:
y, p = bearing_delta(self.last_unprocessed_bearing_received, bearing)
norm = np.sqrt(sum(np.square([abs(y), abs(p)])))
if self._log_angular_velocity:
self.logger.debug(
'Last: {} Counter: {} Delta: {}'.format(self.last_unprocessed_bearing_received, bearing, [y, p]))
self.logger.debug('Norm: {}'.format(norm))
if norm:
theta = np.arcsin(norm)
else:
theta = 0
angular_velocity = theta * frequency
self.angular_velocity_window.append(angular_velocity)
in_degrees = angular_velocity * 180 / np.pi
if self._log_angular_velocity:
self.logger.debug('Angular velocity {0:.2f} deg or {1:.2f} rad/s'.format(in_degrees, angular_velocity))
self.last_unprocessed_bearing_received = bearing
velocity_threshold = 2 if self.state is self.State.recording else 4
gesture_eligible = (len([x for x in self.angular_velocity_window if x > velocity_threshold])
and (fingers == self.pointer_gesture or self.state is self.State.recording))
if self.bearing_zero is not None:
# Constrain gesture to a cone
self.logger.debug('Raw bearing: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
bearing = bearing_delta(self.bearing_zero, bearing)
# self.logger.debug('Processed bearing 1: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
bearing = np.clip(bearing,
-1 / 2 * self.gesture_cone_angle,
1 / 2 * self.gesture_cone_angle)
# self.logger.debug('Processed bearing 2: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
# Scale bearings to ML-ready 0.0-1.0 values
bearing /= self.gesture_cone_angle
# self.logger.debug('Processed bearing 3: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
bearing += 0.5
# self.logger.debug('Processed bearing 4: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
self.logger.debug('Processed bearing: ({:.3f}, {:.3f})'.format(bearing[0], bearing[1]))
if gesture_eligible:
if self.state is not self.State.recording:
self.path_display.delete(ALL)
self.last_coordinate_visualized = None
self.logger.info('Starting sample!')
self.status_line.configure(bg='SeaGreen1')
self.bearing_zero = bearing
self.logger.debug('Bearing zero is ({:.3f}, {:.3f})'.format(self.bearing_zero[0], self.bearing_zero[1]))
bearing = np.array([0.5, 0.5])
self.gesture_buffer.append(bearing)
self.state = self.State.recording
else:
self.gesture_buffer.append(bearing)
self.raw_data_buffer.append({'b': raw_bearing.tolist(), 'a': acceleration.tolist(), 't': microseconds})
self.current_gesture_duration += microseconds
else:
if self.state is self.State.recording:
self.logger.info('Sample done, {} points'.format(len(self.gesture_buffer)))
self.status_line.configure(bg='OliveDrab1')
def flash_red():
self.path_display.configure(bg='salmon1')
self.path_display.after(200, lambda: self.path_display.configure(bg='light grey'))
def flash_blue():
self.path_display.configure(bg='SeaGreen1')
self.path_display.after(200, lambda: self.path_display.configure(bg='light grey'))
def overlay_text(text):
self.path_display.create_text((5, 245), text=text, anchor=SW)
try:
processed_bearings = process_samples(np.array(self.gesture_buffer), standard_gesture_length)
for index, bearing in enumerate(processed_bearings):
self.logger.info("Point {}: ({:.4f}, {:.2f})".format(index, bearing[0], bearing[1]))
new_gesture = Gesture('', processed_bearings, deepcopy(self.raw_data_buffer))
self.visualize(processed_bearings)
except AttributeError:
self.logger.exception("Couldn't create gesture")
new_gesture = None
if new_gesture and len(self.glyph_picker.selection()) is 0:
overlay_text('Discarding - no glyph selected')
if self.current_gesture_duration > 300000:
flash_blue()
else:
flash_red()
elif new_gesture:
# Debug Code!
daters = new_gesture.bearings
print(daters)
# results = self.model.predict(daters.reshape(1, standard_gesture_length, 2))
results = self.model.predict(daters.reshape(1, standard_gesture_length * 2))
# self.logger.info(len(results[0]))
results = sorted([[chr(ord('A') + i), j] for i, j in enumerate(results[0])],
key=lambda item: item[1], reverse=True)
for index, value in results[:2]:
print('{}: {:.0f}'.format(index, value * 100))
if self.current_gesture_duration > 300000:
self.path_display.create_text((125, 125), text=results[0][0],
font=font.Font(family='Comic Sans', size=200))
selected_glyph = self.glyph_picker.selection()[0]
short_glyph = selected_glyph in GestureTrainingSet.short_glyphs
if self.current_gesture_duration > 0 and (short_glyph or self.current_gesture_duration > 300000):
min_yaw = min(sample[0] for sample in self.gesture_buffer)
max_yaw = max(sample[0] for sample in self.gesture_buffer)
min_pitch = min(sample[1] for sample in self.gesture_buffer)
max_pitch = max(sample[1] for sample in self.gesture_buffer)
tiniest_glyph = 1 / 32 * np.pi
if short_glyph or (max_yaw - min_yaw > tiniest_glyph or max_pitch - min_pitch > tiniest_glyph):
new_gesture.glyph = selected_glyph
self.training_set.add(new_gesture)
overlay_text('Accepted sample #{} for glyph {}'.format(
self.training_set.count(selected_glyph), selected_glyph))
self.path_display.configure(bg='pale green')
self.path_display.after(200, lambda: self.path_display.configure(bg='light grey'))
# We can save now! Yay!
self.open_file_has_been_modified = True
self.file_menu.entryconfigure(self.save_entry_index, state=NORMAL)
self.file_menu.entryconfigure(self.save_as_entry_index, state=NORMAL)
if self.open_file_pathspec:
self.change_count_since_last_save += 1 # Autosave. Make Murphy proud.
if self.change_count_since_last_save >= self.autosave_change_threshold:
self.save_file()
self.change_count_since_last_save = 0
self.insert_thumbnail_button_for(new_gesture)
self.glyph_picker.item(new_gesture.glyph,
value=self.training_set.count(new_gesture.glyph))
self.thumbnail_canvas.yview_moveto(1)
else:
overlay_text('Discarding - too small')
flash_red()
else:
overlay_text('Discarding - too short')
flash_red()
self.cancel_gesture()
self.state = self.State.connected
if self.queue.empty():
self.queue.put({'type': 'rx',
'fingers': fingers,
'bearing': bearing,
'freq': frequency})
def update_status(self, fingers, bearing, frequency):
def finger(value):
return '.' if value else '|'
text = 'Received {}{}{}{}_, ({:.2f},{:.2f}) at {}Hz' \
.format(finger(fingers[0]), finger(fingers[1]),
finger(fingers[2]), finger(fingers[3]),
bearing[0], bearing[1],
round(frequency))
self.status_line.configure(text=text, bg='SeaGreen1' if self.state is self.State.recording else 'OliveDrab1')
def cancel_gesture(self):
del self.gesture_buffer[:]
del self.raw_data_buffer[:]
self.current_gesture_duration = 0
self.bearing_zero = None
self.last_unprocessed_bearing_received = None
self.last_coordinate_visualized = None
def visualize(self, path):
self.queue.put({'type': 'viz', 'path': path})
if s[i][j] == y:
num += 1
else:
D[i][j] = -1
def setWall(D, x):
H = len(D)
W = len(D[0]) + 2
D = [[x] * W] + [[x] + [i for i in j] + [x] for j in D] + [[x] * W]
return D
D = setWall(D, -1)
D[1][1] = 1
Q = Queue()
Q.put([1, 1])
while Q.empty() != True:
x, y = Q.get()
t = D[x][y]
for i in (-1, 1):
for j in (0, 1):
a = j * i
b = (1 - j) * i
if D[x + a][y + b] > t + 1 or D[x + a][y + b] == 0:
D[x + a][y + b] = t + 1
Q.put([x + a, y + b])
if D[H][W] == 0:
print(-1)
else:
print(num - D[H][W])
class LongRunningTask(Thread):
""" Runs long running tasks in a background thread to prevent the GUI from becoming
unresponsive.
This is sub-classed from :class:`Threading.Thread` so check documentation there for base
parameters. Additional parameters listed below.
Parameters
----------
widget: tkinter object, optional
The widget that this :class:`LongRunningTask` is associated with. Used for setting the busy
cursor in the correct location. Default: ``None``.
"""
def __init__(self,
group=None,
target=None,
name=None,
args=(),
kwargs=None,
*,
daemon=True,
widget=None):
logger.debug(
"Initializing %s: (group: %s, target: %s, name: %s, args: %s, kwargs: %s, "
"daemon: %s)", self.__class__.__name__, group, target, name, args,
kwargs, daemon)
super().__init__(group=group,
target=target,
name=name,
args=args,
kwargs=kwargs,
daemon=daemon)
self.err = None
self._widget = widget
self._config = get_config()
self._config.set_cursor_busy(widget=self._widget)
self._complete = Event()
self._queue = Queue()
logger.debug(
"Initialized %s",
self.__class__.__name__,
)
@property
def complete(self):
""" :class:`threading.Event`: Event is set if the thread has completed its task,
otherwise it is unset.
"""
return self._complete
def run(self):
""" Commence the given task in a background thread. """
try:
if self._target:
retval = self._target(*self._args, **self._kwargs)
self._queue.put(retval)
except Exception: # pylint: disable=broad-except
self.err = sys.exc_info()
logger.debug("Error in thread (%s): %s", self._name,
self.err[1].with_traceback(self.err[2]))
finally:
self._complete.set()
# Avoid a ref-cycle if the thread is running a function with
# an argument that has a member that points to the thread.
del self._target, self._args, self._kwargs
def get_result(self):
""" Return the result from the given task.
Returns
-------
varies:
The result of the thread will depend on the given task. If a call is made to
:func:`get_result` prior to the thread completing its task then ``None`` will be
returned
"""
if not self._complete.is_set():
logger.warning(
"Aborting attempt to retrieve result from a LongRunningTask that is "
"still running")
return None
if self.err:
logger.debug("Error caught in thread")
self._config.set_cursor_default(widget=self._widget)
raise self.err[1].with_traceback(self.err[2])
logger.debug("Getting result from thread")
retval = self._queue.get()
logger.debug("Got result from thread")
self._config.set_cursor_default(widget=self._widget)
return retval
class QiushiSpider():
def __init__(self):
self.url = 'https://www.qiushibaike.com/8hr/page/{}/'
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}
self.url_q = Queue()
self.pool = Pool(5)
self.total_response_nums = 0
self.total_request_nums = 0
self.is_running = True
def makeUrlList(self):
"""构造所有的url放入队列"""
for i in range(1, 14):
self.url_q.put(self.url.format(i))
self.total_request_nums += 1 # 请求数 + 1 (url数 + 1)
def getHtml(self, url):
"""对一个url发送请求获取响应并返回"""
resp = requests.get(url, headers=self.headers)
return resp.text
def parseItem(self, html_str):
"""提取一个响应中的数据,并返回多条数据构成的list"""
html = etree.HTML(html_str)
# 先分组,再提取
div_list = html.xpath('//div[@id="content-left"]/div')
result_list = [] # 构造最终返回的结果列表
for div in div_list:
item = {}
item['name'] = div.xpath('.//h2/text()')[0].strip() # 用户昵称
item['text'] = div.xpath(
'.//div[@class="content"]/span/text()') # 主要内容
# print(item)
result_list.append(item)
# print(result_list)
return result_list
def saveResultList(self, result_list):
"""保存一个响应中的多条数据组成的列表"""
# print(result_list)
for item in result_list:
print(item)
def excute_requests_item_save(self):
"""从队列中拿出一个url,直到处理完成"""
url = self.url_q.get()
html_str = self.getHtml(url)
result_list = self.parseItem(html_str)
self.saveResultList(result_list)
self.total_response_nums += 1 # 总响应数 + 1
def _callback(self, xxx): # callback指定的函数必须接收一个参数,哪怕用不上!
print(xxx)
# xxx就是excute_requests_item_save这个函数的返回值
"""apply_async异步执行的函数执行完毕后,会把该函数返回的结果作为参数传入callback指定的函数中"""
if self.is_running:
self.pool.apply_async(self.excute_requests_item_save,
callback=self._callback)
def run(self):
"""爬虫运行逻辑"""
# 构造url队列
self.makeUrlList()
# 利用线程池中的线程异步的不断的去执行:处理一个url直到处理完毕
for i in range(5): # 这才是控制并发的规模!
self.pool.apply_async(self.excute_requests_item_save,
callback=self._callback)
while 1:
time.sleep(1) # 一定也要睡一会儿!不然太快导致醒不过来!
# 程序退出的逻辑: 总响应数 == url总数 --> 程序退出
if self.total_response_nums >= self.total_request_nums:
print('=')
print(self.total_request_nums)
print(self.total_response_nums)
print(self.url_q.qsize())
print('=')
self.is_running = False
break
print('程序结束了!')
def create(dataset,
dataset_name,
output_directory,
num_shards,
num_threads,
shuffle=True,
store_images=True):
"""Create the tfrecord files to be used to train or test a model.
Args:
dataset : [{
"filename" : <REQUIRED: path to the image file>,
"id" : <REQUIRED: id of the image>,
"class" : {
"label" : <[0, num_classes)>,
"text" : <text description of class>
},
"object" : {
"bbox" : {
"xmin" : [],
"xmax" : [],
"ymin" : [],
"ymax" : [],
"label" : []
}
}
}]
dataset_name: a name for the dataset
output_directory: path to a directory to write the tfrecord files
num_shards: the number of tfrecord files to create
num_threads: the number of threads to use
shuffle : bool, should the image examples be shuffled or not prior to creating the tfrecords.
Returns:
list : a list of image examples that failed to process.
"""
# Images in the tfrecords set must be shuffled properly
if shuffle:
random.shuffle(dataset)
# Break all images into batches with a [ranges[i][0], ranges[i][1]].
spacing = np.linspace(0, len(dataset), num_threads + 1).astype(np.int)
ranges = []
threads = []
for i in range(len(spacing) - 1):
ranges.append([spacing[i], spacing[i + 1]])
# Launch a thread for each batch.
print('Launching %d threads for spacings: %s' % (num_threads, ranges))
sys.stdout.flush()
# Create a mechanism for monitoring when all threads are finished.
coord = tf.train.Coordinator()
# Create a generic TensorFlow-based utility for converting all image codings.
coder = ImageCoder()
# A Queue to hold the image examples that fail to process.
error_queue = Queue()
threads = []
for thread_index in range(len(ranges)):
args = (coder, thread_index, ranges, dataset_name, output_directory,
dataset, num_shards, store_images, error_queue)
t = threading.Thread(target=_process_image_files_batch, args=args)
t.start()
threads.append(t)
# Wait for all the threads to terminate.
coord.join(threads)
print('%s: Finished writing all %d images in data set.' %
(datetime.now(), len(dataset)))
# Collect the errors
errors = []
while not error_queue.empty():
errors.append(error_queue.get())
print('%d examples failed.' % (len(errors), ))
return errors
class InstagramBot():
"""Selenium Automated Bot For Instagram\n
Varibles Loaded from config.ini\n
Account Credential, File Paths, Hashtag to Parse"""
def __init__(self):
#Instagram Account Login Bool
self.logged_in = False
#Thread Lock Variable
self.lock = RLock()
self.que = Queue()
#Import Config.ini and Configure
cwd = os.getcwd()
platform = sys.platform
if platform == 'win32' or 'cygwin':
config_file = (cwd + '\\config.ini')
self.xmlfile = (cwd + '\\XML.txt')
CHROME_USERDATA = (cwd + '\\chromeuserdata')
CHROMEDRIVER_PATH = (cwd + "\\chromedriver.exe")
else:
config_file = (cwd + '/config.ini')
self.xmlfile = (cwd + '/XML.txt')
CHROME_USERDATA = (cwd + '/chromeuserdata')
CHROMEDRIVER_PATH = (cwd + "/chromedriver.exe")
try:
config = configparser.ConfigParser()
config.read(config_file)
except configparser.Error:
print('''config.ini Error, Please Check File Paths \n
Make sure Chromedriver.exe is in the Current Working Directory and\n
Chromedriver and Google Chrome are at compatable versions.'''
)
#Instagram Account Username and Password
self.username = config['IG_AUTH']['USERNAME']
self.password = config['IG_AUTH']['PASSWORD']
#Logger for Instagram Account Actions- Likes, Follows, ect
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=config['LOGS']['INSTAGRAM'],
filemode='w')
self.logger = logging.getLogger(self.username)
#URL Variables
self.base_url = config['IG_URLS']['BASE']
self.login_url = config['IG_URLS']['LOGIN']
self.nav_url = config['IG_URLS']['NAV_USER']
self.tag_url = config['IG_URLS']['SEARCH_TAGS']
#Instagram Variables
self.tags = config['TAGS']['TAGS_TO_LIKE'].split(',')
self.max_likes = int(config['VARS']['MAX_LIKES'])
self.MINIMUM_NUMBER_OF_LIKES = int(
config['VARS']['MINIMUM_NUMBER_OF_LIKES'])
self.liked_pictures = 0
self.new_followers = 0
#Selenium Driver
options = webdriver.ChromeOptions()
options.add_argument('user-data-dir={}'.format(CHROME_USERDATA))
options.add_argument('--incognito')
# options.add_argument('--headless')
self.driver = webdriver.Chrome(
executable_path=CHROMEDRIVER_PATH,
service_log_path=config['LOGS']['CHROME'],
options=options)
#Populate Queue with Tags from config.ini
for item in self.tags:
self.que.put(item)
def login(self):
"""Uses credentials from config.ini to log into instagram: 3 Attempts"""
self.logger.info('Success')
connection_attempts = 0
while connection_attempts < 3:
try:
self.driver.get(self.login_url)
WebDriverWait(self.driver, 3).until(
EC.presence_of_element_located(
(By.XPATH, '//div[4]//button[1]')))
#Submit Login Credentials
self.driver.find_element_by_name('username').send_keys(
self.username)
self.driver.find_element_by_name('password').send_keys(
self.password)
time.sleep(.5)
# Click Login Button
self.driver.find_element_by_xpath(
'//button[@type=\"submit\"]').click()
WebDriverWait(self.driver, 3).until(
EC.presence_of_element_located(
(By.XPATH, '//input[@placeholder="Search"]')))
#Login Success, Print, Flip Bool, Return Thread
print("Login Success \n Username: {}\n Password: {}".format(
self.username, self.password))
self.logged_in = True
#Try Block Exception Handling
except NoSuchElementException:
connection_attempts += 1
print('XPath Parsing Error')
except Exception as ex:
connection_attempts += 1
print(f'Attempt #{connection_attempts}.')
print(f'Error connecting to {self.login_url}.')
print(ex)
time.sleep(2)
return False
def logged(self):
"""Return Bool of Chromewebdriver Instagram Login\n
Bool is Flipped at end of login function"""
return self.logged_in
def update(self):
"""Update GUI with number of Posts Liked This Session"""
return (self.liked_pictures, self.new_followers)
def queue(self):
"""Return Tag or False if empty"""
que = self.que.get()
if que != None:
return que
else:
return False
def quit(self):
"""Close Web Browser"""
self.driver.quit()
def scroll_down(self):
"""Scroll Down Webpage"""
try:
self.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
time.sleep(.5)
except:
print('Failed Scroll Down Script')
return
def nav_user(self, user=None):
"""Pass in single argument: instagram username handle \n"""
self.driver.get('{}/{}/'.format(self.base_url, user))
return
def follow_user(self, user=None):
""" Pass in single argument: instagram username handle """
#Naviagte to users page
self.driver.get(self.nav_url.format(user))
try:
WebDriverWait(self.driver, 2).until(
EC.element_to_be_clickable(
(By.XPATH,
"//button[contains(text(),'Follow')]"))).click()
time.sleep(.5)
print("Now Following: {}".format(user))
self.logger.info("Now Following: {}".format(user))
self.new_followers += 1
except NoSuchElementException:
print(
'Error Trying to Follow {} -Follow Button Not Found'.format(
user))
self.logger.info(
'Error Trying to Follow {} -Follow Button Not Found'.format(
user))
return False
return True
def unfollow_user(self, user=None):
"""Pass in single argument: instagram username handle"""
try:
#Click Unfollow and Confirm Messagebox
WebDriverWait(self.driver, 2).until(
EC.element_to_be_clickable(
(By.XPATH,
'//button[contains(text(),"Following")]'))).click()
WebDriverWait(self.driver, 2).until(
EC.element_to_be_clickable(
(By.XPATH,
'//button[contains(text(),"Unfollow")]'))).click()
print("Now Not Following: {}".format(user))
self.logger.info("Now Not Following: {}".format(user))
except NoSuchElementException:
print('Error Trying to Un-Follow {} -Un-Follow Button Not Found'.
format(user))
self.logger.info(
'Error Trying to Un-Follow {} -Un-Follow Button Not Found'.
format(user))
return False
return True
def like_hearts(self, url=None):
"""Click glyphsSpriteHeart on a given page if likes >= MINIMUM_NUMBER_OF_LIKES from config.ini """
#Like Buttons
heart = (
By.XPATH,
"//button/span[contains(@class,'glyphsSpriteHeart__outline__24__grey_9 u-__7')]"
)
# filled_heart = (By.XPATH, "//span[contains(@class,'glyphsSpriteHeart__filled__24__red_5 u-__7')]")
# XPATH for scraping number of likes the media has: First:Pictures / Second:Videos
xpath = [(By.XPATH, '//section[2]/div/div/button/span'),
(By.XPATH, '//section[2]/div/span/span')]
try:
self.logger.info('Starting URL Crawl For {}'.format(url))
self.driver.get(url)
#Find the Number of Likes the media.
for by, value in xpath:
try:
element = self.driver.find_element(by, value)
text = element.text.replace(',', "")
#text = elem_var.replace(',',"")
numbers = re.compile(r'[1-9]\d*|0\d+')
likes = re.match(numbers, text)
#If Likes >= config.ini [VARS][MINIUMUM_NUMBER_OF_LIKES] Click Like Button
if likes:
if int(likes.group(0)) >= self.MINIMUM_NUMBER_OF_LIKES:
try:
WebDriverWait(self.driver, 2).until(
EC.element_to_be_clickable(heart)).click()
print('\nLiked Picture at: {} \nLikes = {}\n'.
format(url, likes.group(0)))
self.logger.info(
'\nLiked Picture at: {} \tLikes = {}\n'.
format(url, likes.group(0)))
self.liked_pictures += 1
time.sleep(.5)
except NoSuchElementException:
self.logger.info(
'Selenium Driver Failed to Clck Like Button at {}'
.format(url))
pass
except Exception as e:
self.logger.info('Error: {}'.format(e))
except NoSuchElementException:
pass
return True
except StaleElementReferenceException as SE:
print('Failed to like picture: No Longer Attached to the DOM')
print(SE)
except ElementClickInterceptedException as ECI:
print('Failed to like picture: Overlaying Element')
print(ECI)
except Exception as Ex:
print('Selenium Page Error at {}'.format(url))
print(Ex)
def spider_scrawl(self, tag=None):
"""Selenium + BS4 Spider\n
Spider TAGS_TO_LIKE from config.ini file\n
Likes all pictures with more than 100 likes from each hashtag\n
Once queue is empty, return to GUI"""
links = []
href = []
#Get Webpage
self.driver.get(self.tag_url.format(tag))
WebDriverWait(self.driver, 3).until(
EC.presence_of_element_located((By.XPATH, '//header')))
self.scroll_down()
time.sleep(2)
try: #Get Page HTML Source, use Regex to match the URL ShortCode
text = self.driver.page_source
soup = BS(text, 'html.parser')
html = soup.find_all('a', href=True)
pattern = re.compile(r'\/p\/.{11,12}\/')
for tag in html:
match = re.match(pattern, tag.attrs['href'])
if match != None:
href.append(tag.attrs['href'])
#Debug File, may be used to crawl for usernames in later loop
with open(self.xmlfile, "w") as file:
for item in href:
file.write(item)
file.write('\n')
#Create FQDN for each shortcode, and
for item in href:
link = (self.base_url.format(item))
if link not in links:
links.append(link)
self.logger.info(
'-----{}-----\n\t\t Found Following Links:\n{}'.format(
self.driver.current_url, tuple(links)))
#Open each URL SHORTCODE, find the number of likes and like post if likes >= MINIMUM_NUMBER_OF_LIKES from config.ini
for item in links:
self.like_hearts(item)
except Exception as e:
print('HTML Page Paring Error\n Page: {}'.format(
self.driver.current_url))
print(e)
#Get next tag, or exit function if self.que == empty
print('\nFinished Spider Crawl For {}'.format(tag))
return True
