def test_exception_and_forwarded_and_processed_queues_behave_correctly_when_dummy_handler_and_bad_event(self):
events = Queue()
exceptions_q = Queue()
forward_q = Queue()
processed_event_q = Queue()
looper = EventLoop(events, DummyEventHandler(), exceptions_q=exceptions_q,
forward_q=forward_q, processed_event_q=processed_event_q)
price_thread = Thread(target=looper.start, args=[])
price_thread.start()
ev = None
events.put(ev)
sleep(2*looper.heartbeat)
looper.stop()
price_thread.join(timeout=2*looper.heartbeat)
out_event = exceptions_q.get_nowait()
self.assertIsNone(out_event.orig_event)
self.assertIsNone(ev, forward_q.get_nowait())
try:
processed_event_q.get_nowait()
self.fail('got processed event when exception happened')
except Empty:
pass
def test_max_connections_blocks(self):
"""Getting a connection should block for until available."""
import time
from copy import deepcopy
from threading import Thread
# We use a queue for cross thread communication within the unit test.
try: # Python 3
from queue import Queue
except ImportError:
from Queue import Queue
q = Queue()
q.put_nowait('Not yet got')
pool = self.get_pool(max_connections=2, timeout=5)
c1 = pool.get_connection('_')
c2 = pool.get_connection('_')
target = lambda: q.put_nowait(pool.get_connection('_'))
Thread(target=target).start()
# Blocks while non available.
time.sleep(0.05)
c3 = q.get_nowait()
self.assertEquals(c3, 'Not yet got')
# Then got when available.
pool.release(c1)
time.sleep(0.05)
c3 = q.get_nowait()
self.assertEquals(c1, c3)
class ThreadedClient(threading.Thread):
def __init__(self, host, port=None, start=False, timeout=3.0):
super(ThreadedClient, self).__init__()
self.host = host
self.port = port
self.timeout = timeout
self._q = Queue()
if start:
self.start()
def run(self, *args, **kw):
self.client = Client((self.host, self.port))
while True:
msg = self._q.get()
if msg is None:
break
addr, msg = msg
log.debug("Sending OSC msg %s, %r", addr, msg)
self.client.send(addr, *msg)
self.client.close()
def send(self, addr, *args, **kw):
self._q.put((addr, args), timeout=kw.get('timeout', self.timeout))
def close(self, **kw):
timeout = kw.get('timeout', self.timeout)
log.debug("Emptying send queue...")
while True:
try:
self._q.get_nowait()
except QueueEmpty:
break
if self.is_alive():
log.debug("Signalling OSC client thread to exit...")
self._q.put(None, timeout=timeout)
log.debug("Joining OSC client thread...")
self.join(timeout)
if self.is_alive():
log.warning("OSC client thread still alive after join().")
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def downloads(urls, outputs=[], concurrency=cpu_count()):
# 用于线程同步的队列
exit_queue = Queue(1)
job_queue = Queue()
result_queue = Queue()
# 创建下载任务，并加入到任务队列
outputs = [None for _ in urls] if not outputs else outputs
for url, output in zip(urls, outputs):
job_queue.put(Param(url, output))
job_size = job_queue.qsize()
works = []
# 创建工作线程并启动
concurrency = job_size if concurrency > job_size else concurrency
for _ in range(concurrency):
t = Worker(job_queue, result_queue, exit_queue)
works.append(t)
t.start()
# 检测任务是否完成，主要有两种情况
# 1.所有任务都执行了
# 2.用户主动按ctrl+c结束任务,这里会等待已经运行的任务继续运行
alive = True
try:
while alive:
for work in works:
if work.isAlive():
alive = True
break
else:
alive = False
if result_queue.qsize() == job_size and exit_queue.qsize() == 0:
exit_queue.put(1)
except KeyboardInterrupt:
logger.warning("ctrl + c is precessed!wait running task to complate..")
exit_queue.put(1)
for work in works:
if work.isAlive():
work.join()
# 结果收集并返回
results = []
while job_queue.qsize() > 0:
param = job_queue.get_nowait()
results.append(Result(False, "task not excute", param.url))
while result_queue.qsize() > 0:
result = result_queue.get_nowait()
results.append(result)
return results
def test_forwarded_and_processed_event_should_be_same_as_input_event_when_dummy_handler(self):
events = Queue()
forward_q = Queue()
processed_event_q = Queue()
looper = EventLoop(events, DummyEventHandler(), forward_q=forward_q, processed_event_q=processed_event_q)
price_thread = Thread(target=looper.start, args=[])
price_thread.start()
ev = 'dummy event'
events.put(ev)
sleep(2*looper.heartbeat)
looper.stop()
price_thread.join(timeout=2*looper.heartbeat)
self.assertEqual(ev, forward_q.get_nowait())
self.assertEqual(ev, processed_event_q.get_nowait())
class PeekableIterator(object):
def __init__(self, stream):
self._stream = iter(stream)
self._peeked = Queue()
def __iter__(self):
return self
def __next__(self):
try:
return self._peeked.get_nowait()
except Empty:
return next(self._stream)
def peek(self, n=1):
peeked = tuple(islice(self, None, n))
put = self._peeked.put_nowait
for item in peeked:
put(item)
return peeked
def lookahead_iter(self):
while True:
yield from self.peek(1)
try:
next(self)
except StopIteration:
break
def run(self):
ON_POSIX = 'posix' in sys.builtin_module_names
def enqueue_output(out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
commands = self.make_gatt_commands()
proc = subprocess.Popen([commands],stdout=subprocess.PIPE, stdin=subprocess.PIPE, close_fds=ON_POSIX, shell=True)
q = Queue()
t = Thread(target=enqueue_output, args=(proc.stdout, q))
t.daemon = True # thread dies with the program
t.start()
while True:
try:
line = q.get_nowait().decode()
# print(str(datetime.datetime.now()), line.decode())
self._parse_notification(line)
except Empty:
# print('no output yet')
pass
time.sleep(1)
class ConnectionPool(object):
def __init__(self, max_connections=5):
self.q = Queue()
self.max_connections = max_connections
self._created_connections = Counter()
self.connection_class = Connection
self.connection_kwargs = {}
def configure(self, max_connections=5, **connection_kwargs):
self.max_connections = max_connections
self.connection_kwargs = connection_kwargs
def get(self):
try:
return self.q.get_nowait()
except Empty:
if self._created_connections.current() < self.max_connections:
conn = self.connection_class(**self.connection_kwargs).conn
self._created_connections.incr()
return conn
raise
def put(self, connection):
self.q.put(connection)
self._created_connections.decr()
def created(self):
return self._created_connections.current()
def download_cover(self, log, result_queue, abort, # {{{
title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False):
cached_url = self.get_cached_cover_url(identifiers)
if cached_url is None:
log.info('No cached cover found, running identify')
rq = Queue()
self.identify(log, rq, abort, title=title, authors=authors,
identifiers=identifiers)
if abort.is_set():
return
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
results.sort(key=self.identify_results_keygen(
title=title, authors=authors, identifiers=identifiers))
for mi in results:
cached_url = self.get_cached_cover_url(mi.identifiers)
if cached_url is not None:
break
if cached_url is None:
log.info('No cover found')
return
if abort.is_set():
return
br = self.browser
log('Downloading cover from:', cached_url)
try:
cdata = br.open_novisit(cached_url, timeout=timeout).read()
result_queue.put((self, cdata))
except:
log.exception('Failed to download cover from:', cached_url)
def findFirstTri(vert, lastMFace, mirrorMesh, searchData) : # # Itterates through all faces in the mirror mesh and tests for intersection, first intersecting adjacent face connected to the initial search face is returned # faceQueue = Queue() #Tag keep track on what face we have tested / in queue. False for not tested taggedFaces = [] #Start testing from the initial face! lastMFace.tag = True faceQueue.put_nowait(lastMFace) taggedFaces.append(lastMFace) while not faceQueue.empty() : face = faceQueue.get_nowait() mDat = MirrorMesh.triIntersection(vert.co, face) if mDat is not None and mDat._intersected : searchData[vert.index].setMirror(mDat) break #we found an intersecting tri #Queue connected faces MirrorMesh.queueConnectedFaces(face, mirrorMesh, faceQueue, taggedFaces) for f in taggedFaces : f.tag = False
class Kernel(Thread):
def __init__(self):
Thread.__init__(self)
self.programsQueue = Queue()
self.isFirstLoad = True
self.shouldShutDown= False
def initializeKernel(self, clock, programloader, scheduler):
self.programLoader = programloader
self.scheduler = scheduler
self.clock = clock
def load(self, program):
# Sets a program that the program loader will load to the memory
self.programsQueue.put_nowait(program)
def run(self):
Thread.run(self)
while not self.shouldShutDown:
if not self.programsQueue.qsize() == 0:
program = self.programsQueue.get_nowait()
self.isFirstLoad = len(self.programLoader.pcbTable.pcbs) == 0
self.programLoader.load(program)
if self.isFirstLoad:
self.scheduler.setNextPcbToCpu()
class CarCommandHandler(WebSocketHandler):
need_background = True
def connect_handler(self, request):
self.car_id = int(request.match_info['car_id'])
self.car = core.models.Car.objects.get(id=self.car_id)
print('websocket connection started for car ID', self.car_id)
self.command_queue = Queue()
async def process_msg(self, msg_text: str):
# print('Got', msg_text, now())
self.command_queue.put(msg_text)
async def background(self, ws: web.WebSocketResponse):
commander = car_connector.Commander(self.car)
while not ws.closed:
try:
command = self.command_queue.get_nowait()
commander.send_command(command)
ws.send_str(command)
# print('Response:', response, now())
self.command_queue.task_done()
except Empty:
await asyncio.sleep(0.01)
print('Close background for car ID', self.car_id)
class TestStreamTradingRandom(unittest.TestCase):
def setUp(self):
self.events = Queue()
self.execution_q = Queue()
self.strategy = StrategyOrderManager(BuyOrSellAt5thTickStrategy(), 100)
self.executor = DummyExecutor()
self.algo_trader = AlgoTrader(None, self.strategy, self.executor)
self.trader = EventLoop(self.events, self.algo_trader)
self.trader.started = True
def test_order_generation_in_separate_queue(self):
self.trader.processed_event_q = self.execution_q
for i in range(1, 7):
tick = TickEvent("EUR_GBP", get_time(), round(0.87 + (i / 100), 2), 0.88 + (i / 100))
self.events.put(tick)
self.trader.pull_process()
order = self.execution_q.get_nowait()
self.assertEquals(tick.instrument, order.instrument)
self.assertEquals(EVENT_TYPES_ORDER, order.TYPE)
self.assertEquals(self.strategy.units, order.units)
def test_order_generation_in_same_queue(self):
self.trader.processed_event_q = self.events
for i in range(1, 6):
tick = TickEvent("EUR_GBP", get_time(), round(0.87 + (i / 100), 2), 0.88 + (i / 100))
self.events.put(tick)
self.trader.pull_process()
# order should be generated at the 6th tick
order = self.executor.get_last_event()
self.assertEquals(tick.instrument, order.instrument)
self.assertEquals(EVENT_TYPES_ORDER, order.TYPE)
self.assertEquals(self.strategy.units, order.units)
class Metric(object):
"""
This class stores generic time-series data in a queue.
Values are stored as (timestamp, value) tuples
"""
def __init__(self):
self.metric = Queue()
def push(self, value, timestamp=None):
if timestamp is None:
timestamp = int(time.time())
elif not isinstance(timestamp, int):
raise ValueError(
"Timestamp should be an integer, but it is '%s'" %
type(timestamp))
self.metric.put((timestamp, value))
def next(self):
try:
return self.metric.get_nowait()
except Empty:
raise StopIteration
def get(self):
# TODO: decide what we should return here
return None
def __iter__(self):
return self
def download_cover( # {{{
self,
log,
result_queue,
abort,
title=None,
authors=None,
identifiers={},
timeout=30,
get_best_cover=False
):
cached_url = self.get_cached_cover_url(identifiers)
if cached_url is None:
log.info('No cached cover found, running identify')
rq = Queue()
self.identify(
log,
rq,
abort,
title=title,
authors=authors,
identifiers=identifiers
)
if abort.is_set():
return
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
results.sort(
key=self.identify_results_keygen(
title=title, authors=authors, identifiers=identifiers
)
)
for mi in results:
cached_url = self.get_cached_cover_url(mi.identifiers)
if cached_url is not None:
break
if cached_url is None:
log.info('No cover found')
return
br = self.browser
for candidate in (0, 1):
if abort.is_set():
return
url = cached_url + '&zoom={}'.format(candidate)
log('Downloading cover from:', cached_url)
try:
cdata = br.open_novisit(url, timeout=timeout).read()
if cdata:
if hashlib.md5(cdata).hexdigest() in self.DUMMY_IMAGE_MD5:
log.warning('Google returned a dummy image, ignoring')
else:
result_queue.put((self, cdata))
break
except Exception:
log.exception('Failed to download cover from:', cached_url)
class ElasticDataSink:
def __init__(self, name, conn, model_identifier, workers=5, bound=10000):
self.name = name
self.conn = conn
self.model_identifier = model_identifier
self.queue = Queue(maxsize=bound)
self.pool = ThreadPoolExecutor(max_workers=workers)
def start(self):
self.model_identifier.model_class.init(using=self.conn)
def __sink_item(self):
try:
item = self.queue.get_nowait()
save_status = item.save(using=self.conn)
if not save_status:
logger.error("Error saving the item to the sink")
else:
logger.info("item saved to the sink")
except Empty as e:
logger.warn("sink queue is empty")
logger.warn(e)
def sink_item(self, item):
assert isinstance(item, self.model_identifier.model_class), \
" item must be instance of " + self.model_identifier.model_class
try:
self.queue.put(item, timeout=10)
self.pool.submit(self.__sink_item)
except Full as e:
logger.error("sink queue is full")
logger.error(e)
def breadthFirstExplore(maze,x,y):
'''explore a maze in a breadth first manner'''
qLen=len(maze[0][:])*len(maze)
pointQueue=Queue(qLen)
pointQueue.put_nowait((x,y,qLen))
goal=None
maze[x][y]=qLen
while not pointQueue.empty():
i=pointQueue.get_nowait()
x=i[0]
y=i[1]
k=i[2]
val=getMazeValue(maze, x, y)
if val==2:
goal=(x,y)
return goal
else:
maze[x][y]=k
#explore the neighborhood
a=getMazeValue(maze, x+1, y)
b=getMazeValue(maze, x, y+1)
c=getMazeValue(maze, x-1, y)
d=getMazeValue(maze, x, y-1)
if a==0 or a ==2:
pointQueue.put((x+1,y,k-1))
if b==0 or b==2:
pointQueue.put((x,y+1,k-1))
if c==0 or c==2:
pointQueue.put((x-1,y,k-1))
if d==0 or d==2:
pointQueue.put((x,y-1,k-1))
return goal
class TestStreamTrading(unittest.TestCase):
def setUp(self):
self.events = Queue()
self.execution_q = Queue()
self.strategy = StrategyOrderManager(DummyBuyStrategy(), 100)
self.executor = DummyExecutor()
self.algo_trader = AlgoTrader(None, self.strategy, self.executor)
self.trader = EventLoop(self.events, self.algo_trader)
self.trader.started = True
def test_order_generation_in_separate_queue(self):
self.trader.processed_event_q = self.execution_q
tick = TickEvent('EUR_GBP', get_time(), 0.87, 0.88)
self.events.put(tick)
self.trader.pull_process()
order = self.execution_q.get_nowait()
self.assertEquals(tick.instrument, order.instrument)
self.assertEquals(EVENT_TYPES_ORDER, order.TYPE)
self.assertEquals(self.strategy.units, order.units)
def test_order_generation_in_same_queue(self):
self.trader.processed_event_q = self.events
tick = TickEvent('EUR_GBP', get_time(), 0.87, 0.88)
self.events.put(tick)
self.trader.pull_process()
order = self.executor.get_last_event()
self.assertEquals(tick.instrument, order.instrument)
self.assertEquals(EVENT_TYPES_ORDER, order.TYPE)
self.assertEquals(self.strategy.units, order.units)
class LogReader: """ Tails a log file and allows lines to be retreived in a non-blocking way. If windows: needs cygwin (or just tail.exe) in path needs set CYGWIN=nodosfilewarning environment variable """ def __init__(self, filePath): self.proc = subprocess.Popen( ["tail", "--sleep-interval=0.25", "-F", filePath], stdout=subprocess.PIPE, universal_newlines=True ) self.queue = Queue() self.thread = Thread( target=self.enqueue_output, args=( self.proc.stdout, self.queue ) ) self.thread.daemon = True self.thread.start() def enqueue_output( self, pipe, queue ): for line in iter( pipe.readline, b'' ): queue.put( line ) pipe.close() def get_line( self ): line = None try: line = str( self.queue.get_nowait() ) except Empty: pass return line def kill( self ): self.proc.terminate()
class InMemoryBufferedLogSubscriber(LogSubscriber):
@property
def enabled(self):
return self._enabled
@enabled.setter
def enabled(self, value):
self._enabled = value
@property
def is_buffer_full(self):
return self._log_entry_queue.qsize() >= self._max_buffer_size
def __init__(self, max_buffer_size):
self._log_entry_queue = Queue()
self._max_buffer_size = max_buffer_size
self._enabled = True
def log(self, log_entry):
if not self._enabled or self.is_buffer_full:
return
self._log_entry_queue.put(log_entry)
def flush(self):
items = []
for i in range(self._max_buffer_size):
try:
items.append(self._log_entry_queue.get_nowait())
except Empty:
break
return items
def close(self):
pass # Intentionally blank
class ThreadHandler():
"""
Class for running calculations in a seperate thread
"""
def __init__(self):
self._resultsQueue = Queue()
self._currentThreadID = 0
self._currentCalculationType = None
self._currentResult = None
def startCalculation(self, calculationType, args):
"""
Creates a WorkerThread to carry out func(*args). Will cause any
previous still running calculations to be cancelled
"""
if calculationType == Model.EXP:
function = exponentialModelAnalysis
elif calculationType == Model.POW:
function = powerLawModelAnalysis
elif calculationType == Model.WEI:
function = weibullModelAnalysis
self._currentThreadID += 1
self._currentCalculationType = calculationType
newThread = WorkerThread(function, args, self._calculationFinished, self._resultsQueue, self._currentThreadID)
newThread.setDaemon(True)
newThread.start()
def _calculationFinished(self):
"""
Called internally by the WorkerThreads. If the calculation was cancelled
the result is ignored, otherwise if it ended in an error the result type is changed to error,
and the result of the calculation is stored in _currentResult ready for retrieval
"""
threadID, results = self._resultsQueue.get_nowait()
if threadID == None:
self._currentResult = ("Error",results)
elif threadID == self._currentThreadID:
self._currentResult = (self._currentCalculationType,results)
def cancelLastCalculation(self):
"""
Tells the ThreadHandler to cancel the last calculation
At the moment this involves leaving the calculation to run and
ignoring the result when it is returned
"""
self._currentThreadID += 1
def getCurrentCalculationResult(self):
"""
Returns the result of the last calculation if finished
otherwise returns None
"""
if self._currentResult is not None:
result = self._currentResult
self._currentResult = None
return result
else:
return None
def manage(worker, n_threads = 10, catalogs = download.catalogs):
'Manage a bunch of worker threads, and generate their results.'
# Download in random order
args = []
for catalog in catalogs:
for dataset in download.datasets(catalog):
args.append((catalog, dataset))
random.shuffle(args)
read_queue = Queue()
for a in args:
read_queue.put(a)
write_queue = Queue()
threads = []
for i in range(n_threads):
threads.append(Thread(target = worker, args = (read_queue,write_queue)))
threads[-1].start()
while not (read_queue.empty() and write_queue.empty() and set(t.is_alive() for t in threads) == {False}):
try:
x = write_queue.get_nowait()
except Empty:
pass
else:
yield x
class FakeBot:
def __init__(self, *args, **kwargs):
self.next_message_id = next_message_id()
self.next_update_id = next_message_id()
self.user = telegram.User(1234567890, 'Unittest')
self.updates = Queue()
def getUpdates(self, last_update_id, *args, **kwargs):
updates = []
try:
while not self.updates.empty():
updates.append(self.updates.get_nowait())
except Empty:
pass
return updates
def sendMessage(self, chat_id, message, *args, **kwargs):
chat = telegram.Chat(chat_id, telegram.Chat.SUPERGROUP)
message = telegram.Message(next(self.next_message_id), self.user,
datetime.datetime.now(), chat)
return message
def sendSticker(self, chat_id, *args, **kwargs):
pass
def sendLocation(self, chat_id, *args, **kwargs):
pass
def add_update(self, chat_id, text):
chat = telegram.Chat(chat_id, telegram.Chat.SUPERGROUP)
user = telegram.User(1234, 'test')
message = telegram.Message(next(self.next_message_id), user,
datetime.datetime.now(), chat, text=text)
update = telegram.Update(next(self.next_update_id), message=message)
self.updates.put_nowait(update)
def test_should_allow_to_read_string_journals(self):
filename = os.path.join(OUTPUT_DIR, 'journal_read_ut.txt')
try:
os.remove(filename)
except OSError:
pass
print('writing..')
journaler = FileJournaler(full_path=filename)
journaler.start()
event = 'this is a test event #1'
journaler.log_event(get_time(), event)
sleep(0.2)
journaler.close()
print('reading..')
eq = Queue()
reader = FileJournalerReader(eq, full_path=filename)
reader.read_events()
try:
message = eq.get_nowait()
self.assertEqual(event, message)
except Empty:
pass
class TurtleCanvas(tk.Canvas):
def __init__(self, master, *args, **kwargs):
tk.Canvas.__init__(self, master, *args, **kwargs)
self.pil_image = None
self._q = Queue()
self._cancel_current = Event()
self.update_this()
def begin_new(self):
self._q = Queue()
self._q.put((None, None))
def queue_line(self, line, color):
self._q.put((line, color))
def update_this(self):
try:
for i in range(4000): # draw 2000 lines at a cycle
line, color = self._q.get_nowait()
if line is None:
self.delete('all')
continue
self.create_line(*line, fill=color)
except Empty:
pass
self.update_idletasks()
self.after(5, self.update_this)
def test_sigpipe(self):
r, w = os.pipe()
outstream = os.fdopen(w, 'wb')
task = self.create_task(self.context(console_outstream=outstream))
raised = Queue(maxsize=1)
def execute():
try:
task.execute()
except IOError as e:
raised.put(e)
execution = threading.Thread(target=execute, name='ConsoleTaskTestBase_sigpipe')
execution.setDaemon(True)
execution.start()
try:
data = os.read(r, 5)
self.assertEqual(b'jake\n', data)
os.close(r)
finally:
task.stop()
execution.join()
with self.assertRaises(Empty):
e = raised.get_nowait()
# Instead of taking the generic assertRaises raises message, provide a more detailed failure
# message that shows exactly what untrapped error was on the queue.
self.fail('task raised {0}'.format(e))
class StreamHandler(object):
def __init__(self, stream, name=None):
self.stream = stream
self.name = name
self.alive = True
self.out = Queue()
if self.name is None:
def add():
while self.alive:
line = self.stream.readline()
if line:
self.out.put(line)
else:
self.out.put('\n')
self.alive = False
break
else:
def add():
with open('/tmp/%s.log' % self.name, 'w') as log:
while self.alive:
line = self.stream.readline()
if line:
print(line, file=log)
self.out.put(line)
else:
self.out.put('\n')
print('Finished %s' % self.name, file=log)
self.alive = False
break
self.thd = Thread(target=add)
self.thd.daemon = True
self.thd.start()
def readline(self, timeout=None):
try:
if self.alive:
return self.out.get(block=True, timeout=timeout)
return self.out.get_nowait()
except Empty:
return None
def kill(self):
self.alive = False
def clear(self):
while self.out.qsize() > 0:
self.out.get_nowait()
def run_job_in_thread(jobs):
'''
Manage paralell launch
jobs - dictionary of products to run {prod:[env,mode]}
'''
func_name=run_job_in_thread.__name__
try:
job_queue = Queue()
res_queue = Queue()
#stdout and stderr of subprocess
stdstr_queue = Queue()
start=time.time()
#print "DEBUG function %s job_queue=" %(func_name), job_queue,res_queue
#Determine Number of threads to use, but max out at 25
if len(jobs) < 25:
num_threads = len(jobs)
else:
num_threads = 25
#Start thread pool
for i in range(num_threads):
#for cmd in cmds:
# for job,params in jobs.iteritems():
#print "DEBUG func_name num_threads is " ,job,params
#target is a function to launch
#in my case getResponseparams1 status=getResponseparams1 (prod,env,oper)
# Creates same workers which will take parameteres to launch from the job_queue
worker = Thread(target=getResponseparams, args=(job_queue,res_queue,stdstr_queue))
worker.setDaemon(True)
worker.start()
logger.info('Main Thread Waiting')
for job,params in jobs.items():
logger.info("DEBUG %s job %s,params %s: %s " % (func_name ,job,params[0],params[1]))
#job - product,params[0] - environmnet ,params[1] = operation
job_queue.put([job,params[0],params[1]])
job_queue.join() #wait for all jobs'll come to finish
end=time.time()
logger.info( "Dispatch Completed in %s seconds" % (end-start))
items_to_send=[]
#get result from queue
while res_queue.empty() != True:
#items_to_send.append([prod,env,status])
items_to_send.append(res_queue.get_nowait())
logger.info( "DEBUG %s %s" % (func_name,items_to_send) )
return items_to_send,stdstr_queue
except Exception as err:
logger.error( " %s: %s" % (func_name, err))
finally:
pass
class InsnPool:
def __init__(self, proc, max_threads = None):
self.numThreads = 0
if max_threads is None:
max_threads = multiprocessing.cpu_count() * 5
self.max_threads = max_threads
self.queue = Queue()
self.proc = proc
# A lock used to wake up the polling thread if asynchronous
self.lock = threading.Semaphore()
def query(self, insn):
self.queue.put(insn)
def signal(self):
self.numThreads -= 1
if self.has_cycled():
# Wake up the polling thread if batch is done processing
self.lock.release()
def has_cycled(self):
return self.numThreads == 0
def has_finished(self):
return self.has_cycled() and self.queue.empty()
def poll_all(self, blocking = True, callback = None):
if not blocking and callback is None:
raise ValueError("If called in a non blocking way you must provide a callback function.")
def poll_all_impl():
while not self.has_finished(): # Wait for all threads to process
self.poll()
self.lock.acquire() # Pauses the current thread and waits till batch is processed
if callback:
callback()
if blocking:
poll_all_impl()
else:
thread = threading.Thread(daemon = True, target = poll_all_impl)
thread.start()
def poll(self):
# Batch processing, we only start a new batch when the old
# one has finished. It is only possible to jump to a location once.
if not self.has_cycled(): return
locations = set()
while self.numThreads < self.max_threads and not self.queue.empty():
insn = self.queue.get_nowait()
if insn.pc in locations:
continue
else:
locations.add(insn.pc)
insn.start()
self.numThreads += 1
class QueueCommandManager(CommandManager):
"""
A command manager that takes a queue of functions that act on
the tracer and apply them one at a time with each call to next_command().
"""
def __init__(self):
super(QueueCommandManager, self).__init__()
self.queue = Queue()
self.sent = []
def enqueue(self, fn):
"""
Enqueues a new message to be consumed.
"""
self.queue.put(fn)
def user_wait(self, duration):
"""
Simulates waiting on the user to feed us input for duration seconds.
"""
self.enqueue(lambda t: sleep(duration + int(PY3)))
def clear(self):
"""
Clears the internal list of functions.
"""
self.queue = Queue()
def user_next_command(self, tracer):
"""
Removes one message from the internal queue and apply it to the
debugger.
"""
try:
self.queue.get_nowait()(tracer)
except Empty:
return
def send(self, msg):
# Collect the output so that we can make assertions about it.
self.sent.append(json.loads(msg))
user_stop = clear
def start(self, tracer, auth_msg=''):
pass
class VideoStream:
default_fps = 30.
def __init__(self, stream_source, interval=0.5):
'''
Parameters:
stream_source: RTSP, camera index, or video file name
self.interval: how long to wait before next frame is served (sec)
'''
if stream_source == "":
raise ValueError("stream cannot be empty")
if interval <= 0 or interval >= 24 * 3600:
raise ValueError(
"pulse interval should be positive, shorter than a day")
self.keep_listeing_for_frames = True
self.frame_queue = Queue(100)
self.cam = stream_source
self.interval = interval
self.frame_grabber = None
self.is_rtsp = self.cam.lower().startswith('rtsp')
self.fps = None
self.delay_frames = None
self.delay_time = None
self.video_capture = None
def stop(self):
self.keep_listeing_for_frames = False
if self.frame_grabber is None:
return
try:
if self.frame_grabber.is_alive():
self.frame_grabber.join(1)
self.frame_grabber = None
logging.info("Stopped grabbing frames")
except:
logging.critical("Error while stopping thread")
def reset(self, stream_source, interval):
'''
Any change to stream source or interval will re-set streaming
'''
if stream_source == "":
raise ValueError("stream cannot be empty")
if interval <= 0 or interval >= 24 * 3600:
raise ValueError(
"pulse interval should be positive, shorter than a day")
self.stop()
self.cam = stream_source
self.interval = interval
self.start()
def start(self):
if self.frame_grabber is not None:
self.stop()
self.keep_listeing_for_frames = True
self.frame_grabber = threading.Thread(target=self.stream_video)
self.frame_grabber.daemon = True
self.frame_grabber.start()
logging.info(f"Started listening for {self.cam}")
def get_frame_with_id(self):
'''
Retrieves the frame together with its frame id
'''
try:
frame_and_id = self.frame_queue.get_nowait()
except Empty:
frame_and_id = (-1, None)
return frame_and_id
def setup_stream(self):
self.video_capture = cv2.VideoCapture(self.cam)
# retrieve camera properties.
# self.fps may not always be available
# TODO: Need to support frame counting for RTSP!
self.fps = self.video_capture.get(cv2.CAP_PROP_FPS)
if self.fps is not None and self.fps > 0:
self.delay_frames = int(round(self.interval * self.fps))
logging.info(f"Retrieved FPS: {self.fps}")
else:
self.delay_time = self.interval
def stream_video(self):
repeat = 3
wait = 0.1
frame = None
cur_frame = 0
# this is used for frame delays if the video is on an infinite loop
continuous_frame = 0
# will create a new video capture and determine streaming speed
self.setup_stream()
while self.keep_listeing_for_frames:
start_time = time.time()
for _ in range(repeat):
try:
res, frame = self.video_capture.read()
if not res:
self.video_capture = cv2.VideoCapture(self.cam)
res, frame = self.video_capture.read()
cur_frame = 0
continuous_frame = 0
break
except:
# try to re-capture the stream
logging.info(
"Could not capture video. Recapturing and retrying...")
time.sleep(wait)
if frame is None:
logging.info("Failed to capture frame, sending blank image")
continue
# if we don't know how many frames we should be skipping
# we defer to
if self.delay_frames is None and not self.is_rtsp:
cur_delay = self.delay_time - time.time() + start_time
if cur_delay > 0:
time.sleep(cur_delay)
# we are reading from a file, simulate 30 self.fps streaming
# delay appropriately before enqueueing
cur_frame += 1
continuous_frame += 1
if self.delay_frames is not None and (continuous_frame -
1) % self.delay_frames != 0:
continue
self.frame_queue.put((cur_frame, frame))
self.video_capture.release()
self.video_capture = None
class Run_and_log_command(QWidget):
"""
run a command and send a signal when it complete, or it has failed.
use a Qt timer to check the process
realtime streaming of the terminal output has so proved to be fruitless
"""
finished_signal = pyqtSignal(bool)
display_signal = pyqtSignal(str)
def __init__(self):
super(Run_and_log_command, self).__init__()
self.polling_interval = 10.
self.initUI()
def initUI(self):
"""
Just setup a qlabel showing the shell command
and another showing the status of the process
"""
# Make a grid layout
#layout = QGridLayout()
hbox = QHBoxLayout()
# add the layout to the central widget
self.setLayout(hbox)
# show the command being executed
self.command_label0 = QLabel(self)
self.command_label0.setText('<b>Command:</b>')
self.command_label = QLabel(self)
#self.command_label.setMaximumSize(50, 250)
# show the status of the command
self.status_label0 = QLabel(self)
self.status_label0.setText('<b>Status:</b>')
self.status_label = QLabel(self)
# add to layout
hbox.addWidget(self.status_label0)
hbox.addWidget(self.status_label)
hbox.addWidget(self.command_label0)
hbox.addWidget(self.command_label)
hbox.addStretch(1)
def run_cmd(self, cmd):
from subprocess import PIPE, Popen
import shlex
self.command_label.setText(cmd)
self.status_label.setText('running the command')
self.p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE)
self.q = Queue()
t = Thread(target=enqueue_output, args=(self.p.stdout, self.q))
# make sure the thread dies with the program
t.daemon = True
t.start()
# start a Qt timer to update the status
QTimer.singleShot(self.polling_interval, self.update_status)
def update_status(self):
status = self.p.poll()
if status is None:
self.status_label.setText('Running')
# non blocking readline
try:
line = self.q.get_nowait()
sys.stdout.buffer.write(line)
sys.stdout.flush()
# emmit a signal on 'display:'
line = line.decode("utf-8")
if 'display:' in line:
self.display_signal.emit(line.split('display:')[1].strip())
except Empty:
pass
# start a Qt timer to update the status
QTimer.singleShot(self.polling_interval, self.update_status)
elif status is 0:
self.status_label.setText('Finished')
# non blocking readline
try:
line = self.q.get_nowait()
sys.stdout.buffer.write(line)
sys.stdout.flush()
# emmit a signal on 'display:'
line = line.decode("utf-8")
if 'display:' in line:
self.display_signal.emit(line.split('display:')[1].strip())
except Empty:
pass
# emmit a signal when complete
self.finished_signal.emit(True)
else:
self.status_label.setText(str(status))
# get the output and error msg
for line in iter(self.p.stderr.readline, b''):
sys.stdout.buffer.write(line)
sys.stdout.flush()
# emmit a signal when complete
self.finished_signal.emit(False)
class RobotState:
"""
Class responsible for inner-thread communication. All jobs coming from the robot
are stored in this class to be retrieved by the simulator for updating/rendering
of the simulated robot.
"""
def __init__(self):
cfg = get_config().get_data()
large_sim_type = get_config().is_large_sim_type()
self.address_motor_center = cfg['alloc_settings']['motor']['center'] if large_sim_type else ''
self.address_motor_left = cfg['alloc_settings']['motor']['left']
self.address_motor_right = cfg['alloc_settings']['motor']['right']
self.center_motor_queue = Queue()
self.left_motor_queue = Queue()
self.right_motor_queue = Queue()
self.sound_queue = Queue()
self.left_brick_left_led_color = 1
self.left_brick_right_led_color = 1
self.right_brick_left_led_color = 1
self.right_brick_right_led_color = 1
self.should_reset = False
self.values = {}
self.locks = {}
self.motor_lock = threading.Lock()
def put_center_motor_job(self, job: float):
"""
Add a new move job to the queue for the center motor.
:param job: to add.
"""
self.center_motor_queue.put_nowait(job)
def put_left_motor_job(self, job: float):
"""
Add a new move job to the queue for the left motor.
:param job: to add.
"""
self.left_motor_queue.put_nowait(job)
def put_right_motor_job(self, job: float):
"""
Add a new move job to the queue for the right motor.
:param job: to add.
"""
self.right_motor_queue.put_nowait(job)
def next_motor_jobs(self) -> Tuple[float, float, float]:
"""
Get the next move jobs for the left and right motor from the queues.
:return: a floating point numbers representing the job move distances.
"""
self.motor_lock.acquire()
try:
center = self.center_motor_queue.get_nowait()
except Empty:
center = None
try:
left = self.left_motor_queue.get_nowait()
except Empty:
left = None
try:
right = self.right_motor_queue.get_nowait()
except Empty:
right = None
self.motor_lock.release()
return center, left, right
def clear_motor_jobs(self, side: str):
self.motor_lock.acquire()
if side == 'center':
self.center_motor_queue = Queue()
elif side == 'left':
self.left_motor_queue = Queue()
else:
self.right_motor_queue = Queue()
self.motor_lock.release()
def put_sound_job(self, job: str):
"""
Add a new sound job to the queue to be displayed.
:param job: to add.
"""
self.sound_queue.put_nowait(job)
def next_sound_job(self) -> str:
"""
Get the next sound job from the queue.
:return: a str representing the sound as text to be displayed.
"""
try:
return self.sound_queue.get_nowait()
except Empty:
return None
def set_led_color(self, brick_name, led_id, color):
if brick_name == 'left_brick:':
if led_id == 'led0':
self.left_brick_left_led_color = color
else:
self.left_brick_right_led_color = color
else:
if led_id == 'led0':
self.right_brick_left_led_color = color
else:
self.right_brick_right_led_color = color
def reset(self):
"""
Reset the data of this State
:return:
"""
self.clear_motor_jobs('center')
self.clear_motor_jobs('left')
self.clear_motor_jobs('right')
self.values.clear()
self.should_reset = False
def get_motor_side(self, address: str) -> str:
"""
Get the location of the motor on the actual robot based on its address.
:param address: of the motor
:return 'center', 'left' or 'right'
"""
if self.address_motor_center == address:
return 'center'
if self.address_motor_left == address:
return 'left'
if self.address_motor_right == address:
return 'right'
def load_sensor(self, sensor):
"""
Load the given sensor adding its default value to this state.
Also create a lock for the given sensor.
:param sensor: to load.
"""
self.values[sensor.address] = sensor.get_default_value()
self.locks[sensor.address] = threading.Lock()
def release_locks(self):
"""
Release all the locked sensor locks. This re-allows for reading
the sensor values.
"""
for lock in self.locks.values():
if lock.locked():
lock.release()
def get_value(self, address: str) -> Any:
"""
Get the value of a sensor by its address. Blocks if the lock of
the requested sensor is not available.
:param address: of the sensor to get the value from.
:return: the value of the sensor.
"""
self.locks[address].acquire()
return self.values[address]
def _forest2json_compressed(forest,
compressed_forest,
columns,
name_feature,
get_date,
milestones=None,
dates_are_dates=True,
is_mixed=False):
e_size_scaling, font_scaling, size_scaling, transform_e_size, transform_size = \
get_size_transformations(compressed_forest)
sort_key = lambda n: (
getattr(n, UNRESOLVED, 0),
get_column_value_str(n, name_feature, format_list=True)
if name_feature else '', *(get_column_value_str(
n, column, format_list=True) for column in columns), -getattr(
n, NUM_TIPS_INSIDE), -len(getattr(n, ROOTS)), n.name)
i = 0
node2id = {}
todo = Queue()
for compressed_tree in compressed_forest:
todo.put_nowait(compressed_tree)
node2id[compressed_tree] = i
i += 1
while not todo.empty():
n = todo.get_nowait()
for c in sorted(n.children, key=sort_key):
node2id[c] = i
i += 1
todo.put_nowait(c)
n2state = {}
# Set the cytoscape features
for compressed_tree in compressed_forest:
for n in compressed_tree.traverse():
state = get_column_value_str(
n, name_feature, format_list=False,
list_value='') if name_feature else ''
n2state[n] = state
root_names = [_.name for _ in getattr(n, ROOTS)]
root_dates = [
get_formatted_date(_, dates_are_dates)
for _ in getattr(n, ROOTS)
]
set_cyto_features_compressed(n,
size_scaling,
e_size_scaling,
font_scaling,
transform_size,
transform_e_size,
state,
root_names,
root_dates,
is_mixed=is_mixed)
# Calculate node coordinates
min_size = 2 * min(
min(getattr(_, NODE_SIZE) for _ in compressed_tree.traverse())
for compressed_tree in compressed_forest)
n2width = {}
for compressed_tree in compressed_forest:
for n in compressed_tree.traverse('postorder'):
n2width[n] = max(
getattr(n, NODE_SIZE),
sum(n2width[c]
for c in n.children) + min_size * (len(n.children) - 1))
n2x, n2y = {}, {
compressed_tree: 0
for compressed_tree in compressed_forest
}
n2offset = {}
tree_offset = 0
for compressed_tree in compressed_forest:
n2offset[compressed_tree] = tree_offset
tree_offset += n2width[compressed_tree] + 2 * min_size
for n in compressed_tree.traverse('preorder'):
n2x[n] = n2offset[n] + n2width[n] / 2
offset = n2offset[n]
if not n.is_leaf():
for c in sorted(n.children, key=lambda c: node2id[c]):
n2offset[c] = offset
offset += n2width[c] + min_size
n2y[c] = n2y[n] + getattr(n, NODE_SIZE) / 2 + getattr(
c, NODE_SIZE) / 2 + min_size
for n in compressed_tree.traverse('postorder'):
if not n.is_leaf():
n2x[n] = np.mean([n2x[c] for c in n.children])
def filter_by_date(items, date):
return [_ for _ in items if get_date(_) <= date]
# Set the cytoscape feature for different timeline points
for tree, compressed_tree in zip(forest, compressed_forest):
if len(milestones) > 1:
nodes = list(compressed_tree.traverse())
for i in range(len(milestones) - 1, -1, -1):
milestone = milestones[i]
nodes_i = []
# remove too recent nodes from the original tree
for n in tree.traverse('postorder'):
if n.is_root():
continue
if get_date(n) > milestone:
n.up.remove_child(n)
suffix = '_{}'.format(i)
for n in nodes:
state = n2state[n]
tips_inside, tips_below, internal_nodes_inside, roots = getattr(n, TIPS_INSIDE, []), \
getattr(n, TIPS_BELOW, []), \
getattr(n, INTERNAL_NODES_INSIDE, []), \
getattr(n, ROOTS, [])
tips_inside_i, tips_below_i, internal_nodes_inside_i, roots_i = [], [], [], []
for ti, tb, ini, root in zip(tips_inside, tips_below,
internal_nodes_inside, roots):
if get_date(root) <= milestone:
roots_i.append(root)
ti = filter_by_date(ti, milestone)
tb = [
_ for _ in tb if getattr(_, DATE) <= milestone
]
ini = filter_by_date(ini, milestone)
tips_inside_i.append(
ti + [_ for _ in ini if _.is_leaf()])
tips_below_i.append(tb)
internal_nodes_inside_i.append(
[_ for _ in ini if not _.is_leaf()])
n.add_features(
**{
TIPS_INSIDE: tips_inside_i,
TIPS_BELOW: tips_below_i,
INTERNAL_NODES_INSIDE: internal_nodes_inside_i,
ROOTS: roots_i
})
if roots_i:
n.add_feature(MILESTONE, i)
root_names = [
getattr(_, BRANCH_NAME)
if getattr(_, DATE) > milestone else _.name
for _ in roots_i
]
root_dates = [getattr(_, DATE) for _ in roots_i]
if dates_are_dates:
try:
root_dates = [
numeric2datetime(_).strftime("%d %b %Y")
for _ in root_dates
]
except:
pass
set_cyto_features_compressed(n,
size_scaling,
e_size_scaling,
font_scaling,
transform_size,
transform_e_size,
state,
root_names=root_names,
suffix=suffix,
root_dates=root_dates,
is_mixed=is_mixed)
nodes_i.append(n)
nodes = nodes_i
# Save the structure
clazzes = set()
nodes, edges = [], []
one_column = columns[0] if len(columns) == 1 else None
for n, n_id in node2id.items():
if one_column:
values = getattr(n, one_column, set())
clazz = tuple(sorted(values))
else:
clazz = tuple('{}_{}'.format(
column,
get_column_value_str(
n, column, format_list=False, list_value=''))
for column in columns)
if clazz:
clazzes.add(clazz)
nodes.append(
get_node(n,
n_id,
tooltip=get_tooltip(n, columns),
clazz=clazz,
x=n2x[n],
y=n2y[n]))
for child in sorted(n.children, key=lambda _: node2id[_]):
edge_attributes = {
feature: getattr(child, feature)
for feature in child.features if feature.startswith('edge_')
or feature == MILESTONE or feature == IS_POLYTOMY
}
source_name = n_id
edges.append(
get_edge(source_name, node2id[child], **edge_attributes))
json_dict = {NODES: nodes, EDGES: edges}
return json_dict, sorted(clazzes)
def recursive_update(pkgs, verbose):
if not pkgs:
logger.warn('no package specified')
return set()
# TODO: get distro from environment
distro = get_rosdistro('kinetic')
repositories = [
r for r in distro.repositories.values()
if r.source_repository and r.source_repository.patched_packages
]
pkgs_queue = Queue()
pkgs_done = set()
for pkg in pkgs:
pkgs_queue.put_nowait(pkg)
# mapping from manifest to found location
pkgs_manifests = {}
try:
while True:
pkg = pkgs_queue.get_nowait()
if pkg in pkgs_done:
continue
repo = [
repo for repo in repositories
if pkg in repo.source_repository.patched_packages
]
if len(repo) == 0:
logger.debug("Package '%s' can't be found in a repository",
pkg)
continue
assert len(
repo) == 1, "Package '%s' is in multiple repositories" % pkg
repo = repo[0]
logger.info('installing: %s', pkg)
if not len([m for m in pkgs_manifests if m.name == pkg]):
update_folder(target_path, {repo.name: repo}, verbose)
# which packages did we download?
updated_packages = find_packages_allowing_duplicates(
os.path.join(target_path, repo.name))
# first check for expected packages that were not found
found_names = set(package.name
for package in updated_packages.values())
for package in repo.source_repository.patched_packages:
if package not in found_names:
logger.warning(
"Package '%s' not found in the repo: '%s'",
package, repo.name)
# then check for found packages that were not in the yaml
for subfolder, package in updated_packages.items():
if package.name in repo.source_repository.patched_packages:
logger.info("found '%s'" %
os.path.join(repo.name, subfolder))
pkgs_manifests[package] = os.path.join(
repo.name, subfolder)
else:
logger.debug(
"Found package '%s' in an unexpected repo: '%s'",
package.name, repo.name)
manifest = [m for m in pkgs_manifests if m.name == pkg]
if not len(manifest):
logger.error("Required package '%s' not found", pkg)
exit(1)
assert len(
manifest) == 1, "Package '%s' was found multiple times" % pkg
manifest = manifest[0]
deps = manifest.buildtool_depends + manifest.build_depends + manifest.run_depends + manifest.test_depends
for dep in deps:
if dep not in pkgs_done:
logger.debug("queing: '%s'", dep.name)
pkgs_queue.put_nowait(dep.name)
pkgs_done.add(pkg)
except Empty:
pass
if not pkgs_done:
logger.error('no repository updated, package could not be found')
# create the symlinks in the src folder
for pkg in pkgs_done:
manifest = [m for m in pkgs_manifests if m.name == pkg]
assert len(manifest) == 1
manifest = manifest[0]
folder = pkgs_manifests[manifest]
source = os.path.join(target_path, folder)
link_name = os.path.join(link_dir, manifest.name)
source = os.path.relpath(source, link_dir)
symlink_force(source, link_name)
return pkgs_done
class AbstractBaseThread(QThread):
started = pyqtSignal()
stopped = pyqtSignal()
sender_needs_restart = pyqtSignal()
def __init__(self, freq, sample_rate, bandwidth, gain, if_gain, baseband_gain, receiving: bool,
ip='127.0.0.1', parent=None):
super().__init__(parent)
self.ip = ip
self.gr_port = 1337
self._sample_rate = sample_rate
self._freq = freq
self._gain = gain
self._if_gain = if_gain
self._baseband_gain = baseband_gain
self._bandwidth = bandwidth
self._freq_correction = 1
self._direct_sampling_mode = 0
self._antenna_index = 0
self._channel_index = 0
self._receiving = receiving # False for Sender-Thread
self.device = "USRP"
self.current_index = 0
self.is_in_spectrum_mode = False
self.context = None
self.socket = None
gnuradio_path_file = os.path.join(tempfile.gettempdir(), "gnuradio_path.txt")
if constants.SETTINGS.value("use_gnuradio_install_dir", False, bool):
gnuradio_dir = constants.SETTINGS.value("gnuradio_install_dir", "")
with open(gnuradio_path_file, "w") as f:
f.write(gnuradio_dir)
if os.path.isfile(os.path.join(gnuradio_dir, "gr-python27", "pythonw.exe")):
self.python2_interpreter = os.path.join(gnuradio_dir, "gr-python27", "pythonw.exe")
else:
self.python2_interpreter = os.path.join(gnuradio_dir, "gr-python27", "python.exe")
else:
try:
os.remove(gnuradio_path_file)
except OSError:
pass
self.python2_interpreter = constants.SETTINGS.value("python2_exe", "")
self.queue = Queue()
self.data = None # Placeholder for SenderThread
self.current_iteration = 0 # Counts number of Sendings in SenderThread
self.tb_process = None
@property
def sample_rate(self):
return self._sample_rate
@sample_rate.setter
def sample_rate(self, value):
self._sample_rate = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'SR:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def freq(self):
return self._freq
@freq.setter
def freq(self, value):
self._freq = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'F:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def gain(self):
return self._gain
@gain.setter
def gain(self, value):
self._gain = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'G:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def if_gain(self):
return self._if_gain
@if_gain.setter
def if_gain(self, value):
self._if_gain = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'IFG:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def baseband_gain(self):
return self._baseband_gain
@baseband_gain.setter
def baseband_gain(self, value):
self._baseband_gain = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'BBG:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def bandwidth(self):
return self._bandwidth
@bandwidth.setter
def bandwidth(self, value):
self._bandwidth = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'BW:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def freq_correction(self):
return self._freq_correction
@freq_correction.setter
def freq_correction(self, value):
self._freq_correction = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'FC:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
@property
def channel_index(self):
return self._channel_index
@channel_index.setter
def channel_index(self, value):
self._channel_index = value
@property
def antenna_index(self):
return self._antenna_index
@antenna_index.setter
def antenna_index(self, value):
self._antenna_index = value
@property
def direct_sampling_mode(self):
return self._direct_sampling_mode
@direct_sampling_mode.setter
def direct_sampling_mode(self, value):
self._direct_sampling_mode = value
if self.tb_process:
try:
self.tb_process.stdin.write(b'DSM:' + bytes(str(value), "utf8") + b'\n')
self.tb_process.stdin.flush()
except BrokenPipeError:
pass
def initialize_process(self):
self.started.emit()
if not hasattr(sys, 'frozen'):
rp = os.path.realpath(os.path.join(os.path.dirname(__file__), "scripts"))
else:
rp = os.path.realpath(os.path.dirname(sys.executable))
suffix = "_recv.py" if self._receiving else "_send.py"
filename = self.device.lower().split(" ")[0] + suffix
if not self.python2_interpreter:
self.stop("FATAL: Could not find python 2 interpreter. Make sure you have a running gnuradio installation.")
return
options = [self.python2_interpreter, os.path.join(rp, filename),
"--samplerate", str(self.sample_rate), "--freq", str(self.freq),
"--gain", str(self.gain), "--bandwidth", str(self.bandwidth),
"--port", str(self.gr_port)]
if self.device.upper() == "HACKRF":
options.extend(["--if-gain", str(self.if_gain), "--baseband-gain", str(self.baseband_gain)])
if self.device.upper() == "RTL-SDR":
options.extend(["--freq-correction", str(self.freq_correction),
"--direct-sampling", str(self.direct_sampling_mode)])
logger.info("Starting Gnuradio")
logger.debug(" ".join(options))
self.tb_process = Popen(options, stdout=PIPE, stderr=PIPE, stdin=PIPE, bufsize=1)
logger.info("Started Gnuradio")
t = Thread(target=self.enqueue_output, args=(self.tb_process.stderr, self.queue))
t.daemon = True # thread dies with the program
t.start()
def init_recv_socket(self):
logger.info("Initalizing receive socket")
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PULL)
self.socket.setsockopt(zmq.RCVTIMEO, 90)
logger.info("Initalized receive socket")
while not self.isInterruptionRequested():
try:
time.sleep(0.1)
logger.info("Trying to get a connection to gnuradio...")
self.socket.connect("tcp://{0}:{1}".format(self.ip, self.gr_port))
logger.info("Got connection")
break
except (ConnectionRefusedError, ConnectionResetError):
continue
except Exception as e:
logger.error("Unexpected error", str(e))
def run(self):
pass
def read_errors(self, initial_errors=None):
result = [] if initial_errors is None else initial_errors
while True:
try:
result.append(self.queue.get_nowait())
except Empty:
break
result = b"".join(result)
try:
return result.decode("utf-8")
except UnicodeDecodeError:
return "Could not decode device message"
def enqueue_output(self, out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
def stop(self, msg: str):
if msg and not msg.startswith("FIN"):
self.requestInterruption()
time.sleep(0.1)
try:
logger.info("Kill grc process")
self.tb_process.kill()
logger.info("Term grc process")
self.tb_process.terminate()
self.tb_process = None
except AttributeError:
pass
logger.info(msg)
self.stopped.emit()
class printcore():
def __init__(self, port=None, baud=None, dtr=None):
"""Initializes a printcore instance. Pass the port and baud rate to
connect immediately"""
self.baud = None
self.dtr = None
self.port = None
self.analyzer = gcoder.GCode()
# Serial instance connected to the printer, should be None when
# disconnected
self.printer = None
# clear to send, enabled after responses
# FIXME: should probably be changed to a sliding window approach
self.clear = 0
# The printer has responded to the initial command and is active
self.online = False
# is a print currently running, true if printing, false if paused
self.printing = False
self.mainqueue = None
self.priqueue = Queue(0)
self.queueindex = 0
self.lineno = 0
self.resendfrom = -1
self.paused = False
self.sentlines = {}
self.log = deque(maxlen=10000)
self.sent = []
self.writefailures = 0
self.tempcb = None # impl (wholeline)
self.recvcb = None # impl (wholeline)
self.sendcb = None # impl (wholeline)
self.preprintsendcb = None # impl (wholeline)
self.printsendcb = None # impl (wholeline)
self.layerchangecb = None # impl (wholeline)
self.errorcb = None # impl (wholeline)
self.startcb = None # impl ()
self.endcb = None # impl ()
self.onlinecb = None # impl ()
self.loud = False # emit sent and received lines to terminal
self.tcp_streaming_mode = False
self.greetings = ['start', 'Grbl ']
self.wait = 0 # default wait period for send(), send_now()
self.read_thread = None
self.stop_read_thread = False
self.send_thread = None
self.stop_send_thread = False
self.print_thread = None
self.event_handler = PRINTCORE_HANDLER
# Not all platforms need to do this parity workaround, and some drivers
# don't support it. Limit it to platforms that actually require it
# here to avoid doing redundant work elsewhere and potentially breaking
# things.
self.needs_parity_workaround = platform.system(
) == "linux" and os.path.exists("/etc/debian")
for handler in self.event_handler:
try:
handler.on_init()
except:
logging.error(traceback.format_exc())
if port is not None and baud is not None:
self.connect(port, baud)
self.xy_feedrate = None
self.z_feedrate = None
def addEventHandler(self, handler):
'''
Adds an event handler.
@param handler: The handler to be added.
'''
self.event_handler.append(handler)
def logError(self, error):
for handler in self.event_handler:
try:
handler.on_error(error)
except:
logging.error(traceback.format_exc())
if self.errorcb:
try:
self.errorcb(error)
except:
logging.error(traceback.format_exc())
else:
logging.error(error)
@locked
def disconnect(self):
"""Disconnects from printer and pauses the print
"""
if self.printer:
if self.read_thread:
self.stop_read_thread = True
if threading.current_thread() != self.read_thread:
self.read_thread.join()
self.read_thread = None
if self.print_thread:
self.printing = False
self.print_thread.join()
self._stop_sender()
try:
self.printer.close()
except socket.error:
pass
except OSError:
pass
for handler in self.event_handler:
try:
handler.on_disconnect()
except:
logging.error(traceback.format_exc())
self.printer = None
self.online = False
self.printing = False
@locked
def connect(self, port=None, baud=None, dtr=None):
"""Set port and baudrate if given, then connect to printer
"""
if self.printer:
self.disconnect()
if port is not None:
self.port = port
if baud is not None:
self.baud = baud
if dtr is not None:
self.dtr = dtr
if self.port is not None and self.baud is not None:
# Connect to socket if "port" is an IP, device if not
host_regexp = re.compile(
"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$"
)
is_serial = True
if ":" in self.port:
bits = self.port.split(":")
if len(bits) == 2:
hostname = bits[0]
try:
port_number = int(bits[1])
if host_regexp.match(
hostname) and 1 <= port_number <= 65535:
is_serial = False
except:
pass
self.writefailures = 0
if not is_serial:
self.printer_tcp = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self.printer_tcp.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
self.timeout = 0.25
self.printer_tcp.settimeout(1.0)
try:
self.printer_tcp.connect((hostname, port_number))
self.printer_tcp.settimeout(self.timeout)
self.printer = self.printer_tcp.makefile()
except socket.error as e:
if (e.strerror is None): e.strerror = ""
self.logError(
_("Could not connect to %s:%s:") %
(hostname, port_number) + "\n" +
_("Socket error %s:") % e.errno + "\n" + e.strerror)
self.printer = None
self.printer_tcp = None
return
else:
disable_hup(self.port)
self.printer_tcp = None
try:
if self.needs_parity_workaround:
self.printer = Serial(port=self.port,
baudrate=self.baud,
timeout=0.25,
parity=PARITY_ODD)
self.printer.close()
self.printer.parity = PARITY_NONE
else:
self.printer = Serial(baudrate=self.baud,
timeout=0.25,
parity=PARITY_NONE)
self.printer.setPort(self.port)
try: #this appears not to work on many platforms, so we're going to call it but not care if it fails
self.printer.setDTR(dtr)
except:
#self.logError(_("Could not set DTR on this platform")) #not sure whether to output an error message
pass
self.printer.open()
except SerialException as e:
self.logError(
_("Could not connect to %s at baudrate %s:") %
(self.port, self.baud) + "\n" +
_("Serial error: %s") % e)
self.printer = None
return
except IOError as e:
self.logError(
_("Could not connect to %s at baudrate %s:") %
(self.port, self.baud) + "\n" + _("IO error: %s") % e)
self.printer = None
return
for handler in self.event_handler:
try:
handler.on_connect()
except:
logging.error(traceback.format_exc())
self.stop_read_thread = False
self.read_thread = threading.Thread(target=self._listen)
self.read_thread.start()
self._start_sender()
def reset(self):
"""Reset the printer
"""
if self.printer and not self.printer_tcp:
self.printer.setDTR(1)
time.sleep(0.2)
self.printer.setDTR(0)
def _readline(self):
try:
try:
try:
line = self.printer.readline().decode('ascii')
except UnicodeDecodeError:
self.logError(
_("Got rubbish reply from %s at baudrate %s:") %
(self.port, self.baud) + "\n" +
_("Maybe a bad baudrate?"))
return None
if self.printer_tcp and not line:
raise OSError(-1, "Read EOF from socket")
except socket.timeout:
return ""
if len(line) > 1:
self.log.append(line)
for handler in self.event_handler:
try:
handler.on_recv(line)
except:
logging.error(traceback.format_exc())
if self.recvcb:
try:
self.recvcb(line)
except:
self.logError(traceback.format_exc())
if self.loud: logging.info("RECV: %s" % line.rstrip())
return line
except SelectError as e:
if 'Bad file descriptor' in e.args[1]:
self.logError(
_("Can't read from printer (disconnected?) (SelectError {0}): {1}"
).format(e.errno, decode_utf8(e.strerror)))
return None
else:
self.logError(
_("SelectError ({0}): {1}").format(e.errno,
decode_utf8(
e.strerror)))
raise
except SerialException as e:
self.logError(
_("Can't read from printer (disconnected?) (SerialException): {0}"
).format(decode_utf8(str(e))))
return None
except socket.error as e:
self.logError(
_("Can't read from printer (disconnected?) (Socket error {0}): {1}"
).format(e.errno, decode_utf8(e.strerror)))
return None
except OSError as e:
if e.errno == errno.EAGAIN: # Not a real error, no data was available
return ""
self.logError(
_("Can't read from printer (disconnected?) (OS Error {0}): {1}"
).format(e.errno, e.strerror))
return None
def _listen_can_continue(self):
if self.printer_tcp:
return not self.stop_read_thread and self.printer
return (not self.stop_read_thread and self.printer
and self.printer.isOpen())
def _listen_until_online(self):
while not self.online and self._listen_can_continue():
self._send("M105")
if self.writefailures >= 4:
logging.error(
_("Aborting connection attempt after 4 failed writes."))
return
empty_lines = 0
while self._listen_can_continue():
line = self._readline()
if line is None: break # connection problem
# workaround cases where M105 was sent before printer Serial
# was online an empty line means read timeout was reached,
# meaning no data was received thus we count those empty lines,
# and once we have seen 15 in a row, we just break and send a
# new M105
# 15 was chosen based on the fact that it gives enough time for
# Gen7 bootloader to time out, and that the non received M105
# issues should be quite rare so we can wait for a long time
# before resending
if not line:
empty_lines += 1
if empty_lines == 15: break
else: empty_lines = 0
if line.startswith(tuple(self.greetings)) \
or line.startswith('ok') or "T:" in line:
self.online = True
for handler in self.event_handler:
try:
handler.on_online()
except:
logging.error(traceback.format_exc())
if self.onlinecb:
try:
self.onlinecb()
except:
self.logError(traceback.format_exc())
return
def _listen(self):
"""This function acts on messages from the firmware
"""
self.clear = True
if not self.printing:
self._listen_until_online()
while self._listen_can_continue():
line = self._readline()
if line is None:
break
if line.startswith('DEBUG_'):
continue
if line.startswith(tuple(self.greetings)) or line.startswith('ok'):
self.clear = True
if line.startswith('ok') and "T:" in line:
for handler in self.event_handler:
try:
handler.on_temp(line)
except:
logging.error(traceback.format_exc())
if line.startswith('ok') and "T:" in line and self.tempcb:
# callback for temp, status, whatever
try:
self.tempcb(line)
except:
self.logError(traceback.format_exc())
elif line.startswith('Error'):
self.logError(line)
# Teststrings for resend parsing # Firmware exp. result
# line="rs N2 Expected checksum 67" # Teacup 2
if line.lower().startswith("resend") or line.startswith("rs"):
for haystack in ["N:", "N", ":"]:
line = line.replace(haystack, " ")
linewords = line.split()
while len(linewords) != 0:
try:
toresend = int(linewords.pop(0))
self.resendfrom = toresend
break
except:
pass
self.clear = True
self.clear = True
def _start_sender(self):
self.stop_send_thread = False
self.send_thread = threading.Thread(target=self._sender)
self.send_thread.start()
def _stop_sender(self):
if self.send_thread:
self.stop_send_thread = True
self.send_thread.join()
self.send_thread = None
def _sender(self):
while not self.stop_send_thread:
try:
command = self.priqueue.get(True, 0.1)
except QueueEmpty:
continue
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
self._send(command)
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
def _checksum(self, command):
return reduce(lambda x, y: x ^ y, map(ord, command))
def startprint(self, gcode, startindex=0):
"""Start a print, gcode is an array of gcode commands.
returns True on success, False if already printing.
The print queue will be replaced with the contents of the data array,
the next line will be set to 0 and the firmware notified. Printing
will then start in a parallel thread.
"""
if self.printing or not self.online or not self.printer:
return False
self.queueindex = startindex
self.mainqueue = gcode
self.printing = True
self.lineno = 0
self.resendfrom = -1
self._send("M110", -1, True)
if not gcode or not gcode.lines:
return True
self.clear = False
resuming = (startindex != 0)
self.print_thread = threading.Thread(target=self._print,
kwargs={"resuming": resuming})
self.print_thread.start()
return True
def cancelprint(self):
self.pause()
self.paused = False
self.mainqueue = None
self.clear = True
# run a simple script if it exists, no multithreading
def runSmallScript(self, filename):
if filename is None: return
f = None
try:
with open(filename) as f:
for i in f:
l = i.replace("\n", "")
l = l[:l.find(";")] # remove comments
self.send_now(l)
except:
pass
def pause(self):
"""Pauses the print, saving the current position.
"""
if not self.printing: return False
self.paused = True
self.printing = False
# try joining the print thread: enclose it in try/except because we
# might be calling it from the thread itself
try:
self.print_thread.join()
except RuntimeError as e:
if e.message == "cannot join current thread":
pass
else:
self.logError(traceback.format_exc())
except:
self.logError(traceback.format_exc())
self.print_thread = None
# saves the status
self.pauseX = self.analyzer.abs_x
self.pauseY = self.analyzer.abs_y
self.pauseZ = self.analyzer.abs_z
self.pauseE = self.analyzer.abs_e
self.pauseF = self.analyzer.current_f
self.pauseRelative = self.analyzer.relative
def resume(self):
"""Resumes a paused print.
"""
if not self.paused: return False
if self.paused:
# restores the status
self.send_now("G90") # go to absolute coordinates
xyFeedString = ""
zFeedString = ""
if self.xy_feedrate is not None:
xyFeedString = " F" + str(self.xy_feedrate)
if self.z_feedrate is not None:
zFeedString = " F" + str(self.z_feedrate)
self.send_now("G1 X%s Y%s%s" %
(self.pauseX, self.pauseY, xyFeedString))
self.send_now("G1 Z" + str(self.pauseZ) + zFeedString)
self.send_now("G92 E" + str(self.pauseE))
# go back to relative if needed
if self.pauseRelative: self.send_now("G91")
# reset old feed rate
self.send_now("G1 F" + str(self.pauseF))
self.paused = False
self.printing = True
self.print_thread = threading.Thread(target=self._print,
kwargs={"resuming": True})
self.print_thread.start()
def send(self, command, wait=0):
"""Adds a command to the checksummed main command queue if printing, or
sends the command immediately if not printing"""
if self.online:
if self.printing:
self.mainqueue.append(command)
else:
self.priqueue.put_nowait(command)
else:
self.logError(_("Not connected to printer."))
def send_now(self, command, wait=0):
"""Sends a command to the printer ahead of the command queue, without a
checksum"""
if self.online:
self.priqueue.put_nowait(command)
else:
self.logError(_("Not connected to printer."))
def _print(self, resuming=False):
self._stop_sender()
try:
for handler in self.event_handler:
try:
handler.on_start(resuming)
except:
logging.error(traceback.format_exc())
if self.startcb:
# callback for printing started
try:
self.startcb(resuming)
except:
self.logError(
_("Print start callback failed with:") + "\n" +
traceback.format_exc())
while self.printing and self.printer and self.online:
self._sendnext()
self.sentlines = {}
self.log.clear()
self.sent = []
for handler in self.event_handler:
try:
handler.on_end()
except:
logging.error(traceback.format_exc())
if self.endcb:
# callback for printing done
try:
self.endcb()
except:
self.logError(
_("Print end callback failed with:") + "\n" +
traceback.format_exc())
except:
self.logError(
_("Print thread died due to the following error:") + "\n" +
traceback.format_exc())
finally:
self.print_thread = None
self._start_sender()
def process_host_command(self, command):
"""only ;@pause command is implemented as a host command in printcore, but hosts are free to reimplement this method"""
command = command.lstrip()
if command.startswith(";@pause"):
self.pause()
def _sendnext(self):
if not self.printer:
return
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
# Only wait for oks when using serial connections or when not using tcp
# in streaming mode
if not self.printer_tcp or not self.tcp_streaming_mode:
self.clear = False
if not (self.printing and self.printer and self.online):
self.clear = True
return
if self.resendfrom < self.lineno and self.resendfrom > -1:
self._send(self.sentlines[self.resendfrom], self.resendfrom, False)
self.resendfrom += 1
return
self.resendfrom = -1
if not self.priqueue.empty():
self._send(self.priqueue.get_nowait())
self.priqueue.task_done()
return
if self.printing and self.mainqueue.has_index(self.queueindex):
(layer, line) = self.mainqueue.idxs(self.queueindex)
gline = self.mainqueue.all_layers[layer][line]
if self.queueindex > 0:
(prev_layer,
prev_line) = self.mainqueue.idxs(self.queueindex - 1)
if prev_layer != layer:
for handler in self.event_handler:
try:
handler.on_layerchange(layer)
except:
logging.error(traceback.format_exc())
if self.layerchangecb and self.queueindex > 0:
(prev_layer,
prev_line) = self.mainqueue.idxs(self.queueindex - 1)
if prev_layer != layer:
try:
self.layerchangecb(layer)
except:
self.logError(traceback.format_exc())
for handler in self.event_handler:
try:
handler.on_preprintsend(gline, self.queueindex,
self.mainqueue)
except:
logging.error(traceback.format_exc())
if self.preprintsendcb:
if self.mainqueue.has_index(self.queueindex + 1):
(next_layer,
next_line) = self.mainqueue.idxs(self.queueindex + 1)
next_gline = self.mainqueue.all_layers[next_layer][
next_line]
else:
next_gline = None
gline = self.preprintsendcb(gline, next_gline)
if gline is None:
self.queueindex += 1
self.clear = True
return
tline = gline.raw
if tline.lstrip().startswith(";@"): # check for host command
self.process_host_command(tline)
self.queueindex += 1
self.clear = True
return
# Strip comments
tline = gcoder.gcode_strip_comment_exp.sub("", tline).strip()
if tline:
self._send(tline, self.lineno, True)
self.lineno += 1
for handler in self.event_handler:
try:
handler.on_printsend(gline)
except:
logging.error(traceback.format_exc())
if self.printsendcb:
try:
self.printsendcb(gline)
except:
self.logError(traceback.format_exc())
else:
self.clear = True
self.queueindex += 1
else:
self.printing = False
self.clear = True
if not self.paused:
self.queueindex = 0
self.lineno = 0
self._send("M110", -1, True)
def _send(self, command, lineno=0, calcchecksum=False):
# Only add checksums if over serial (tcp does the flow control itself)
if calcchecksum and not self.printer_tcp:
prefix = "N" + str(lineno) + " " + command
command = prefix + "*" + str(self._checksum(prefix))
if "M110" not in command:
self.sentlines[lineno] = command
if self.printer:
self.sent.append(command)
# run the command through the analyzer
gline = None
try:
gline = self.analyzer.append(command, store=False)
except:
logging.warning(
_("Could not analyze command %s:") % command + "\n" +
traceback.format_exc())
if self.loud:
logging.info("SENT: %s" % command)
for handler in self.event_handler:
try:
handler.on_send(command, gline)
except:
logging.error(traceback.format_exc())
if self.sendcb:
try:
self.sendcb(command, gline)
except:
self.logError(traceback.format_exc())
try:
self.printer.write((command + "\n").encode('ascii'))
if self.printer_tcp:
try:
self.printer.flush()
except socket.timeout:
pass
self.writefailures = 0
except socket.error as e:
if e.errno is None:
self.logError(
_("Can't write to printer (disconnected ?):") + "\n" +
traceback.format_exc())
else:
self.logError(
_("Can't write to printer (disconnected?) (Socket error {0}): {1}"
).format(e.errno, decode_utf8(e.strerror)))
self.writefailures += 1
except SerialException as e:
self.logError(
_("Can't write to printer (disconnected?) (SerialException): {0}"
).format(decode_utf8(str(e))))
self.writefailures += 1
except RuntimeError as e:
self.logError(
_("Socket connection broken, disconnected. ({0}): {1}").
format(e.errno, decode_utf8(e.strerror)))
self.writefailures += 1
class RunPanel(mforms.Table):
def __init__(self, editor, log_callback):
mforms.Table.__init__(self)
self.set_managed()
self.set_release_on_add()
self.set_row_count(2)
self.set_column_count(1)
self.set_padding(-1)
self.label = mforms.newLabel("Running script...")
self.add(self.label, 0, 1, 0, 1, mforms.HFillFlag)
self.progress = mforms.newProgressBar()
self.add(self.progress, 0, 1, 1, 2, mforms.HFillFlag)
self.progress.set_size(400, -1)
self.log_callback = log_callback
self.editor = editor
self.importer = MySQLScriptImporter(editor.connection)
self.importer.report_progress = self.report_progress
self.importer.report_output = self.report_output
self._worker_queue = Queue()
self._worker = None
self._progress_status = None
self._progress_value = 0
self._update_timer = None
def __del__(self):
if self._update_timer:
mforms.Utilities.cancel_timeout(self._update_timer)
@property
def is_busy(self):
return self._worker != None
def report_progress(self, status, current, total):
self._progress_status = status
if status:
self._worker_queue.put(status)
if total > 0:
self._progress_value = float(current) / total
def report_output(self, message):
log_info("%s\n" % message)
self._worker_queue.put(message)
def update_ui(self):
try:
while True:
data = self._worker_queue.get_nowait()
if data is None:
self._worker.join()
self._worker = None
self._update_timer = None
self.progress.show(False)
self.log_callback(None)
return False
if isinstance(data, Exception):
self.log_callback(str(data) + "\n")
if isinstance(data, grt.DBError) and data.args[1] == 1044:
mforms.Utilities.show_error(
"Run SQL Script",
"The current MySQL account does not have enough privileges to execute the script.",
"OK", "", "")
elif isinstance(data, grt.DBLoginError):
username = self.editor.connection.parameterValues[
"userName"]
host = self.editor.connection.hostIdentifier
mforms.Utilities.forget_password(host, username)
mforms.Utilities.show_error(
"Run SQL Script",
"Error executing SQL script.\n" + str(data), "OK",
"", "")
else:
mforms.Utilities.show_error(
"Run SQL Script",
"Error executing SQL script.\n" + str(data), "OK",
"", "")
else:
self.log_callback(data + "\n")
if self._progress_status is not None:
self.label.set_text(self._progress_status)
self.progress.set_value(self._progress_value)
except Empty:
pass
if self._progress_status is not None:
self.label.set_text(self._progress_status)
self.progress.set_value(self._progress_value)
return True
def work(self, file, schema, charset):
try:
log_info("Executing %s...\n" % file)
self._progress_status = "Executing %s..." % file
self._progress_value = 0
self.importer.import_script(file,
default_schema=schema,
default_charset=charset)
log_info("Run script finished\n")
except grt.DBLoginError as e:
log_error("MySQL login error running script: %s\n" % e)
self._worker_queue.put(e)
except grt.DBError as e:
log_error("MySQL error running script: %s\n" % e)
self._worker_queue.put(e)
except Exception as e:
import traceback
log_error("Unexpected exception running script: %s\n%s\n" %
(e, traceback.format_exc()))
self._worker_queue.put(e)
self._worker_queue.put(None)
def start(self, what, default_db, default_charset):
parameterValues = self.editor.connection.parameterValues
username = parameterValues["userName"]
host = self.editor.connection.hostIdentifier
ok, pwd = mforms.Utilities.find_cached_password(host, username)
if not ok:
accepted, pwd = mforms.Utilities.find_or_ask_for_password(
"Run SQL Script", host, username, False)
if not accepted:
return
self.importer.set_password(pwd)
self._worker = Thread(target=self.work,
args=(what, default_db, default_charset))
self._worker.start()
self._update_timer = mforms.Utilities.add_timeout(0.2, self.update_ui)
def q_learning(
env,
state_converter,
max_steps=200,
n_step=1,
alpha=0.1,
gamma=0.95,
episode_count=2000,
render=False,
figure_path=None,
):
gamma_n = gamma**n_step
def update_q_value(q_value, prev_state, action, state, return_value):
prev_index = state_converter.state_to_index(prev_state)
delta = return_value - q_value[prev_index][action]
if state is not None:
index = state_converter.state_to_index(state)
delta += gamma_n * np.max(q_value[index])
q_value[prev_index][action] += alpha * delta
q_value = state_converter.get_initial_q_value()
queue = Queue()
time_steps = []
for i in range(episode_count):
if not queue.empty():
queue.get_nowait()
for _ in range(n_step):
queue.put((None, None, 0))
queue.get()
G = 0
prev_state = env.reset()
for t in range(max_steps):
if render:
env.render()
action = get_action(q_value, state_converter, prev_state)
state, reward, is_terminated, _ = env.step(action)
reward = - \
100 if is_terminated and t < int(max_steps * 0.975) else 1
queue.put((prev_state, action, reward))
target_state, target_action, target_reward = queue.get()
G += reward * gamma_n
G /= gamma
if t + 1 >= n_step and target_state is not None:
update_q_value(q_value, target_state, target_action, state, G)
G -= target_reward
prev_state = state
if is_terminated:
while not queue.empty():
target_state, target_action, target_reward = queue.get()
G /= gamma
update_q_value(q_value, target_state, target_action, None,
G)
G -= target_reward
print("Episode {} finished after {} timesteps".format(
i, t + 1))
time_steps.append(t + 1)
break
if (i + 1) % 20 == 0:
plot(time_steps, 'Time Step', figure_path)
class CommandPort(threading.Thread):
""" Command port runs on a separate thread """
def __init__(self, queue_size=0, timeout=.1, port=5000, buffersize=4096, max_connections=5,
return_result=False, result_as_json=False, redirect_output=False, share_environ=True):
super(CommandPort, self).__init__()
self.do_run = True
self.return_result = return_result
self.result_as_json = result_as_json
self.redirect_output = redirect_output
self.buffersize = buffersize
self.max_connections = max_connections
self.share_environ = share_environ
self.commands_queue = Queue(queue_size)
self.output_queue = Queue(queue_size)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.settimeout(timeout)
self.socket.bind(('localhost', port))
def run(self):
# ---- Run while main thread of Blender is Alive
# This feels a little bit hacky, but that was the most reliable way I've found
# There's no method or callback that Blender calls on exit that could be used to close the port
# So I'm detecting if a main thread of blender did finish working.
# If it did, then I'm breaking the loop and closing the port.
threads = threading.enumerate()
while any([t.name == "MainThread" and t.isAlive() for t in threads]):
if not self.do_run:
# ---- Break also if user requested closing the port.
print("do_run is False")
break
self.socket.listen(self.max_connections)
try:
connection, address = self.socket.accept()
data = connection.recv(self.buffersize)
size = sys.getsizeof(data)
if size >= self.buffersize:
print("The length of input is probably too long: {}".format(size))
if size >= 0:
command = data.decode()
self.commands_queue.put(command)
if self.redirect_output or self.return_result:
while True:
try:
output = self.output_queue.get_nowait()
except Empty:
continue
else:
if isinstance(output, ResultContainer):
if self.result_as_json:
result = json.dumps(output.value)
else:
result = str(output.value)
connection.sendall(result.encode())
break
elif output and output != "\n":
connection.sendall(output.encode())
else:
connection.sendall('OK'.encode())
connection.shutdown(socket.SHUT_RDWR)
connection.close()
except socket.timeout:
pass
self.socket.close()
print("Closing the socket")
return
class ResourcePool(object):
def __init__(self,
cls,
*args,
capacity=0,
idle_timeout=None,
autowrap=False,
close_on_exc=False,
**kwargs):
"""Create a new pool object.
@param cls(object): The object class to be manage.
@param args(tuple): The positional parameters of cls.
@param kwargs(dict): The key parameters of cls.
@param capacity(int): The maximum capacity of the pool.
If 0, the capacity is infinite.
@param idle_timeout(int): The idle timeout. The unit is second.
If None or 0, never time out.
@param autowrap(bool): If True, it will wrap the obj in ResourcePoolSession
automatically, which will release the obj into the
pool when the session is closed or deleted.
@param close_on_exc(bool): If True and autowrap is True, in with context,
the session will close the obj firstly,
then new an new one into the pool.
"""
capacity = capacity if capacity >= 0 else 0
self._cls = cls
self._args = args
self._kwargs = kwargs
self._closed = False
self._lock = Lock()
self._capacity = capacity
self._timeout = idle_timeout
self._pools = Queue(capacity)
self._autowrap = autowrap
self._close_on_exc = close_on_exc
while capacity > 0:
self.put(None)
capacity -= 1
def __del__(self):
self.close()
def _get_now(self):
return int(time.time())
def _close_obj(self, obj):
if obj:
try:
obj.close()
except Exception:
pass
def close(self):
"""Close the pool and release all the objects.
When closed, it will raise an RuntimeError if putting an object into it.
"""
with self._lock:
if self._closed:
return
self._closed = True
while True:
try:
self._close_obj(self._pools.get_nowait()[0])
self._pools.task_done()
except Empty:
return
def get(self, timeout=None):
"""Get an object from the pool.
When the pool is closed, it will raise a RuntimeError if calling this
method.
"""
with self._lock:
if self._closed:
raise RuntimeError("The pool has been closed.")
_get = lambda obj: _ResourcePoolSession(self, obj, self._close_on_exc) \
if obj and self._autowrap else obj
if not self._capacity:
try:
obj = self._pools.get_nowait()
self._pools.task_done()
except Empty:
obj = (self._cls(*self._args, **self._kwargs), self._get_now())
else:
obj = self._pools.get(timeout=timeout)
self._pools.task_done()
if obj and obj[0]:
if self._timeout and self._get_now() - obj[1] > self._timeout:
return _get(self.get(timeout=timeout))
return _get(obj[0])
return _get(self._cls(*self._args, **self._kwargs))
def put(self, obj):
"""Put an object into the pool.
When the pool is closed, it will close the object, not put it into the
pool, if calling this method.
"""
with self._lock:
if self._closed:
self._close_obj(obj)
return
if isinstance(obj, _ResourcePoolSession):
obj.release_to_pool()
else:
self._pools.put_nowait((obj, self._get_now()))
def put_with_close(self, obj):
self._close_obj(obj)
self.put(None)
def _put_from_session(self, obj):
self._pools.put_nowait((obj, self._get_now()))
def uniform_cost_search(problem_subclass):
num_nodes_expanded = 0
# Define a node of the form: (path_cost, state)
node = Node(state=problem_subclass.initial, path_cost=0)
frontier = Queue()
# Add the initial node to the frontier:
frontier.put_nowait(node)
print("Just Added Node (State: %s, Action: %s, PC: %d) to Frontier." %
(node.state, node.action, node.path_cost))
print("Frontier Now: %s" % frontier.queue)
# Initialize the explored set:
explored = set()
while True:
if frontier.empty():
# Failure, no solution.
print(
"CRITICAL: Frontier now empty. No solution possible. Search Failure."
)
return None, num_nodes_expanded
node = frontier.get_nowait()
print(
"Just Removed Node (State: %s, Action: %s, PC: %d) from Frontier for expansion."
% (node.state, node.action, node.path_cost))
# print("Frontier Now Length %d: %s" % (len(frontier.queue), frontier.queue))
if problem_subclass.goal_test(node.state):
print("CRITICAL: Reached goal state! Returning solution...")
solution_string = get_solution_from_node(goal_node=node)
return solution_string, num_nodes_expanded
explored.add(node)
print(
"Just added Node (State: %s, Action: %s, PC: %d) to Explored (if not already in set)."
% (node.state, node.action, node.path_cost))
# print("Explored now: %s" % explored)
for action in problem_subclass.actions(node.state):
resultant_state = problem_subclass.result(state=node.state,
action=action)
path_cost = problem_subclass.path_cost(c=1,
state1=node.state,
action=action,
state2=resultant_state)
child_node = Node(state=resultant_state,
path_cost=path_cost + node.path_cost,
problem=problem_subclass,
node=node,
action=action)
num_nodes_expanded += 1
# child_node.problem.print_world(child_node.state)
print(
"Generated new child_node (State: %s, Action Performed: %s, PC: %d) for consideration."
% (child_node.state, child_node.action, child_node.path_cost))
print("Agent moved FROM: %s TO: %s with action %s" %
(node.state['agent_loc'], child_node.state['agent_loc'],
action))
if child_node not in explored:
# The child node is not explored.
if child_node not in frontier.queue:
# The child node is not explored, and is not in the frontier.
# Add the child node to the frontier for expansion.
frontier.put_nowait(child_node)
print(
"Just Added Node (State: %s, Action: %s, PC: %d) to Frontier."
% (child_node.state, child_node.action,
child_node.path_cost))
# print("Frontier Now: %s" % frontier.queue)
else:
# The child node is not explored, and is in the frontier.
# Does the child in the frontier have a higher path cost:
for index, frontier_node in enumerate(frontier.queue):
if frontier_node == child_node:
if frontier_node.path_cost > child_node.path_cost:
# The frontier's copy of the node has a higher path-cost.
# Replace the frontier's copy with the new copy with lower path cost.
frontier.queue[index] = child_node
else:
# The child node is explored.
print(
"The generated child node was already in Explored. Generating a new one..."
)
class IntcodeComputer:
default_instruction_classes: List[Type[Instruction]] = [
AddInstruction,
MultiplyInstruction,
HaltInstruction,
]
def __init__(self,
initial: Union[str, Iterable[Union[int, str]]],
instruction_classes: Iterable[Type[Instruction]] = None,
stdin: Iterable[int] = None):
if hasattr(initial, 'split'):
initial = initial.split(',')
self.memory: MutableMapping[int, int] = defaultdict(
int, enumerate(int(val) for val in initial))
self.iptr = 0
self.rbptr = 0
self._stdin = Queue()
for item in stdin or []:
self.put_stdin(item)
self._stdout = Queue()
self.jumped: Optional[bool] = None
self.instructions: MutableMapping[Opcode, Instruction] = {}
self._alive = False
self._started = False
for instruction in instruction_classes or self.default_instruction_classes:
self.register_instr(instruction)
@property
def is_terminated(self) -> bool:
return self._started and not self._alive
def get_stdin(self) -> int:
return self._stdin.get()
def get_stdout(self) -> int:
return self._stdout.get()
def get_stdout_nowait(self) -> int:
return self._stdout.get_nowait()
def put_stdout(self, value: int) -> None:
self._stdout.put(value)
def put_stdin(self, value: int) -> None:
self._stdin.put(value)
@property
def stdout(self) -> Iterable[int]:
with suppress(Empty):
while True:
yield self._stdout.get_nowait()
def register_instr(self, instr_cls: Type[Instruction]) -> None:
self.instructions[instr_cls.opcode] = instr_cls(self)
def kill(self) -> None:
self._alive = False
def run(self) -> None:
with suppress(Halt):
while not self.is_terminated:
self.tick()
def tick(self) -> None:
if not self._started:
self._started = True
self._alive = True
if not self._alive:
raise InvalidState("Computer has already halted!")
# LOG.debug('iptr=%s, rbptr=%s, MEM=%s', self.iptr, self.rbptr, str(self))
opcode = Opcode(self.memory[self.iptr] % 100)
# LOG.debug('Executing %s', opcode.name)
instr = self.instructions[opcode]
self.jumped = False
instr.execute()
if not self.jumped:
self.iptr += 1 + instr.parameter_count
def jump(self, newptr: int) -> None:
self.jumped = True
self.iptr = newptr
def get_parameter(self, idx: int) -> int:
parameter_mode = self.get_parameter_mode(idx)
if parameter_mode == ParameterMode.POSITION:
return self.memory[self.memory[self.iptr + idx]]
elif parameter_mode == ParameterMode.IMMEDIATE:
return self.memory[self.iptr + idx]
elif parameter_mode == ParameterMode.RELATIVE:
return self.memory[self.memory[self.iptr + idx] + self.rbptr]
else:
raise ParameterError("Invalid parameter mode: %s", parameter_mode)
def get_parameter_mode(self, idx: int) -> ParameterMode:
digit = (self.memory[self.iptr] // 10**(idx + 1)) % 10
return ParameterMode(digit)
def set_parameter(self, idx: int, value: int) -> None:
parameter_mode = self.get_parameter_mode(idx)
if parameter_mode == ParameterMode.POSITION:
self.memory[self.memory[self.iptr + idx]] = value
elif parameter_mode == ParameterMode.RELATIVE:
self.memory[self.memory[self.iptr + idx] + self.rbptr] = value
else:
raise ParameterError("Invalid parameter mode: %s", parameter_mode)
def __str__(self) -> str:
ret = []
last_address = 0
for address, value in self.memory.items():
item = ''
if address > last_address + 1:
item += f'...{hex(address)}::'
item += str(value)
ret.append(item)
last_address = address
return ','.join(ret)
class ThreadedSelectReactor(posixbase.PosixReactorBase):
"""A threaded select() based reactor - runs on all POSIX platforms and on
Win32.
"""
def __init__(self):
threadable.init(1)
self.reads = {}
self.writes = {}
self.toThreadQueue = Queue()
self.toMainThread = Queue()
self.workerThread = None
self.mainWaker = None
posixbase.PosixReactorBase.__init__(self)
self.addSystemEventTrigger("after", "shutdown", self._mainLoopShutdown)
def wakeUp(self):
# we want to wake up from any thread
self.waker.wakeUp()
def callLater(self, *args, **kw):
tple = posixbase.PosixReactorBase.callLater(self, *args, **kw)
self.wakeUp()
return tple
def _sendToMain(self, msg, *args):
self.toMainThread.put((msg, args))
if self.mainWaker is not None:
self.mainWaker()
def _sendToThread(self, fn, *args):
self.toThreadQueue.put((fn, args))
def _preenDescriptorsInThread(self):
log.msg("Malformed file descriptor found. Preening lists.")
readers = self.reads.keys()
writers = self.writes.keys()
self.reads.clear()
self.writes.clear()
for selDict, selList in ((self.reads, readers), (self.writes,
writers)):
for selectable in selList:
try:
select.select([selectable], [selectable], [selectable], 0)
except BaseException:
log.msg("bad descriptor %s" % selectable)
else:
selDict[selectable] = 1
def _workerInThread(self):
try:
while 1:
fn, args = self.toThreadQueue.get()
fn(*args)
except SystemExit:
pass # Exception indicates this thread should exit
except BaseException:
f = failure.Failure()
self._sendToMain("Failure", f)
def _doSelectInThread(self, timeout):
"""Run one iteration of the I/O monitor loop.
This will run all selectables who had input or output readiness
waiting for them.
"""
reads = self.reads
writes = self.writes
while 1:
try:
r, w, ignored = _select(reads.keys(), writes.keys(), [],
timeout)
break
except ValueError:
# Possibly a file descriptor has gone negative?
log.err()
self._preenDescriptorsInThread()
except TypeError:
# Something *totally* invalid (object w/o fileno, non-integral
# result) was passed
log.err()
self._preenDescriptorsInThread()
except OSError as se:
# select(2) encountered an error
if se.args[0] in (0, 2):
# windows does this if it got an empty list
if (not reads) and (not writes):
return
else:
raise
elif se.args[0] == EINTR:
return
elif se.args[0] == EBADF:
self._preenDescriptorsInThread()
else:
# OK, I really don't know what's going on. Blow up.
raise
self._sendToMain("Notify", r, w)
def _process_Notify(self, r, w):
reads = self.reads
writes = self.writes
_drdw = self._doReadOrWrite
_logrun = log.callWithLogger
for selectables, method, dct in ((r, "doRead", reads), (w, "doWrite",
writes)):
for selectable in selectables:
# if this was disconnected in another thread, kill it.
if selectable not in dct:
continue
# This for pausing input when we're not ready for more.
_logrun(selectable, _drdw, selectable, method, dct)
def _process_Failure(self, f):
f.raiseException()
_doIterationInThread = _doSelectInThread
def ensureWorkerThread(self):
if self.workerThread is None or not self.workerThread.isAlive():
self.workerThread = Thread(target=self._workerInThread)
self.workerThread.start()
def doThreadIteration(self, timeout):
self._sendToThread(self._doIterationInThread, timeout)
self.ensureWorkerThread()
msg, args = self.toMainThread.get()
getattr(self, "_process_" + msg)(*args)
doIteration = doThreadIteration
def _interleave(self):
while self.running:
self.runUntilCurrent()
t2 = self.timeout()
t = self.running and t2
self._sendToThread(self._doIterationInThread, t)
yield None
msg, args = self.toMainThread.get_nowait()
getattr(self, "_process_" + msg)(*args)
def interleave(self, waker, *args, **kw):
"""
interleave(waker) interleaves this reactor with the
current application by moving the blocking parts of
the reactor (select() in this case) to a separate
thread. This is typically useful for integration with
GUI applications which have their own event loop
already running.
See the module docstring for more information.
"""
self.startRunning(*args, **kw)
loop = self._interleave()
def mainWaker(waker=waker, loop=loop):
waker(partial(next, loop))
self.mainWaker = mainWaker
next(loop)
self.ensureWorkerThread()
def _mainLoopShutdown(self):
self.mainWaker = None
if self.workerThread is not None:
self._sendToThread(raiseException, SystemExit)
self.wakeUp()
try:
while 1:
msg, args = self.toMainThread.get_nowait()
except Empty:
pass
self.workerThread.join()
self.workerThread = None
try:
while 1:
fn, args = self.toThreadQueue.get_nowait()
if fn is self._doIterationInThread:
log.msg("Iteration is still in the thread queue!")
elif fn is raiseException and args[0] is SystemExit:
pass
else:
fn(*args)
except Empty:
pass
def _doReadOrWrite(self, selectable, method, dict):
try:
why = getattr(selectable, method)()
handfn = getattr(selectable, "fileno", None)
if not handfn:
why = _NO_FILENO
elif handfn() == -1:
why = _NO_FILEDESC
except BaseException:
why = sys.exc_info()[1]
log.err()
if why:
self._disconnectSelectable(selectable, why, method == "doRead")
def addReader(self, reader):
"""Add a FileDescriptor for notification of data available to read."""
self._sendToThread(self.reads.__setitem__, reader, 1)
self.wakeUp()
def addWriter(self, writer):
"""Add a FileDescriptor for notification of data available to write."""
self._sendToThread(self.writes.__setitem__, writer, 1)
self.wakeUp()
def removeReader(self, reader):
"""Remove a Selectable for notification of data available to read."""
self._sendToThread(dictRemove, self.reads, reader)
def removeWriter(self, writer):
"""Remove a Selectable for notification of data available to write."""
self._sendToThread(dictRemove, self.writes, writer)
def removeAll(self):
return self._removeAll(self.reads, self.writes)
def getReaders(self):
return list(self.reads.keys())
def getWriters(self):
return list(self.writes.keys())
def stop(self):
"""
Extend the base stop implementation to also wake up the select thread so
that C{runUntilCurrent} notices the reactor should stop.
"""
posixbase.PosixReactorBase.stop(self)
self.wakeUp()
def run(self, installSignalHandlers=True):
self.startRunning(installSignalHandlers=installSignalHandlers)
self.mainLoop()
def mainLoop(self):
q = Queue()
self.interleave(q.put)
while self.running:
try:
q.get()()
except StopIteration:
break
class Boss(Thread):
daemon = True
def __init__(self, glfw_window, opts, args):
Thread.__init__(self, name='ChildMonitor')
startup_session = create_session(opts, args)
self.cursor_blink_zero_time = monotonic()
self.cursor_blinking = True
self.glfw_window_title = None
self.action_queue = Queue()
self.pending_resize = False
self.resize_gl_viewport = False
self.shutting_down = False
self.screen_update_delay = opts.repaint_delay / 1000.0
self.signal_fd = handle_unix_signals()
self.read_wakeup_fd, self.write_wakeup_fd = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
self.read_dispatch_map = {self.signal_fd: self.signal_received, self.read_wakeup_fd: self.on_wakeup}
self.all_writers = []
self.timers = Timers()
self.ui_timers = Timers()
self.pending_ui_thread_calls = Queue()
self.write_dispatch_map = {}
set_boss(self)
cell_size.width, cell_size.height = set_font_family(opts.font_family, opts.font_size)
self.opts, self.args = opts, args
self.glfw_window = glfw_window
glfw_window.framebuffer_size_callback = self.on_window_resize
glfw_window.char_mods_callback = self.on_text_input
glfw_window.key_callback = self.on_key
glfw_window.mouse_button_callback = self.on_mouse_button
glfw_window.scroll_callback = self.on_mouse_scroll
glfw_window.cursor_pos_callback = self.on_mouse_move
glfw_window.window_focus_callback = self.on_focus
self.tab_manager = TabManager(opts, args, startup_session)
self.sprites = Sprites()
self.cell_program = ShaderProgram(*cell_shader)
self.cursor_program = ShaderProgram(*cursor_shader)
self.borders_program = BordersProgram()
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
self.sprites.do_layout(cell_size.width, cell_size.height)
self.glfw_window.set_click_cursor(False)
self.show_mouse_cursor()
self.start_cursor_blink()
def signal_received(self):
try:
data = os.read(self.signal_fd, io.DEFAULT_BUFFER_SIZE)
except BlockingIOError:
return
if data:
signals = struct.unpack('%uB' % len(data), data)
if signal.SIGINT in signals or signal.SIGTERM in signals:
if not self.shutting_down:
self.glfw_window.set_should_close(True)
glfw_post_empty_event()
@property
def current_tab_bar_height(self):
return self.tab_manager.tab_bar_height
def __iter__(self):
return iter(self.tab_manager)
def iterwindows(self):
for t in self:
yield from t
def queue_action(self, func, *args):
self.action_queue.put((func, args))
wakeup()
def on_wakeup(self):
if not self.shutting_down:
drain_read(self.read_wakeup_fd)
while True:
try:
func, args = self.action_queue.get_nowait()
except Empty:
break
try:
func(*args)
except Exception:
import traceback
traceback.print_exc()
def add_child_fd(self, child_fd, read_ready, write_ready):
self.read_dispatch_map[child_fd] = read_ready
self.write_dispatch_map[child_fd] = write_ready
def remove_child_fd(self, child_fd):
self.read_dispatch_map.pop(child_fd, None)
self.write_dispatch_map.pop(child_fd, None)
try:
self.all_writers.remove(child_fd)
except Exception:
pass
def queue_ui_action(self, func, *args):
self.pending_ui_thread_calls.put((func, args))
glfw_post_empty_event()
def close_window(self, window=None):
' Can be called in either thread, will first kill the child, then remove the window from the gui '
if window is None:
window = self.active_window
if current_thread() is main_thread:
self.queue_action(self.close_window, window)
else:
self.remove_child_fd(window.child_fd)
window.destroy()
self.queue_ui_action(self.gui_close_window, window)
def close_tab(self, tab=None):
' Can be called in either thread, will first kill all children, then remove the tab from the gui '
if tab is None:
tab = self.active_tab
if current_thread() is main_thread:
self.queue_action(self.close_tab, tab)
else:
for window in tab:
self.remove_child_fd(window.child_fd)
window.destroy()
self.queue_ui_action(self.gui_close_window, window)
def run(self):
if self.args.profile:
import cProfile
import pstats
pr = cProfile.Profile()
pr.enable()
self.loop()
if self.args.profile:
pr.disable()
pr.create_stats()
s = pstats.Stats(pr)
s.dump_stats(self.args.profile)
def loop(self):
while not self.shutting_down:
all_readers = list(self.read_dispatch_map)
all_writers = [w.child_fd for w in self.iterwindows() if w.write_buf]
readers, writers, _ = select.select(all_readers, all_writers, [], self.timers.timeout())
for r in readers:
self.read_dispatch_map[r]()
for w in writers:
self.write_dispatch_map[w]()
self.timers()
for w in self.iterwindows():
if w.screen.is_dirty():
self.timers.add_if_missing(self.screen_update_delay, w.update_screen)
@callback
def on_window_resize(self, window, w, h):
# debounce resize events
self.pending_resize = True
yield
self.timers.add(0.02, self.apply_pending_resize, w, h)
def apply_pending_resize(self, w, h):
viewport_size.width, viewport_size.height = w, h
self.tab_manager.resize()
self.resize_gl_viewport = True
self.pending_resize = False
glfw_post_empty_event()
def tabbar_visibility_changed(self):
self.tab_manager.resize(only_tabs=True)
glfw_post_empty_event()
@property
def active_tab(self):
return self.tab_manager.active_tab
def is_tab_visible(self, tab):
return self.active_tab is tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
@callback
def on_text_input(self, window, codepoint, mods):
data = interpret_text_event(codepoint, mods)
if data:
w = self.active_window
if w is not None:
yield w
w.write_to_child(data)
@callback
def on_key(self, window, key, scancode, action, mods):
is_key_pressed[key] = action == GLFW_PRESS
self.start_cursor_blink()
self.cursor_blink_zero_time = monotonic()
if action == GLFW_PRESS or action == GLFW_REPEAT:
func = get_shortcut(self.opts.keymap, mods, key)
if func is not None:
f = getattr(self, func, None)
if f is not None:
passthrough = f()
if not passthrough:
return
tab = self.active_tab
if tab is None:
return
window = self.active_window
if window is not None:
yield window
if func is not None:
f = getattr(tab, func, getattr(window, func, None))
if f is not None:
passthrough = f()
if not passthrough:
return
if window.screen.auto_repeat_enabled or action == GLFW_PRESS:
if window.char_grid.scrolled_by and key not in MODIFIER_KEYS:
window.scroll_end()
data = interpret_key_event(key, scancode, mods)
if data:
window.write_to_child(data)
@callback
def on_focus(self, window, focused):
w = self.active_window
if w is not None:
yield w
w.focus_changed(focused)
def display_scrollback(self, data):
if self.opts.scrollback_in_new_tab:
self.queue_ui_action(self.display_scrollback_in_new_tab, data)
else:
tab = self.active_tab
if tab is not None:
tab.new_special_window(SpecialWindow(self.opts.scrollback_pager, data, _('History')))
def window_for_pos(self, x, y):
tab = self.active_tab
if tab is not None:
for w in tab:
if w.is_visible_in_layout and w.contains(x, y):
return w
def in_tab_bar(self, y):
th = self.current_tab_bar_height
return th > 0 and y >= viewport_size.height - th
@callback
def on_mouse_button(self, window, button, action, mods):
mouse_button_pressed[button] = action == GLFW_PRESS
self.show_mouse_cursor()
x, y = mouse_cursor_pos
w = self.window_for_pos(x, y)
if w is None:
if self.in_tab_bar(y):
if button == GLFW_MOUSE_BUTTON_1 and action == GLFW_PRESS:
self.tab_manager.activate_tab_at(x)
return
focus_moved = False
old_focus = self.active_window
tab = self.active_tab
yield
if button == GLFW_MOUSE_BUTTON_1 and w is not old_focus:
tab.set_active_window(w)
focus_moved = True
if focus_moved:
if old_focus is not None and not old_focus.destroyed:
old_focus.focus_changed(False)
w.focus_changed(True)
w.on_mouse_button(button, action, mods)
@callback
def on_mouse_move(self, window, xpos, ypos):
mouse_cursor_pos[:2] = xpos, ypos
self.show_mouse_cursor()
w = self.window_for_pos(xpos, ypos)
if w is not None:
yield w
w.on_mouse_move(xpos, ypos)
else:
self.change_mouse_cursor(self.in_tab_bar(ypos))
@callback
def on_mouse_scroll(self, window, x, y):
self.show_mouse_cursor()
w = self.window_for_pos(*mouse_cursor_pos)
if w is not None:
yield w
w.on_mouse_scroll(x, y)
# GUI thread API {{{
def show_mouse_cursor(self):
self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_NORMAL)
if self.opts.mouse_hide_wait > 0:
self.ui_timers.add(self.opts.mouse_hide_wait, self.hide_mouse_cursor)
def hide_mouse_cursor(self):
self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
def change_mouse_cursor(self, click=False):
self.glfw_window.set_click_cursor(click)
def start_cursor_blink(self):
self.cursor_blinking = True
if self.opts.cursor_stop_blinking_after > 0:
self.ui_timers.add(self.opts.cursor_stop_blinking_after, self.stop_cursor_blinking)
def stop_cursor_blinking(self):
self.cursor_blinking = False
def render(self):
if self.pending_resize:
return
if self.resize_gl_viewport:
glViewport(0, 0, viewport_size.width, viewport_size.height)
self.resize_gl_viewport = False
tab = self.active_tab
if tab is not None:
if tab.title != self.glfw_window_title:
self.glfw_window_title = tab.title
self.glfw_window.set_title(self.glfw_window_title)
with self.sprites:
self.sprites.render_dirty_cells()
tab.render()
render_data = {window: window.char_grid.prepare_for_render(self.sprites) for window in tab.visible_windows() if not window.needs_layout}
with self.cell_program:
self.tab_manager.render(self.cell_program, self.sprites)
for window, rd in render_data.items():
if rd is not None:
window.char_grid.render_cells(rd, self.cell_program, self.sprites)
active = self.active_window
rd = render_data.get(active)
if rd is not None:
draw_cursor = True
if self.cursor_blinking and self.opts.cursor_blink_interval > 0:
now = monotonic() - self.cursor_blink_zero_time
t = int(now * 1000)
d = int(self.opts.cursor_blink_interval * 1000)
n = t // d
draw_cursor = n % 2 == 0
self.ui_timers.add_if_missing(((n + 1) * d / 1000) - now, glfw_post_empty_event)
if draw_cursor:
with self.cursor_program:
active.char_grid.render_cursor(rd, self.cursor_program)
def gui_close_window(self, window):
if window.char_grid.buffer_id is not None:
self.sprites.destroy_sprite_map(window.char_grid.buffer_id)
window.char_grid.buffer_id = None
for tab in self.tab_manager:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
self.tab_manager.remove(tab)
tab.destroy()
if len(self.tab_manager) == 0:
if not self.shutting_down:
self.glfw_window.set_should_close(True)
glfw_post_empty_event()
def destroy(self):
# Must be called in the main thread as it manipulates signal handlers
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.shutting_down = True
wakeup()
self.join()
for t in self.tab_manager:
t.destroy()
del self.tab_manager
self.sprites.destroy()
del self.sprites
del self.glfw_window
def paste_from_clipboard(self):
text = self.glfw_window.get_clipboard_string()
if text:
w = self.active_window
if w is not None:
self.queue_action(w.paste, text)
def next_tab(self):
self.queue_action(self.tab_manager.next_tab)
def previous_tab(self):
self.queue_action(self.tab_manager.next_tab, -1)
def new_tab(self):
self.tab_manager.new_tab()
def move_tab_forward(self):
self.queue_action(self.tab_manager.move_tab, 1)
def move_tab_backward(self):
self.queue_action(self.tab_manager.move_tab, -1)
def display_scrollback_in_new_tab(self, data):
self.tab_manager.new_tab(special_window=SpecialWindow(self.opts.scrollback_pager, data, _('History')))
class MainWindow(QMainWindow):
def __init__(self):
super(QMainWindow, self).__init__()
self.img_num = 10
self.scale = 5
self.fq = Queue(maxsize=self.img_num)
self.fq_lock = Lock()
self.UI_resized = False
self.img_wnds = [None] * self.img_num
self.init_UI()
def put_fq(self, frame):
self.img_w = int(frame.shape[1] / self.scale)
self.img_h = int(frame.shape[0] / self.scale)
show = cv2.resize(frame, (self.img_w, self.img_h),
interpolation=cv2.INTER_AREA)
show = cv2.cvtColor(show, cv2.COLOR_BGR2RGB)
show = QtGui.QImage(show.data, show.shape[1], show.shape[0],
QtGui.QImage.Format_RGB888)
with self.fq_lock:
if self.fq.full():
self.fq.get_nowait()
self.fq.put_nowait(show)
#resize windows
if not self.UI_resized:
self.UI_resized = True
self.scroll.show()
self.imgarea.resize(self.img_w, self.img_h *
self.img_num) #######设置scrollarea幕布的尺寸
self.setFixedWidth(frame.shape[1] + self.img_w + 50)
self.setFixedHeight(frame.shape[0])
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame = QtGui.QImage(frame.data, frame.shape[1], frame.shape[0],
QtGui.QImage.Format_RGB888)
self.video_wnd.setPixmap(QtGui.QPixmap.fromImage(frame))
self.show_fq()
def init_UI(self):
self.imgarea = QWidget()
vbox = QVBoxLayout()
for img_cnt in range(self.img_num):
self.img_wnds[img_cnt] = QLabel()
vbox.addWidget(self.img_wnds[img_cnt])
self.imgarea.setLayout(vbox)
self.scroll = QScrollArea()
self.scroll.setWidget(self.imgarea)
self.scroll.hide()
self.video_wnd = QLabel()
hbox = QHBoxLayout()
hbox.addWidget(self.video_wnd)
hbox.addWidget(self.scroll)
w = QWidget()
w.setLayout(hbox)
self.setCentralWidget(w)
menu_height = 0
self.resize(641, 481 + menu_height)
self.video_wnd.setGeometry(
QtCore.QRect(0, menu_height, 641, 481 + menu_height))
self.move(100, 100)
def show_fq(self):
with self.fq_lock:
for frame_idx in range(self.fq.qsize()):
self.img_wnds[frame_idx].setPixmap(
QtGui.QPixmap.fromImage(self.fq.queue[frame_idx]))
class ItemBuffer(threading.Thread, Singleton):
dedup = None
def __init__(self, table_folder):
if not hasattr(self, "_table_item"):
super(ItemBuffer, self).__init__()
self._thread_stop = False
self._is_adding_to_db = False
self._table_folder = table_folder
self._items_queue = Queue(maxsize=MAX_ITEM_COUNT)
self._db = RedisDB()
self._table_item = setting.TAB_ITEM
self._table_request = setting.TAB_REQUSETS.format(table_folder=table_folder)
self._item_tables = {
# 'xxx_item': {'tab_item': 'xxx:xxx_item'} # 记录item名与redis中item名对应关系
}
self._item_update_keys = {
# 'xxx:xxx_item': ['id', 'name'...] # 记录redis中item名与需要更新的key对应关系
}
self._export_data = ExportData() if setting.ADD_ITEM_TO_MYSQL else None
self.db_tip()
if setting.ITEM_FILTER_ENABLE and not self.__class__.dedup:
self.__class__.dedup = Dedup(to_md5=False)
def db_tip(self):
msg = "\n"
if setting.ADD_ITEM_TO_MYSQL:
msg += "item 自动入mysql\n"
if setting.ADD_ITEM_TO_REDIS:
msg += "item 自动入redis\n"
if msg == "\n":
log.warning("*** 请注意检查item是否入库 !!!")
else:
log.info(msg)
def run(self):
while not self._thread_stop:
self.flush()
tools.delay_time(0.5)
self.close()
def stop(self):
self._thread_stop = True
def put_item(self, item):
self._items_queue.put(item)
def flush(self):
try:
items = []
update_items = []
requests = []
callbacks = []
items_fingerprints = []
data_count = 0
while not self._items_queue.empty():
data = self._items_queue.get_nowait()
data_count += 1
# data 分类
if callable(data):
callbacks.append(data)
elif isinstance(data, UpdateItem):
update_items.append(data)
elif isinstance(data, Item):
items.append(data)
if setting.ITEM_FILTER_ENABLE:
items_fingerprints.append(data.fingerprint)
else: # request-redis
requests.append(data)
if data_count >= UPLOAD_BATCH_MAX_SIZE:
self.__add_item_to_db(
items, update_items, requests, callbacks, items_fingerprints
)
items = []
update_items = []
requests = []
callbacks = []
items_fingerprints = []
data_count = 0
if data_count:
self.__add_item_to_db(
items, update_items, requests, callbacks, items_fingerprints
)
except Exception as e:
log.exception(e)
def get_items_count(self):
return self._items_queue.qsize()
def is_adding_to_db(self):
return self._is_adding_to_db
def __dedup_items(self, items, items_fingerprints):
"""
去重
@param items:
@param items_fingerprints:
@return: 返回去重后的items, items_fingerprints
"""
if not items:
return items, items_fingerprints
is_exists = self.__class__.dedup.get(items_fingerprints)
is_exists = is_exists if isinstance(is_exists, list) else [is_exists]
dedup_items = []
dedup_items_fingerprints = []
items_count = dedup_items_count = dup_items_count = 0
while is_exists:
item = items.pop(0)
items_fingerprint = items_fingerprints.pop(0)
is_exist = is_exists.pop(0)
items_count += 1
if not is_exist:
dedup_items.append(item)
dedup_items_fingerprints.append(items_fingerprint)
dedup_items_count += 1
else:
dup_items_count += 1
log.info(
"待入库数据 {} 条， 重复 {} 条，实际待入库数据 {} 条".format(
items_count, dup_items_count, dedup_items_count
)
)
return dedup_items, dedup_items_fingerprints
def __pick_items(self, items, is_update_item=False):
"""
将每个表之间的数据分开 拆分后 原items为空
@param items:
@param is_update_item:
@return:
"""
datas_dict = {
# 'xxx:xxx_item': [{}, {}] redis 中的item名与对应的数据
}
while items:
item = items.pop(0)
# 取item下划线格式的名
# 下划线类的名先从dict中取，没有则现取，然后存入dict。加快下次取的速度
item_name = item.item_name
item_table = self._item_tables.get(item_name)
if not item_table:
item_name_underline = item.name_underline
tab_item = self._table_item.format(
table_folder=self._table_folder, item_name=item_name_underline
)
item_table = {}
item_table["tab_item"] = tab_item
self._item_tables[item_name] = item_table
else:
tab_item = item_table.get("tab_item")
# 入库前的回调
item.per_to_db()
if tab_item not in datas_dict:
datas_dict[tab_item] = []
datas_dict[tab_item].append(item.to_dict)
if is_update_item and tab_item not in self._item_update_keys:
self._item_update_keys[tab_item] = item.update_key
return datas_dict
def __export_to_db(self, tab_item, datas, is_update=False, update_keys=()):
export_success = False
# 打点 校验
to_table = tools.get_info(tab_item, ":s_(.*?)_item", fetch_one=True)
item_name = to_table + "_item"
self.check_datas(table=to_table, datas=datas)
if setting.ADD_ITEM_TO_MYSQL: # 任务表需要入mysql
if isinstance(setting.ADD_ITEM_TO_MYSQL, (list, tuple)):
for item in setting.ADD_ITEM_TO_MYSQL:
if item in item_name:
export_success = (
self._export_data.export_items(tab_item, datas)
if not is_update
else self._export_data.update_items(
tab_item, datas, update_keys=update_keys
)
)
else:
export_success = (
self._export_data.export_items(tab_item, datas)
if not is_update
else self._export_data.update_items(
tab_item, datas, update_keys=update_keys
)
)
if setting.ADD_ITEM_TO_REDIS:
if isinstance(setting.ADD_ITEM_TO_REDIS, (list, tuple)):
for item in setting.ADD_ITEM_TO_REDIS:
if item in item_name:
self._db.sadd(tab_item, datas)
export_success = True
log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item))
break
else:
self._db.sadd(tab_item, datas)
export_success = True
log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item))
return export_success
def __add_item_to_db(
self, items, update_items, requests, callbacks, items_fingerprints
):
export_success = False
self._is_adding_to_db = True
# 去重
if setting.ITEM_FILTER_ENABLE:
items, items_fingerprints = self.__dedup_items(items, items_fingerprints)
# 分捡
items_dict = self.__pick_items(items)
update_items_dict = self.__pick_items(update_items, is_update_item=True)
# item批量入库
while items_dict:
tab_item, datas = items_dict.popitem()
log.debug(
"""
-------------- item 批量入库 --------------
表名: %s
datas: %s
"""
% (tab_item, tools.dumps_json(datas, indent=16))
)
export_success = self.__export_to_db(tab_item, datas)
# 执行批量update
while update_items_dict:
tab_item, datas = update_items_dict.popitem()
log.debug(
"""
-------------- item 批量更新 --------------
表名: %s
datas: %s
"""
% (tab_item, tools.dumps_json(datas, indent=16))
)
update_keys = self._item_update_keys.get(tab_item)
export_success = self.__export_to_db(
tab_item, datas, is_update=True, update_keys=update_keys
)
# 执行回调
while callbacks:
try:
callback = callbacks.pop(0)
callback()
except Exception as e:
log.exception(e)
# 删除做过的request
if requests:
self._db.zrem(self._table_request, requests)
# 去重入库
if export_success and setting.ITEM_FILTER_ENABLE:
if items_fingerprints:
self.__class__.dedup.add(items_fingerprints, skip_check=True)
self._is_adding_to_db = False
def check_datas(self, table, datas):
"""
打点 记录总条数及每个key情况
@param table: 表名
@param datas: 数据 列表
@return:
"""
pass
def close(self):
pass
class Process(object):
def __init__(self, commands, context, cwd):
startupinfo = None
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
self._proc = subprocess.Popen(commands,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
startupinfo=startupinfo,
cwd=cwd)
self._eof = False
self._context = context
self._queue_out = Queue()
self._thread = Thread(target=self.enqueue_output)
self._thread.start()
def eof(self):
return self._eof
def kill(self):
if not self._proc:
return
if self._proc.poll() is None:
# _proc is running
self._proc.kill()
self._proc.wait()
self._proc = None
self._queue_out = None
self._thread.join(1.0)
self._thread = None
def enqueue_output(self):
for line in self._proc.stdout:
if not self._queue_out:
return
self._queue_out.put(
line.decode(self._context['encoding'],
errors='replace').strip('\r\n'))
def communicate(self, timeout):
if not self._proc:
return ([], [])
start = time()
outs = []
if self._queue_out.empty():
sleep(0.1)
while not self._queue_out.empty() and time() < start + timeout:
outs.append(self._queue_out.get_nowait())
if self._thread.is_alive() or not self._queue_out.empty():
return (outs, [])
_, errs = self._proc.communicate(timeout=timeout)
errs = errs.decode(self._context['encoding'],
errors='replace').splitlines()
self._eof = True
self._proc = None
self._thread = None
self._queue = None
return (outs, errs)
class TesterManager(object):
"""Manage evaluate data."""
def __init__(self, config_info, broker_master, s3_result_path=None):
self.config_info = config_info
self.s3_path = s3_result_path
self.processed_model_count = 0
self.record_station_buf = dict()
self.eval_rewards = deque([], maxlen=50)
self.recv_broker = broker_master.register("eval_result")
self.send_broker = broker_master.recv_local_q
self.eval_info_queue = Queue()
eval_info = self.config_info.get("benchmark", {}).get("eval", dict())
self.max_instance = eval_info.get("evaluator_num", 2)
self.eval_interval = eval_info.get(
"gap", XBConf.default_train_interval_per_eval)
self.max_step_per_episode = eval_info.get("max_step_per_episode",
18000)
self.used_node = dict()
self.avail_node = list(((i, "test{}".format(tid))
for i in range(broker_master.node_num)
for tid in range(self.max_instance)))
self.last_eval_index = -9999
def check_finish_stat(self, target_model_count):
"""Check finish status."""
return self.processed_model_count >= target_model_count
def if_eval(self, train_count):
if train_count - self.last_eval_index < self.eval_interval:
return False
return True
def to_eval(self, weights, train_count, actual_step, elapsed_time,
train_reward, train_loss):
# update current evaluate index
self.last_eval_index = train_count
train_info = {
"train_index": train_count,
"sample_step": actual_step,
"elapsed_sec": elapsed_time, # time.time() - abs_start,
"train_reward": train_reward,
"loss": np.nan,
}
if type(train_loss) in (float, np.float64, np.float32, np.float16,
np.float):
train_info.update({"loss": train_loss})
self.record_station_buf.update({train_count: train_info})
self.put_test_model({train_count: weights})
# return self.get_avail_node()
def fetch_eval_result(self):
"""Fetch eval results with no wait."""
ret = list()
while True:
try:
item = self.eval_info_queue.get_nowait()
ret.append(item)
except Empty as err:
break
return ret
def _parse_eval_result_and_archive(self, eval_result):
_model_index = eval_result[-1][
"train_count"] # model receive is a list
# fixme: only a model been test in an agent.
_agent_id = list(eval_result[0].keys())[0]
# find the eval model info, and update the eval reward
if _model_index in self.record_station_buf:
eval_info_dict = self.record_station_buf.pop(_model_index)
else:
eval_info_dict = dict()
print("-->", eval_result)
return # test.api will use print replace write db record
eval_info_dict.update({
"agent_id":
_agent_id,
"eval_reward":
np.nanmean(eval_result[0][_agent_id]["reward"]),
"eval_name":
_model_index,
})
# custom evaluate
for _key in ("custom_criteria", "battle_won"):
if _key not in eval_result[0][_agent_id].keys():
continue
eval_info_dict.update(
{_key: np.nanmean(eval_result[0][_agent_id][_key])})
self.eval_info_queue.put(eval_info_dict)
def recv_result(self):
"""Recieve test result."""
while True:
recv_data = self.recv_broker.recv()
result_data = get_msg_data(recv_data)
self.processed_model_count += 1
logging.debug("result_data: \n{}".format(result_data))
_meta = result_data[1]
self.used_node[(_meta["broker_id"], _meta["test_id"])] -= 1
self._parse_eval_result_and_archive(result_data)
def put_test_model(self, model_weights):
"""Send test model."""
key = self.get_avail_node()
ctr_info = {"cmd": "eval", "broker_id": key[0], "test_id": key[1]}
eval_cmd = message(model_weights, **ctr_info)
self.send_broker.send(eval_cmd)
logging.debug("put evaluate model: {}".format(type(model_weights)))
self.used_node[key] += 1
def send_create_evaluator_msg(self, broker_id, test_id):
"""Create evaluator."""
config = deepcopy(self.config_info)
config.update({"test_id": test_id})
create_cmd = message(config,
cmd="create_evaluator",
broker_id=broker_id)
self.send_broker.send(create_cmd)
def get_avail_node(self):
"""Get available test node."""
if self.used_node:
min_key = min(self.used_node, key=self.used_node.get)
if self.used_node.get(min_key) < 1 or not self.avail_node:
return min_key
# create new evaluator
new_key = self.avail_node.pop(0)
self.send_create_evaluator_msg(*new_key) # (broker_id, test_id)
self.used_node.update({new_key: 0})
logging.info("create evaluator: {}".format(new_key))
return new_key
def start(self):
t = threading.Thread(target=self.recv_result)
t.start()
class Unit(Entity):
"""
A unit is an entity that can have its own entitys. (e.g. a Gunner that owns its fired bullets)
"""
def __init__(self,
player_number: int,
sprite: UnitSprite = None,
max_entities: int = -1,
nb_lives: int = 1,
surviving_entities: bool = False,
speed: int = 150):
"""
Instantiates a unit in the game
Args:
sprite: The sprite to draw on the board
max_entities: The maximum number of entities for this unit (-1 = infinite)
nb_lives: The number of lives this unit has before it dies
surviving_entities: If true, the entities of this unit won't die with this unit
"""
super().__init__(sprite, nb_lives=nb_lives, speed=speed)
self.survivingentitys = surviving_entities
self.lastAction = None
self.currentAction = None
self.playerNumber = player_number
self._entitiesSpriteGroup = pygame.sprite.Group()
self._entitiesQueue = Queue()
self._entitiesList = []
self._maxentitys = max_entities
def setLastAction(self, last_action):
"""
Sets the last action of this unit
Args:
last_action: The last action performed by this unit
"""
self.lastAction = last_action
def setCurrentAction(self, current_action):
"""
Sets the current action of this unit
Args:
current_action: The action this unit is currently performing
"""
self.currentAction = current_action
def draw(self, surface: pygame.Surface) -> None:
"""
Draws the unit and its entities
Args:
surface: The surface the unit and its entity will be drawn on
"""
super().draw(surface)
self._entitiesSpriteGroup.draw(surface)
def kill(self) -> None:
"""
Kills the unit and all its entity if "surviving entities" was set to False at the creation of this unit
"""
super().kill()
if not self.isAlive():
if not self.survivingentitys:
while len(self._entitiesSpriteGroup) != 0:
self.removeOldestentity()
def addentity(self, entity: Entity) -> None:
"""
Adds an entity linked to this unit
Args:
entity: the entity to add to this unit
"""
if 0 <= self._maxentitys <= len(self._entitiesList):
self.removeOldestentity()
if self._entitiesQueue is not None:
self._entitiesQueue.put(entity)
self._entitiesList.append(entity)
if entity.sprite is not None:
self._entitiesSpriteGroup.add(entity.sprite)
def removeOldestentity(self) -> None:
"""
Removes the oldest entity belonging to this unit-
"""
try:
oldest_entity = self._entitiesList.pop(0)
oldest_entity.kill()
if self._entitiesQueue is not None:
queue_first = self._entitiesQueue.get_nowait() # type: Entity
assert queue_first is oldest_entity
if oldest_entity.sprite is not None:
self._entitiesSpriteGroup.remove(oldest_entity.sprite)
except Empty:
pass
def removeentity(self, entity: Entity) -> None:
"""
Removes the given entity from the belongings of this unit
Args:
entity: The entity to remove
"""
temp_queue = Queue()
try:
while True: # Will stop when the Empty exception comes out from the Queue
current = self._entitiesQueue.get_nowait()
if current is not entity:
temp_queue.put(current)
except Empty:
pass
self._entitiesList.remove(entity)
self._entitiesQueue = temp_queue
entity.kill()
if entity.sprite is not None:
self._entitiesSpriteGroup.remove(entity.sprite)
def hasentity(self, entity: Entity) -> bool:
"""
Checks if the given entity belongs to this unit
Args:
entity: The entity to check
Returns: True if the given entity belongs to this unit
"""
return entity in self._entitiesList or (
self._entitiesSpriteGroup is not None
and self._entitiesSpriteGroup.has(entity.sprite))
def getentitys(self) -> List[Entity]:
"""
Returns: A list containing all the entities of this unit
"""
return self._entitiesList
def getentitysSpriteGroup(self) -> pygame.sprite.Group:
"""
Gets the "SpriteGroup" of this unit, including all of its entity. Can be used for pygame's collision check
"""
return self._entitiesSpriteGroup
def isColliding(self, other_sprite_group) -> bool:
"""
Returns True if this unit or any of its entity collide with the other unit or any of its entity.
(Uses the pygame collision check)
Args:
other_sprite_group: The other Sprite group with which we want to check if there is a collision
Returns: True if the two group are colliding, False otherwise
"""
for sprite in other_sprite_group.sprites: # type: pygame.sprite.Sprite
if pygame.sprite.collide_rect(self.sprite, sprite):
return True
return False
def __deepcopy__(self, memo={}):
cls = self.__class__
result = cls.__new__(cls)
memo[id(self)] = result
for k, v in self.__dict__.items():
if k != "_drawable" and k != "_entitiesQueue":
value = deepcopy(v, memo)
else:
value = None
setattr(result, k, value)
return result
def interpreter(state, env):
global dimension_process, game_state, t, q, prev_step
player1 = state[0]
player2 = state[1]
### 1.1: Initialize dimensions in the background within the orchestrator if we haven't already ###
if dimension_process is None:
# dimension_process = Popen(["ts-node", "-P", path.abspath(path.join(dir_path, "dimensions/tsconfig.json")), path.abspath(path.join(dir_path, "dimensions/run.ts"))], stdin=PIPE, stdout=PIPE)
try:
dimension_process = Popen([
"node",
path.abspath(path.join(dir_path, "dimensions/main.js"))
],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
except FileNotFoundError:
import warnings
warnings.warn("Node not installed")
return state
# following 4 lines from https://stackoverflow.com/questions/375427/a-non-blocking-read-on-a-subprocess-pipe-in-python
q = Queue()
t = Thread(target=enqueue_output, args=(dimension_process.stdout, q))
t.daemon = True # thread dies with the program
t.start()
atexit.register(cleanup_dimensions)
# filter out actions such as debug annotations so they aren't saved
filter_actions(state, env)
### 1.2: Initialize a blank state game if new episode is starting ###
if env.done:
# TODO: allow resetting to a specific state
# print("Initialize game", "steps", len(env.steps), "prev_step", prev_step)
# last_state = None
# if prev_step >= len(env.steps):
# last_state = env.steps[-1]
# prev_step = len(env.steps)
# print("prev_step now", prev_step)
if "seed" in env.configuration:
seed = env.configuration["seed"]
else:
seed = math.floor(random.random() * 1e9)
env.configuration["seed"] = seed
if "loglevel" in env.configuration:
loglevel = env.configuration["loglevel"]
else:
loglevel = 0 # warnings, 1: errors, 0: none
env.configuration["loglevel"] = loglevel
if "annotations" in env.configuration:
annotations = env.configuration["annotations"]
else:
annotations = False # warnings, 1: errors, 0: none
env.configuration["annotations"] = annotations
if "width" in env.configuration:
width = env.configuration["width"]
else:
width = -1 # -1 for randomly selected
env.configuration["width"] = width
if "height" in env.configuration:
height = env.configuration["height"]
else:
height = -1 # -1 for randomly selected
env.configuration["height"] = height
initiate = {
"type": "start",
"agent_names": [], # unsure if this is provided?
"config": env.configuration
}
# if last_state is not None:
# initiate["state"] = last_state
dimension_process.stdin.write((json.dumps(initiate) + "\n").encode())
dimension_process.stdin.flush()
agent1res = get_message(dimension_process)
agent2res = get_message(dimension_process)
match_obs_meta = get_message(dimension_process)
player1.observation.player = 0
player2.observation.player = 1
player1.observation.updates = agent1res
# player2.observation.updates = agent2res # duplicated and not added
player1.observation.globalCityIDCount = match_obs_meta[
"globalCityIDCount"]
player1.observation.globalUnitIDCount = match_obs_meta[
"globalUnitIDCount"]
player1.observation.width = match_obs_meta["width"]
player1.observation.height = match_obs_meta["height"]
game_state = Game()
game_state._initialize(agent1res)
return state
# print("prev_step", prev_step, "stored steps", len(env.steps))
# prev_step += 1
### 2. : Pass in actions (json representation along with id of who made that action), agent information (id, status) to dimensions via stdin
dimension_process.stdin.write((json.dumps(state) + "\n").encode())
dimension_process.stdin.flush()
### 3.1 : Receive and parse the observations returned by dimensions via stdout
agent1res = json.loads(dimension_process.stderr.readline())
agent2res = json.loads(dimension_process.stderr.readline())
game_state._update(agent1res)
# receive meta info such as global ID and map sizes for purposes of being able to start from specific state
match_obs_meta = json.loads(dimension_process.stderr.readline())
match_status = json.loads(dimension_process.stderr.readline())
while True:
try:
line = q.get_nowait()
except Empty:
# no standard error received, break
break
else:
# standard error output received, print it out
print(line.decode(), file=sys.stderr, end='')
### 3.2 : Send observations to each agent through here. Like dimensions, first observation can include initialization stuff, then we do the looping
player1.observation.updates = agent1res
player1.observation.globalCityIDCount = match_obs_meta["globalCityIDCount"]
player1.observation.globalUnitIDCount = match_obs_meta["globalUnitIDCount"]
player1.observation.width = match_obs_meta["width"]
player1.observation.height = match_obs_meta["height"]
# player2.observation.updates = agent2res # duplicated and not added
player1.observation.player = 0
player2.observation.player = 1
### 3.3 : handle rewards
# reward here is defined as the sum of number of city tiles
player1.reward = compute_reward(game_state.players[0])
player2.reward = compute_reward(game_state.players[1])
player1.observation.reward = int(player1.reward)
player2.observation.reward = int(player2.reward)
### 3.4 Handle finished match status
if match_status["status"] == "finished":
if player1.status == "ACTIVE":
player1.status = "DONE"
if player2.status == "ACTIVE":
player2.status = "DONE"
return state
class MainTab(SingleLinkableSetting, QFrame):
set_progressbar_signal = pyqtSignal(int) # max progress
increment_progressbar_signal = pyqtSignal(int) # increment value
update_label_signal = pyqtSignal(str)
write_to_terminal_signal = pyqtSignal(str)
add_result_signal = pyqtSignal(object) # Result
add_url_analysis_result_signal = pyqtSignal(object) # URLAnalysisResult
add_run_to_queue_signal = pyqtSignal(object) # Run object (or a subclass)
update_run_status_signal = pyqtSignal(int, str) # run_id, status_str
print_results_signal = pyqtSignal(
) # called after a run finishes to flush the results queue before printing "Done"
def __init__(self):
QFrame.__init__(self)
SingleLinkableSetting.__init__(self, "api_key")
# lazy loaded, see self#library
self._library = None
self.print_results_signal.connect(self.print_results)
self.write_to_terminal_signal.connect(self.write)
self.q = Queue()
# `AnalysisResult`s get put here when we get a url scheme event, we need
# to create the visualizer from the main thread so we use a queue to
# kick back the work to the main thread. This is checked at the same
# time `self.q` is
self.url_analysis_q = Queue()
# reset at the beginning of every run, used to print something after
# every run only if a cheat wasn't found
self.show_no_cheat_found = True
self.print_results_event = threading.Event()
self.cg_q = Queue()
self.helper_thread_running = False
self.runs = [] # Run objects for cancelling runs
self.run_id = 0
self.visualizer = None
terminal = QTextEdit(self)
terminal.setFocusPolicy(Qt.ClickFocus)
terminal.setReadOnly(True)
terminal.ensureCursorVisible()
self.terminal = terminal
self.run_button = RunButton()
self.run_button.setFixedHeight(30)
self.run_button.setText("Run")
font = self.run_button.font()
font.setPointSize(15)
self.run_button.setFont(font)
self.run_button.clicked.connect(self.add_circleguard_run)
# disable button if no api_key is stored
self.on_setting_changed("api_key", get_setting("api_key"))
investigate_label = QLabel("Investigate For:")
investigate_label.setFixedWidth(130)
investigate_label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.investigation_checkboxes = InvestigationCheckboxes()
investigations = WidgetCombiner(investigate_label,
self.investigation_checkboxes, self)
investigations.setFixedHeight(25)
self.loadable_creation = LoadableCreation()
self.investigation_checkboxes.similarity_cb.checkbox.stateChanged.connect(
self.loadable_creation.similarity_cb_state_changed)
layout = QGridLayout()
layout.addWidget(investigations, 0, 0, 1, 16)
layout.addWidget(self.loadable_creation, 1, 0, 4, 16)
layout.addWidget(self.terminal, 5, 0, 2, 16)
layout.addWidget(self.run_button, 7, 0, 1, 16)
self.setLayout(layout)
def on_setting_changed(self, setting, text):
# TODO `setting_value` should be updated automatically by
# `LinkableSetting`
self.setting_value = text
self.run_button.setEnabled(text != "")
def write(self, message):
self.terminal.append(str(message).strip())
self.scroll_to_bottom()
def scroll_to_bottom(self):
cursor = QTextCursor(self.terminal.document())
cursor.movePosition(QTextCursor.End)
self.terminal.setTextCursor(cursor)
@property
def library(self):
if not self._library:
from slider import Library
self._library = Library(get_setting("cache_dir"))
return self._library
def add_circleguard_run(self):
# osu!api v2 uses a client secret which is still 40 chars long but
# includes characters after f, unlike v1's key which is in hex.
if re.search("[g-z]", self.setting_value):
message_box = QMessageBox()
message_box.setTextFormat(Qt.RichText)
message_box.setText(
"Your api key is invalid. You are likely using "
"an api v2 key.\n"
"Please ensure your api key comes from "
"<a href=\"https://osu.ppy.sh/p/api\">https://osu.ppy.sh/p/api</a>."
)
message_box.exec()
return
if not self.loadable_creation.check_and_mark_required_fields():
return
# loadables can have their required fields filled, but with invalid
# input, like having "1a" as a span. In this case we print the error
# and early return.
try:
loadables = self.loadable_creation.cg_loadables()
except ValueError as e:
self.write_to_terminal_signal.emit(
f"<div style='color:#ff5252'>Invalid arguments:</div> {str(e)}"
)
return
enabled_investigations = self.investigation_checkboxes.enabled_investigations(
)
# if no loadables have been filled out, don't proceed
if not loadables:
return
# similarly for investigations, but give a heads up since this mistake
# is slightly subtler
if not enabled_investigations:
# nbsp is necessary because apparently ending with a closing tag
# makes qt not recognize it and causes all text afterwards to be
# affected by the div's color
self.write_to_terminal_signal.emit(
"<div style='color:#ff5252'>You must select "
"at least one investigation before running</div> ")
return
run = Run(loadables, enabled_investigations, self.run_id,
threading.Event())
self.runs.append(run)
self.add_run_to_queue_signal.emit(run)
self.cg_q.put(run)
self.run_id += 1
# called every 1/4 seconds by timer, but force a recheck so we don't
# wait for that delay
self.check_circleguard_queue()
def check_circleguard_queue(self):
def _check_circleguard_queue(self):
try:
while True:
run = self.cg_q.get_nowait()
# occurs if run is canceled before being started, it will
# still stop before actually loading anything but we don't
# want the labels to flicker
if run.event.wait(0):
continue
thread = threading.Thread(target=self.run_circleguard,
args=[run])
self.helper_thread_running = True
thread.start()
# run sequentially to not confuse user with terminal output
thread.join()
except Empty:
self.helper_thread_running = False
return
# don't launch another thread running cg if one is already running,
# or else multiple runs will occur at once (defeats the whole purpose
# of sequential runs)
if not self.helper_thread_running:
# have to do a double thread use if we start the threads in
# the main thread and .join, it will block the gui thread (very bad).
thread = threading.Thread(target=_check_circleguard_queue,
args=[self])
thread.start()
def run_circleguard(self, run):
from circleguard import (Circleguard, UnknownAPIException, ReplayPath,
NoInfoAvailableException, Loader,
LoadableContainer, replay_pairs)
class TrackerLoader(Loader, QObject):
"""
A circleguard.Loader subclass that emits a signal when the loader is
ratelimited. It inherits from QObject to allow us to use qt signals.
"""
ratelimit_signal = pyqtSignal(
int) # length of the ratelimit in seconds
check_stopped_signal = pyqtSignal()
# how often to emit check_stopped_signal when ratelimited, in seconds
INTERVAL = 0.250
def __init__(self, key, cacher=None):
Loader.__init__(self, key, cacher)
QObject.__init__(self)
def _ratelimit(self, length):
# sometimes the ratelimit length can get very close to zero or
# even negative due to network request time and rounding. As
# displaying a zero or negative wait time is confusing, don't
# wait for any less than 2 seconds.
length = max(length, 2)
self.ratelimit_signal.emit(length)
# how many times to wait for 1/4 second (rng standing for range)
# we do this loop in order to tell run_circleguard to check if
# the run was canceled, or the application quit, instead of
# hanging on a long time.sleep
rng = math.ceil(length / self.INTERVAL)
for _ in range(rng):
time.sleep(self.INTERVAL)
self.check_stopped_signal.emit()
self.update_label_signal.emit("Loading Replays")
self.update_run_status_signal.emit(run.run_id, "Loading Replays")
# reset every run
self.show_no_cheat_found = True
event = run.event
try:
core_cache = get_setting("cache_dir") + "circleguard.db"
slider_cache = get_setting("cache_dir")
should_cache = get_setting("caching")
cg = Circleguard(get_setting("api_key"),
core_cache,
slider_dir=slider_cache,
cache=should_cache,
loader=TrackerLoader)
def _ratelimited(length):
message = get_setting("message_ratelimited")
ts = datetime.now()
self.write_to_terminal_signal.emit(
message.format(s=length, ts=ts))
self.update_label_signal.emit("Ratelimited")
self.update_run_status_signal.emit(run.run_id, "Ratelimited")
def _check_event(event):
"""
Checks the given event to see if it is set. If it is, the run has been canceled
through the queue tab or by the application being quit, and this thread exits
through sys.exit(0). If the event is not set, returns silently.
"""
if event.wait(0):
self.update_label_signal.emit("Canceled")
self.set_progressbar_signal.emit(-1)
# the loadables of a run may be extremely large. We keep
# this run object around in a list so people can cancel it;
# if we want to prevent a memory leak we need to remove a
# run's loadables (we don't need it anymore after this run
# finishes).
run.loadables = None
# may seem dirty, but actually relatively clean since it only affects this thread.
# Any cleanup we may want to do later can occur here as well
sys.exit(0)
cg.loader.ratelimit_signal.connect(_ratelimited)
cg.loader.check_stopped_signal.connect(partial(
_check_event, event))
if "Similarity" in run.enabled_investigations:
loadables1 = []
loadables2 = []
for loadable in run.loadables:
if loadable.sim_group == 1:
loadables1.append(loadable)
else:
loadables2.append(loadable)
lc1 = LoadableContainer(loadables1)
lc2 = LoadableContainer(loadables2)
else:
lc = LoadableContainer(run.loadables)
message_loading_info = get_setting("message_loading_info").format(
ts=datetime.now())
self.write_to_terminal_signal.emit(message_loading_info)
if "Similarity" in run.enabled_investigations:
cg.load_info(lc1)
cg.load_info(lc2)
replays1 = lc1.all_replays()
replays2 = lc2.all_replays()
all_replays = replays1 + replays2
else:
cg.load_info(lc)
all_replays = lc.all_replays()
replays1 = all_replays
replays2 = []
num_replays = len(all_replays)
self.set_progressbar_signal.emit(num_replays)
message_loading_replays = get_setting(
"message_loading_replays").format(ts=datetime.now(),
num_replays=num_replays)
self.write_to_terminal_signal.emit(message_loading_replays)
def _skip_replay_with_message(replay, message):
self.write_to_terminal_signal.emit(message)
# the replay very likely (perhaps certainly) didn't get
# loaded if the above exception fired. just remove it.
# If two different loadables both contain this problematic
# replay, we will attempt to remove it from the list twice.
# Guard against this by checking for membership first.
if replay in all_replays:
all_replays.remove(replay)
# check has already been initialized with the replay,
# remove it here too or cg will try and load it again
# when the check is ran
if replay in replays1:
replays1.remove(replay)
if replay in replays2:
replays2.remove(replay)
# `[:]` implicitly copies the list, so we don't run into trouble
# when removing elements from it while iterating
for replay in all_replays[:]:
_check_event(event)
try:
cg.load(replay)
# circleparse sets replay_data to None if it's not a std
# replay, which means replay.has_data() will be false, so
# we need to do this check first to give a better error
# message than "the replay is not available for download",
# which is incorrect for local replays.
if isinstance(replay,
ReplayPath) and not replay.has_data():
_skip_replay_with_message(
replay, "<div style='color:#ff5252'>The replay " +
str(replay) + " is " +
"not an osu!std replay.</div> We currently only support std replays. "
"This replay has been skipped because of this.")
elif not replay.has_data():
_skip_replay_with_message(
replay, "<div style='color:#ff5252'>The replay " +
str(replay) + " is " +
"not available for download.</div> This is likely because it is not in the top 1k scores of "
"the beatmap. This replay has been skipped because of this."
)
except NoInfoAvailableException as e:
_skip_replay_with_message(
replay, "<div style='color:#ff5252'>The replay " +
str(replay) + " does "
"not exist.</div>\nDouble check your map and/or user id. This replay has "
"been skipped because of this.")
except UnknownAPIException as e:
_skip_replay_with_message(
replay,
"<div style='color:#ff5252'>The osu! api provided an invalid "
"response:</div> " + str(e) + ". The replay " +
str(replay) + " has been skipped because of this.")
except LZMAError as e:
_skip_replay_with_message(
replay,
"<div style='color:#ff5252'>lzma error while parsing a replay:</div> "
+ str(e) +
". The replay is either corrupted or has no replay data. The replay "
+ str(replay) + " has been skipped because of this.")
except Exception as e:
# print full traceback here for more debugging info.
# Don't do it for the previous exceptions because the
# cause of those is well understood, but that's not
# necessarily the case for generic exceptions here.
# attempting to use divs/spans here to color the beginning
# of this string made the traceback collapse down into
# one line with none of the usual linebreaks, I'm not
# sure why. So you win qt, no red color for this error.
_skip_replay_with_message(
replay, "error while loading a replay: " + str(e) +
"\n" + traceback.format_exc() + "The replay " +
str(replay) + " has been skipped because of this.")
finally:
self.increment_progressbar_signal.emit(1)
if "Similarity" in run.enabled_investigations:
lc1.loaded = True
lc2.loaded = True
else:
lc.loaded = True
# change progressbar into an undetermined state (animation with
# stripes sliding horizontally) to indicate we're processing
# the data
self.set_progressbar_signal.emit(0)
setting_end_dict = {
"Similarity": "steal",
"Unstable Rate": "relax",
"Snaps": "correction",
"Frametime": "timewarp",
"Manual Analysis": "analysis"
}
for investigation in run.enabled_investigations:
setting = "message_starting_" + setting_end_dict[investigation]
message_starting_investigation = get_setting(setting).format(
ts=datetime.now())
self.write_to_terminal_signal.emit(
message_starting_investigation)
if investigation == "Manual Analysis":
map_ids = [r.map_id for r in all_replays]
if len(set(map_ids)) > 1:
self.write_to_terminal_signal.emit(
"Manual analysis expected replays from a single map, "
f"but got replays from maps {set(map_ids)}. Please use a different Manual Analysis "
"Check for each map.")
self.update_label_signal.emit(
"Analysis Error (Multiple maps)")
self.update_run_status_signal.emit(
run.run_id, "Analysis Error (Multiple maps)")
self.set_progressbar_signal.emit(-1)
run.loadables = None
sys.exit(0)
# if a replay was removed from all_replays (eg if that replay was not available for download),
# and that leaves all_replays with no replays, we don't want to add a result because
# the rest of guard expects >=1 replay, leading to confusing errors.
if len(all_replays) != 0:
self.q.put(AnalysisResult(all_replays))
continue
self.update_label_signal.emit("Investigating Replays...")
self.update_run_status_signal.emit(run.run_id,
"Investigating Replays")
if investigation == "Similarity":
# core relies on the second replays list being the one empty
# if only one of the lists are empty, but we want to allow
# users to select whichever group they want if they're only
# using a single group, so switch the two if this is the
# case.
if replays2 and not replays1:
replays1, replays2 = replays2, replays1
pairs = replay_pairs(replays1, replays2)
for (replay1, replay2) in pairs:
_check_event(event)
sim = cg.similarity(replay1, replay2)
result = StealResult(sim, replay1, replay2)
self.q.put(result)
if investigation == "Unstable Rate":
for replay in all_replays:
_check_event(event)
# skip replays which have no map info
if replay.map_info.available():
try:
ur = cg.ur(replay)
# Sometimes, a beatmap will have a bugged download where it returns an empty response
# when we try to download it (https://github.com/ppy/osu-api/issues/171). Since peppy
# doesn't plan on fixing this for unranked beatmaps, we just ignore / skip the error
# in all cases.
# StopIteration is a bit of a weird exception to catch here, but because of how slider
# interacts with beatmaps it will attempt to call `next` on an empty generator if the
# beatmap is empty, which of course raises StopIteration.
except StopIteration:
import requests
if requests.get(
f"https://osu.ppy.sh/osu/{replay.map_id}"
).content == b"":
self.write_to_terminal_signal.emit(
"<div style='color:#ff5252'>The "
"map " + str(replay.map_id) +
"'s download is bugged</div>, so its ur cannot "
"be calculated. The replay " +
str(replay) +
" has been skipped because of this, "
"but please report this to the developers through discord or github so it can be "
"tracked.")
break
# If we happen to catch an unrelated error with this `except`, we still want to
# raise that so it can be tracked and fixed.
else:
raise
result = RelaxResult(ur, replay)
self.q.put(result)
else:
self.write_to_terminal_signal.emit(
"<div style='color:#ff5252'>The "
"replay " + str(replay) +
" has no map id</div>, so its ur cannot "
"be calculated. This replay has been skipped because of this."
)
if investigation == "Snaps":
max_angle = get_setting("correction_max_angle")
min_distance = get_setting("correction_min_distance")
only_on_hitobjs = get_setting("ignore_snaps_off_hitobjs")
for replay in all_replays:
_check_event(event)
snaps = cg.snaps(replay, max_angle, min_distance,
only_on_hitobjs)
result = CorrectionResult(snaps, replay)
self.q.put(result)
if investigation == "Frametime":
for replay in all_replays:
_check_event(event)
frametime = cg.frametime(replay)
frametimes = cg.frametimes(replay)
result = TimewarpResult(frametime, frametimes, replay)
self.q.put(result)
# flush self.q. Since the next investigation will be processed
# so quickly afterwards, we actually need to forcibly wait
# until the results have been printed before proceeding.
self.print_results_event.clear()
self.print_results_signal.emit()
self.print_results_event.wait()
self.set_progressbar_signal.emit(-1) # empty progressbar
# this event is necessary because `print_results` will set
# `show_no_cheat_found`, and since it happens asynchronously we need
# to wait for it to finish before checking it. So we clear it here,
# then wait for it to get set before proceeding.
self.print_results_event.clear()
# 'flush' self.q so there's no more results left and message_finished_investigation
# won't print before results from that investigation which looks strange.
self.print_results_signal.emit()
self.print_results_event.wait()
if self.show_no_cheat_found:
self.write_to_terminal_signal.emit(
get_setting("message_no_cheat_found").format(
ts=datetime.now()))
self.write_to_terminal_signal.emit(
get_setting("message_finished_investigation").format(
ts=datetime.now()))
# prevents an error when a user closes the application. Because
# we're running inside a new thread, if we don't do this, cg (and)
# the library) will get gc'd in another thread. Because library's
# ``__del__`` closes the sqlite connection, this causes:
# ```
# Traceback (most recent call last):
# File "/Users/tybug/Desktop/coding/osu/slider/slider/library.py", line 98, in __del__
# self.close()
# File "/Users/tybug/Desktop/coding/osu/slider/slider/library.py", line 94, in close
# self._db.close()
# sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread.
# The object was created in thread id 123145483210752 and this is thread id 4479481280.
# ```
cg.library.close()
except Exception:
# if the error happens before we set the progressbar it stays at
# 100%. make sure we reset it here
self.set_progressbar_signal.emit(-1)
log.exception(
"Error while running circlecore. Please "
"report this to the developers through discord or github.\n")
self.update_label_signal.emit("Idle")
self.update_run_status_signal.emit(run.run_id, "Finished")
# necessary to prevent memory leaks, as we keep the run object around
# after the run finishes, but don't need its loadables anymore.
run.loadables = None
def print_results(self):
try:
while True:
result = self.q.get_nowait()
whitelisted_ids = []
whitelist_file = get_setting("whitelist_file_location")
# whitelist_file is the empty strign if no file has been specified
if whitelist_file:
whitelist_file = Path(whitelist_file)
with open(whitelist_file) as f:
lines = f.readlines()
for line in lines:
# remove comment if there is one
line = line.rsplit("#")[0]
# allow whitespace on either end of the id
line = line.strip()
# allow empty lines anywhere in the file
if line == "":
continue
# TODO: nicer error if the line is an invalid id?
try:
line = int(line)
except ValueError:
self.write_to_terminal_signal.emit(
"<div style='color:#ff5252'>Invalid "
f"user id in whitelist file:</div> \"{line}\" is not a valid user id."
)
continue
whitelisted_ids.append(line)
ts = datetime.now() # ts = timestamp
message = None
ischeat = False
if isinstance(result, StealResult):
if result.earlier_replay.user_id in whitelisted_ids:
pass
elif result.similarity < get_setting("steal_max_sim"):
ischeat = True
message = get_setting("message_steal_found").format(
ts=ts,
sim=result.similarity,
r=result,
replay1=result.replay1,
replay2=result.replay2,
earlier_replay_mods_short_name=result.
earlier_replay.mods.short_name(),
earlier_replay_mods_long_name=result.
earlier_replay.mods.long_name(),
later_replay_mods_short_name=result.later_replay.
mods.short_name(),
later_replay_mods_long_name=result.later_replay.
mods.long_name())
elif result.similarity < get_setting(
"steal_max_sim_display"):
message = get_setting(
"message_steal_found_display").format(
ts=ts,
sim=result.similarity,
r=result,
replay1=result.replay1,
earlier_replay_mods_short_name=result.
earlier_replay.mods.short_name(),
earlier_replay_mods_long_name=result.
earlier_replay.mods.long_name(),
later_replay_mods_short_name=result.
later_replay.mods.short_name(),
later_replay_mods_long_name=result.
later_replay.mods.long_name())
if isinstance(result, RelaxResult):
if result.replay.user_id in whitelisted_ids:
pass
elif result.ur < get_setting("relax_max_ur"):
ischeat = True
message = get_setting("message_relax_found").format(
ts=ts,
r=result,
replay=result.replay,
ur=result.ur,
mods_short_name=result.replay.mods.short_name(),
mods_long_name=result.replay.mods.long_name())
elif result.ur < get_setting("relax_max_ur_display"):
message = get_setting(
"message_relax_found_display").format(
ts=ts,
r=result,
replay=result.replay,
ur=result.ur,
mods_short_name=result.replay.mods.short_name(
),
mods_long_name=result.replay.mods.long_name())
if isinstance(result, CorrectionResult):
if result.replay.user_id in whitelisted_ids:
pass
elif len(result.snaps) >= get_setting(
"correction_min_snaps"):
ischeat = True
snap_message = get_setting("message_correction_snaps")
snap_text = "\n".join([
snap_message.format(time=snap.time,
angle=snap.angle,
distance=snap.distance)
for snap in result.snaps
])
message = get_setting(
"message_correction_found").format(
ts=ts,
r=result,
replay=result.replay,
snaps=snap_text,
mods_short_name=result.replay.mods.short_name(
),
mods_long_name=result.replay.mods.long_name())
elif len(result.snaps) >= get_setting(
"correction_min_snaps_display"):
snap_message = get_setting("message_correction_snaps")
snap_text = "\n".join([
snap_message.format(time=snap.time,
angle=snap.angle,
distance=snap.distance)
for snap in result.snaps
])
message = get_setting(
"message_correction_found_display").format(
ts=ts,
r=result,
replay=result.replay,
snaps=snap_text,
mods_short_name=result.replay.mods.short_name(
),
mods_long_name=result.replay.mods.long_name())
if isinstance(result, TimewarpResult):
if result.replay.user_id in whitelisted_ids:
pass
elif result.frametime < get_setting(
"timewarp_max_frametime"):
ischeat = True
message = get_setting("message_timewarp_found").format(
ts=ts,
r=result,
replay=result.replay,
frametime=result.frametime,
mods_short_name=result.replay.mods.short_name(),
mods_long_name=result.replay.mods.long_name())
elif result.frametime < get_setting(
"timewarp_max_frametime_display"):
message = get_setting(
"message_timewarp_found_display").format(
ts=ts,
r=result,
replay=result.replay,
frametime=result.frametime,
mods_short_name=result.replay.mods.short_name(
),
mods_long_name=result.replay.mods.long_name())
if message or isinstance(result, AnalysisResult):
self.show_no_cheat_found = False
if message:
self.write(message)
if isinstance(result, AnalysisResult):
self.add_result_signal.emit(result)
else:
if ischeat:
if get_setting("alert_on_result"):
QApplication.beep()
QApplication.alert(self)
# add to Results Tab so it can be played back on demand
self.add_result_signal.emit(result)
except Empty:
try:
result = self.url_analysis_q.get_nowait()
self.add_url_analysis_result_signal.emit(result)
except Empty:
pass
self.print_results_event.set()
def visualize(self, replays, beatmap_id, result, start_at=0):
from circlevis import BeatmapInfo
# only run one instance at a time
if self.visualizer is not None:
self.visualizer.close()
snaps = []
if isinstance(result, CorrectionResult):
snaps = [snap.time for snap in result.snaps]
beatmap_info = BeatmapInfo(map_id=beatmap_id)
CGVisualizer = get_visualizer()
self.visualizer = CGVisualizer(beatmap_info, replays, snaps,
self.library)
self.visualizer.show()
if start_at != 0:
self.visualizer.seek_to(start_at)
self.visualizer.pause()
def visualize_from_url(self, result):
"""
called when our url scheme (circleguard://) was entered, giving
us a replay to visualize
"""
map_id = result.replays[0].map_id
if self.visualizer and self.visualizer.isVisible(
) and self.visualizer.replays and self.visualizer.replays[
0].map_id == map_id:
# pause even if we're currently playing - it's important that this
# happens before seeking, or else the new frame won't correctly
# display
self.visualizer.force_pause()
# if we're visualizing the same replay that's in the url, just jump
# to the new timestamp
self.visualizer.seek_to(result.timestamp)
return
# otherwise visualize as normal (which will close any existing
# visualizers)
self.visualize(result.replays,
map_id,
result,
start_at=result.timestamp)
class Palette2:
def __init__(self, config):
self.printer = config.get_printer()
self.reactor = self.printer.get_reactor()
try:
self.virtual_sdcard = self.printer.load_object(
config, "virtual_sdcard")
except config.error:
raise self.printer.config_error(
"Palette 2 requires [virtual_sdcard] to work,"
" please add it to your config!")
try:
self.pause_resume = self.printer.load_object(
config, "pause_resume")
except config.error:
raise self.printer.config_error(
"Palette 2 requires [pause_resume] to work,"
" please add it to your config!")
self.gcode_move = self.printer.load_object(config, 'gcode_move')
self.gcode = self.printer.lookup_object("gcode")
self.gcode.register_command("PALETTE_CONNECT",
self.cmd_Connect,
desc=self.cmd_Connect_Help)
self.gcode.register_command("PALETTE_DISCONNECT",
self.cmd_Disconnect,
desc=self.cmd_Disconnect_Help)
self.gcode.register_command("PALETTE_CLEAR",
self.cmd_Clear,
desc=self.cmd_Clear_Help)
self.gcode.register_command("PALETTE_CUT",
self.cmd_Cut,
desc=self.cmd_Cut_Help)
self.gcode.register_command("PALETTE_SMART_LOAD",
self.cmd_Smart_Load,
desc=self.cmd_Smart_Load_Help)
self.serial = None
self.serial_port = config.get("serial")
if not self.serial_port:
raise config.error("Invalid serial port specific for Palette 2")
self.baud = config.getint("baud", default=115200)
self.feedrate_splice = config.getfloat("feedrate_splice",
default=0.8,
minval=0.,
maxval=1.)
self.feedrate_normal = config.getfloat("feedrate_normal",
default=1.0,
minval=0.,
maxval=1.)
self.auto_load_speed = config.getint("auto_load_speed", 2)
self.auto_cancel_variation = config.getfloat("auto_cancel_variation",
default=None,
minval=0.01,
maxval=0.2)
# Omega code matchers
self.omega_header = [None] * 9
omega_handlers = ["O" + str(i) for i in range(33)]
for cmd in omega_handlers:
func = getattr(self, 'cmd_' + cmd, None)
desc = getattr(self, 'cmd_' + cmd + '_help', None)
if func:
self.gcode.register_command(cmd, func, desc=desc)
else:
self.gcode.register_command(cmd, self.cmd_OmegaDefault)
self._reset()
self.read_timer = None
self.read_buffer = ""
self.read_queue = Queue()
self.write_timer = None
self.write_queue = Queue()
self.heartbeat_timer = None
self.heartbeat = None
self.signal_disconnect = False
self.is_printing = False
self.smart_load_timer = None
def _reset(self):
self.files = []
self.is_setup_complete = False
self.is_splicing = False
self.is_loading = False
self.remaining_load_length = None
self.omega_algorithms = []
self.omega_algorithms_counter = 0
self.omega_splices = []
self.omega_splices_counter = 0
self.omega_pings = []
self.omega_pongs = []
self.omega_current_ping = None
self.omega_header = [None] * 9
self.omega_header_counter = 0
self.omega_last_command = ""
self.omega_drivers = []
def _check_P2(self, gcmd=None):
if self.serial:
return True
if gcmd:
gcmd.respond_info(INFO_NOT_CONNECTED)
return False
cmd_Connect_Help = ("Connect to the Palette 2")
def cmd_Connect(self, gcmd):
if self.serial:
gcmd.respond_info(
"Palette 2 serial port is already active, disconnect first")
return
self.signal_disconnect = False
logging.info("Connecting to Palette 2 on port (%s) at (%s)" %
(self.serial_port, self.baud))
try:
self.serial = serial.Serial(self.serial_port,
self.baud,
timeout=0,
write_timeout=0)
except SerialException:
gcmd.respond_info("Unable to connect to the Palette 2")
return
with self.write_queue.mutex:
self.write_queue.queue.clear()
with self.read_queue.mutex:
self.read_queue.queue.clear()
self.read_timer = self.reactor.register_timer(self._run_Read,
self.reactor.NOW)
self.write_timer = self.reactor.register_timer(self._run_Write,
self.reactor.NOW)
self.heartbeat_timer = self.reactor.register_timer(
self._run_Heartbeat, self.reactor.NOW)
# Tell the device we're alive
self.write_queue.put("\n")
self.write_queue.put(COMMAND_FIRMWARE)
self._wait_for_heartbeat()
cmd_Disconnect_Help = ("Disconnect from the Palette 2")
def cmd_Disconnect(self, gmcd=None):
self.gcode.respond_info("Disconnecting from Palette 2")
if self.serial:
self.serial.close()
self.serial = None
self.reactor.unregister_timer(self.read_timer)
self.reactor.unregister_timer(self.write_timer)
self.reactor.unregister_timer(self.heartbeat_timer)
self.read_timer = None
self.write_timer = None
self.heartbeat = None
self.is_printing = False
cmd_Clear_Help = ("Clear the input and output of the Palette 2")
def cmd_Clear(self, gcmd):
logging.info("Clearing Palette 2 input and output")
if self._check_P2(gcmd):
for l in COMMAND_CLEAR:
self.write_queue.put(l)
cmd_Cut_Help = ("Cut the outgoing filament")
def cmd_Cut(self, gcmd):
logging.info("Cutting outgoing filament in Palette 2")
if self._check_P2(gcmd):
self.write_queue.put(COMMAND_CUT)
cmd_Smart_Load_Help = ("Automatically load filament through the extruder")
def cmd_Smart_Load(self, gcmd):
if self._check_P2(gcmd):
if not self.is_loading:
gcmd.respond_info(
"Cannot auto load when the Palette 2 is not ready")
return
self.p2cmd_O102(params=None)
def cmd_OmegaDefault(self, gcmd):
logging.debug("Omega Code: %s" % (gcmd.get_command()))
if self._check_P2(gcmd):
self.write_queue.put(gcmd.get_commandline())
def _wait_for_heartbeat(self):
startTs = self.reactor.monotonic()
currTs = startTs
while self.heartbeat is None and self.heartbeat < (
currTs - SETUP_TIMEOUT) and startTs > (currTs - SETUP_TIMEOUT):
currTs = self.reactor.pause(currTs + 1.)
if self.heartbeat < (currTs - SETUP_TIMEOUT):
self.signal_disconnect = True
raise self.printer.command_error("No response from Palette 2")
cmd_O1_help = (
"Initialize the print, and check connection with the Palette 2")
def cmd_O1(self, gcmd):
logging.info("Initializing print with Pallete 2")
if not self._check_P2(gcmd):
raise self.printer.command_error(
"Cannot initialize print, palette 2 is not connected")
self.reactor.update_timer(self.heartbeat_timer, self.reactor.NOW)
self._wait_for_heartbeat()
self.write_queue.put(gcmd.get_commandline())
self.gcode.respond_info("Palette 2 waiting on user to complete setup")
self.pause_resume.send_pause_command()
cmd_O9_help = ("Reset print information")
def cmd_O9(self, gcmd):
logging.info("Print finished, resetting Palette 2 state")
if self._check_P2(gcmd):
self.write_queue.put(gcmd.get_commandline())
self.is_printing = False
def cmd_O21(self, gcmd):
logging.debug("Omega version: %s" % (gcmd.get_commandline()))
self._reset()
self.omega_header[0] = gcmd.get_commandline()
self.is_printing = True
def cmd_O22(self, gcmd):
logging.debug("Omega printer profile: %s" % (gcmd.get_commandline()))
self.omega_header[1] = gcmd.get_commandline()
def cmd_O23(self, gcmd):
logging.debug("Omega slicer profile: %s" % (gcmd.get_commandline()))
self.omega_header[2] = gcmd.get_commandline()
def cmd_O24(self, gcmd):
logging.debug("Omega PPM: %s" % (gcmd.get_commandline()))
self.omega_header[3] = gcmd.get_commandline()
def cmd_O25(self, gcmd):
logging.debug("Omega inputs: %s" % (gcmd.get_commandline()))
self.omega_header[4] = gcmd.get_commandline()
drives = self.omega_header[4][4:].split()
for idx in range(len(drives)):
state = drives[idx][:2]
if state == "D1":
drives[idx] = "U" + str(60 + idx)
self.omega_drives = [d for d in drives if d != "D0"]
logging.info("Omega drives: %s" % self.omega_drives)
def cmd_O26(self, gcmd):
logging.debug("Omega splices %s" % (gcmd.get_commandline()))
self.omega_header[5] = gcmd.get_commandline()
def cmd_O27(self, gcmd):
logging.debug("Omega pings: %s" % (gcmd.get_commandline()))
self.omega_header[6] = gcmd.get_commandline()
def cmd_O28(self, gcmd):
logging.debug("Omega MSF NA: %s" % (gcmd.get_commandline()))
self.omega_header[7] = gcmd.get_commandline()
def cmd_O29(self, gcmd):
logging.debug("Omega MSF NH: %s" % (gcmd.get_commandline()))
self.omega_header[8] = gcmd.get_commandline()
def cmd_O30(self, gcmd):
try:
param_drive = gcmd.get_commandline()[5:6]
param_distance = gcmd.get_commandline()[8:]
except IndexError:
gmcd.respond_info(
"Incorrect number of arguments for splice command")
try:
self.omega_splices.append((int(param_drive), param_distance))
except ValueError:
gcmd.respond_info("Incorrectly formatted splice command")
logging.debug("Omega splice command drive %s distance %s" %
(param_drive, param_distance))
def cmd_O31(self, gcmd):
if self._check_P2(gcmd):
self.omega_current_ping = gcmd.get_commandline()
logging.debug("Omega ping command: %s" % (gcmd.get_commandline()))
self.write_queue.put(COMMAND_PING)
self.gcode.create_gcode_command("G4", "G4", {"P": "10"})
def cmd_O32(self, gcmd):
logging.debug("Omega algorithm: %s" % (gcmd.get_commandline()))
self.omega_algorithms.append(gcmd.get_commandline())
def p2cmd_O20(self, params):
if not self.is_printing:
return
# First print, we can ignore
if params[0] == "D5":
logging.info("First print on Palette")
return
try:
n = int(params[0][1:])
except (TypeError, IndexError):
logging.error("O20 command has invalid parameters")
return
if n == 0:
logging.info("Sending omega header %s" % self.omega_header_counter)
self.write_queue.put(self.omega_header[self.omega_header_counter])
self.omega_header_counter = self.omega_header_counter + 1
elif n == 1:
logging.info("Sending splice info %s" % self.omega_splices_counter)
splice = self.omega_splices[self.omega_splices_counter]
self.write_queue.put("O30 D%d D%s" % (splice[0], splice[1]))
self.omega_splices_counter = self.omega_splices_counter + 1
elif n == 2:
logging.info("Sending current ping info %s" %
self.omega_current_ping)
self.write_queue.put(self.omega_current_ping)
elif n == 4:
logging.info("Sending algorithm info %s" %
self.omega_algorithms_counter)
self.write_queue.put(
self.omega_algorithms[self.omega_algorithms_counter])
self.omega_algorithms_counter = self.omega_algorithms_counter + 1
elif n == 8:
logging.info("Resending the last command to Palette 2")
self.write_queue.put(self.omega_last_command)
def p2cmd_O34(self, params):
if not self.is_printing:
return
def check_ping_variation(last_ping):
if self.auto_cancel_variation is not None:
ping_max = 100. + (self.auto_cancel_variation * 100.)
ping_min = 100. - (self.auto_cancel_variation * 100.)
if last_ping < ping_min or last_ping > ping_max:
logging.info("Ping variation is too high, "
"cancelling print")
self.gcode.run_script("CANCEL_PRINT")
if len(params) > 2:
percent = float(params[1][1:])
if params[0] == "D1":
number = len(self.omega_pings) + 1
d = {"number": number, "percent": percent}
logging.info("Ping %d, %d percent" % (number, percent))
self.omega_pings.append(d)
check_ping_variation(percent)
elif params[0] == "D2":
number = len(self.omega_pongs) + 1
d = {"number": number, "percent": percent}
logging.info("Pong %d, %d percent" % (number, percent))
self.omega_pongs.append(d)
def p2cmd_O40(self, params):
logging.info("Resume request from Palette 2")
self.pause_resume.send_resume_command()
def p2cmd_O50(self, params):
if len(params) > 1:
try:
fw = params[0][1:]
logging.info("Palette 2 firmware version %s detected" %
os.fwalk)
except (TypeError, IndexError):
logging.error("Unable to parse firmware version")
if fw < "9.0.9":
raise self.printer.command_error(
"Palette 2 firmware version is too old, "
"update to at least 9.0.9")
else:
self.files = [
file for (file, size) in self.virtual_sdcard.get_file_list(
check_subdirs=True) if ".mcf.gcode" in file
]
for file in self.files:
self.write_queue.put("%s D%s" % (COMMAND_FILENAME, file))
self.write_queue.put(COMMAND_FILENAMES_DONE)
def p2cmd_O53(self, params):
if len(params) > 1 and params[0] == "D1":
try:
idx = int(params[1][1:], 16)
file = self.files[::-1][idx]
self.gcode.run_script("SDCARD_PRINT_FILE FILENAME=%s" % file)
except (TypeError, IndexError):
logging.error("O53 has invalid command parameters")
def p2cmd_O88(self, params):
logging.error("Palette 2 error detected")
try:
error = int(params[0][1:], 16)
logging.error("Palette 2 error code %d" % error)
except (TypeError, IndexError):
logging.error("Unable to parse Palette 2 error")
def p2cmd_O97(self, params):
def printCancelling(params):
logging.info("Print Cancelling")
self.gcode.run_script("CLEAR_PAUSE")
self.gcode.run_script("CANCEL_PRINT")
def printCancelled(params):
logging.info("Print Cancelled")
self._reset()
def loadingOffsetStart(params):
logging.info("Waiting for user to load filament into printer")
self.is_loading = True
def loadingOffset(params):
self.remaining_load_length = int(params[1][1:])
logging.debug("Loading filamant remaining %d" %
self.remaining_load_length)
if self.remaining_load_length >= 0 and self.smart_load_timer:
logging.info("Smart load filament is complete")
self.reactor.unregister_timer(self.smart_load_timer)
self.smart_load_timer = None
self.is_loading = False
def feedrateStart(params):
logging.info("Setting feedrate to %f for splice" %
self.feedrate_splice)
self.is_splicing = True
self.gcode.run_script("M220 S%d" % (self.feedrate_splice * 100))
def feedrateEnd(params):
logging.info("Setting feedrate to %f splice done" %
self.feedrate_normal)
self.is_splicing = False
self.gcode.run_script("M220 S%d" % (self.feedrate_normal * 100))
matchers = []
if self.is_printing:
matchers = matchers + [
[printCancelling, 2, "U0", "D2"],
[printCancelled, 2, "U0", "D3"],
[loadingOffset, 2, "U39"],
[loadingOffsetStart, 1, "U39"],
]
matchers.append([feedrateStart, 3, "U25", "D0"])
matchers.append([feedrateEnd, 3, "U25", "D1"])
self._param_Matcher(matchers, params)
def p2cmd_O100(self, params):
logging.info("Pause request from Palette 2")
self.is_setup_complete = True
self.pause_resume.send_pause_command()
def p2cmd_O102(self, params):
toolhead = self.printer.lookup_object("toolhead")
if not toolhead.get_extruder().get_heater().can_extrude:
self.write_queue.put(COMMAND_SMART_LOAD_STOP)
self.gcode.respond_info(
"Unable to auto load filament, extruder is below minimum temp")
return
if self.smart_load_timer is None:
logging.info("Smart load starting")
self.smart_load_timer = self.reactor.register_timer(
self._run_Smart_Load, self.reactor.NOW)
def p2cmd(self, line):
t = line.split()
ocode = t[0]
params = t[1:]
params_count = len(params)
if params_count:
res = [i for i in params if i[0] == "D" or i[0] == "U"]
if not all(res):
logging.error("Omega parameters are invalid")
return
func = getattr(self, 'p2cmd_' + ocode, None)
if func is not None:
func(params)
def _param_Matcher(self, matchers, params):
# Match the command with the handling table
for matcher in matchers:
if len(params) >= matcher[1]:
match_params = matcher[2:]
res = all([
match_params[i] == params[i]
for i in range(len(match_params))
])
if res:
matcher[0](params)
return True
return False
def _run_Read(self, eventtime):
if self.signal_disconnect:
self.cmd_Disconnect()
return self.reactor.NEVER
# Do non-blocking reads from serial and try to find lines
while True:
try:
raw_bytes = self.serial.read()
except SerialException:
logging.error("Unable to communicate with the Palette 2")
self.cmd_Disconnect()
return self.reactor.NEVER
if len(raw_bytes):
text_buffer = self.read_buffer + raw_bytes.decode()
while True:
i = text_buffer.find("\n")
if i >= 0:
line = text_buffer[0:i + 1]
self.read_queue.put(line.strip())
text_buffer = text_buffer[i + 1:]
else:
break
self.read_buffer = text_buffer
else:
break
# Process any decoded lines from the device
while not self.read_queue.empty():
try:
text_line = self.read_queue.get_nowait()
except Empty:
pass
heartbeat_strings = [COMMAND_HEARTBEAT, "Connection Okay"]
if not any(x in text_line for x in heartbeat_strings):
logging.debug("%0.3f P2 -> : %s" % (eventtime, text_line))
# Received a heartbeat from the device
if text_line == COMMAND_HEARTBEAT:
self.heartbeat = eventtime
elif text_line[0] == "O":
self.p2cmd(text_line)
return eventtime + SERIAL_TIMER
def _run_Heartbeat(self, eventtime):
self.write_queue.put(COMMAND_HEARTBEAT)
eventtime = self.reactor.pause(eventtime + 5)
if self.heartbeat and self.heartbeat < (eventtime - HEARTBEAT_TIMEOUT):
logging.error("P2 has not responded to heartbeat")
if not self.is_printing or self.is_setup_complete:
self.cmd_Disconnect()
return self.reactor.NEVER
return eventtime + HEARTBEAT_SEND
def _run_Write(self, eventtime):
while not self.write_queue.empty():
try:
text_line = self.write_queue.get_nowait()
except Empty:
continue
if text_line:
self.omega_last_command = text_line
l = text_line.strip()
if COMMAND_HEARTBEAT not in l:
logging.debug("%s -> P2 : %s" %
(self.reactor.monotonic(), l))
terminated_line = "%s\n" % (l)
try:
self.serial.write(terminated_line.encode())
except SerialException:
logging.error("Unable to communicate with the Palette 2")
self.signal_disconnect = True
return self.reactor.NEVER
return eventtime + SERIAL_TIMER
def _run_Smart_Load(self, eventtime):
if not self.is_splicing and self.remaining_load_length < 0:
# Make sure toolhead class isn't busy
toolhead = self.printer.lookup_object("toolhead")
print_time, est_print_time, lookahead_empty = toolhead.check_busy(
eventtime)
idle_time = est_print_time - print_time
if not lookahead_empty or idle_time < 0.5:
return eventtime + \
max(0., min(1., print_time - est_print_time))
extrude = abs(self.remaining_load_length)
extrude = min(50, extrude / 2)
if extrude <= 10:
extrude = 1
logging.info("Smart loading %dmm filament with %dmm remaining" %
(extrude, abs(self.remaining_load_length)))
self.gcode.run_script("G92 E0")
self.gcode.run_script("G1 E%d F%d" %
(extrude, self.auto_load_speed * 60))
return self.reactor.NOW
return eventtime + AUTOLOAD_TIMER
def get_status(self, eventtime=None):
status = {
"ping": self.omega_pings[-1],
"remaining_load_length": self.remaining_load_length,
"is_splicing": self.is_splicing
}
return status
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
EbookDownloadMixin
):
'The main GUI'
proceed_requested = pyqtSignal(object, object)
book_converted = pyqtSignal(object, object)
shutting_down = False
def __init__(self, opts, parent=None, gui_debug=None):
global _gui
MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
self.setWindowIcon(QApplication.instance().windowIcon())
self.jobs_pointer = Pointer(self)
self.proceed_requested.connect(self.do_proceed,
type=Qt.QueuedConnection)
self.proceed_question = ProceedQuestion(self)
self.job_error_dialog = JobError(self)
self.keyboard = Manager(self)
_gui = self
self.opts = opts
self.device_connected = None
self.gui_debug = gui_debug
self.iactions = OrderedDict()
# Actions
for action in interface_actions():
if opts.ignore_plugins and action.plugin_path is not None:
continue
try:
ac = self.init_iaction(action)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if action.plugin_path is None:
raise
continue
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
self.add_iaction(ac)
self.load_store_plugins()
def init_iaction(self, action):
ac = action.load_actual_plugin(self)
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
action.actual_iaction_plugin_loaded = True
return ac
def add_iaction(self, ac):
acmap = self.iactions
if ac.name in acmap:
if ac.priority >= acmap[ac.name].priority:
acmap[ac.name] = ac
else:
acmap[ac.name] = ac
def load_store_plugins(self):
from calibre.gui2.store.loader import Stores
self.istores = Stores()
for store in available_store_plugins():
if self.opts.ignore_plugins and store.plugin_path is not None:
continue
try:
st = self.init_istore(store)
self.add_istore(st)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if store.plugin_path is None:
raise
continue
self.istores.builtins_loaded()
def init_istore(self, store):
st = store.load_actual_plugin(self)
st.plugin_path = store.plugin_path
st.base_plugin = store
store.actual_istore_plugin_loaded = True
return st
def add_istore(self, st):
stmap = self.istores
if st.name in stmap:
if st.priority >= stmap[st.name].priority:
stmap[st.name] = st
else:
stmap[st.name] = st
def initialize(self, library_path, db, listener, actions, show_gui=True):
opts = self.opts
self.preferences_action, self.quit_action = actions
self.library_path = library_path
self.library_broker = GuiLibraryBroker(db)
self.content_server = None
self.server_change_notification_timer = t = QTimer(self)
self.server_changes = Queue()
t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True)
self._spare_pool = None
self.must_restart_before_config = False
self.listener = Listener(listener)
self.check_messages_timer = QTimer()
self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk)
self.check_messages_timer.start(1000)
for ac in list(self.iactions.values()):
try:
ac.do_genesis()
except Exception:
# Ignore errors in third party plugins
import traceback
traceback.print_exc()
if getattr(ac, 'plugin_path', None) is None:
raise
self.donate_action = QAction(QIcon(I('donate.png')),
_('&Donate to support calibre'), self)
for st in list(self.istores.values()):
st.do_genesis()
MainWindowMixin.init_main_window_mixin(self, db)
# Jobs Button {{{
self.job_manager = JobManager()
self.jobs_dialog = JobsDialog(self, self.job_manager)
self.jobs_button = JobsButton(parent=self)
self.jobs_button.initialize(self.jobs_dialog, self.job_manager)
# }}}
LayoutMixin.init_layout_mixin(self)
DeviceMixin.init_device_mixin(self)
self.progress_indicator = ProgressIndicator(self)
self.progress_indicator.pos = (0, 20)
self.verbose = opts.verbose
self.get_metadata = GetMetadata()
self.upload_memory = {}
self.metadata_dialogs = []
self.default_thumbnail = None
self.tb_wrapper = textwrap.TextWrapper(width=40)
self.viewers = collections.deque()
self.system_tray_icon = None
if config['systray_icon']:
self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre')
if self.system_tray_icon is not None:
self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False)))
if not (iswindows or isosx):
self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-gui', self.system_tray_icon.icon()))
self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip())
self.system_tray_icon.setVisible(True)
self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip)
elif config['systray_icon']:
prints('Failed to create system tray icon, your desktop environment probably does not support the StatusNotifier spec')
self.system_tray_menu = QMenu(self)
self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '')
self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated)
self.system_tray_menu.addAction(self.donate_action)
self.eject_action = self.system_tray_menu.addAction(
QIcon(I('eject.png')), _('&Eject connected device'))
self.eject_action.setEnabled(False)
self.addAction(self.quit_action)
self.system_tray_menu.addAction(self.quit_action)
self.keyboard.register_shortcut('quit calibre', _('Quit calibre'),
default_keys=('Ctrl+Q',), action=self.quit_action)
if self.system_tray_icon is not None:
self.system_tray_icon.setContextMenu(self.system_tray_menu)
self.system_tray_icon.activated.connect(self.system_tray_icon_activated)
self.quit_action.triggered[bool].connect(self.quit)
self.donate_action.triggered[bool].connect(self.donate)
self.minimize_action = QAction(_('Minimize the calibre window'), self)
self.addAction(self.minimize_action)
self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(),
default_keys=(), action=self.minimize_action)
self.minimize_action.triggered.connect(self.showMinimized)
self.esc_action = QAction(self)
self.addAction(self.esc_action)
self.keyboard.register_shortcut('clear current search',
_('Clear the current search'), default_keys=('Esc',),
action=self.esc_action)
self.esc_action.triggered.connect(self.esc)
self.shift_esc_action = QAction(self)
self.addAction(self.shift_esc_action)
self.keyboard.register_shortcut('focus book list',
_('Focus the book list'), default_keys=('Shift+Esc',),
action=self.shift_esc_action)
self.shift_esc_action.triggered.connect(self.shift_esc)
self.ctrl_esc_action = QAction(self)
self.addAction(self.ctrl_esc_action)
self.keyboard.register_shortcut('clear virtual library',
_('Clear the virtual library'), default_keys=('Ctrl+Esc',),
action=self.ctrl_esc_action)
self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
self.alt_esc_action = QAction(self)
self.addAction(self.alt_esc_action)
self.keyboard.register_shortcut('clear additional restriction',
_('Clear the additional restriction'), default_keys=('Alt+Esc',),
action=self.alt_esc_action)
self.alt_esc_action.triggered.connect(self.clear_additional_restriction)
# ###################### Start spare job server ########################
QTimer.singleShot(1000, self.create_spare_pool)
# ###################### Location Manager ########################
self.location_manager.location_selected.connect(self.location_selected)
self.location_manager.unmount_device.connect(self.device_manager.umount_device)
self.location_manager.configure_device.connect(self.configure_connected_device)
self.location_manager.update_device_metadata.connect(self.update_metadata_on_device)
self.eject_action.triggered.connect(self.device_manager.umount_device)
# ################### Update notification ###################
UpdateMixin.init_update_mixin(self, opts)
# ###################### Search boxes ########################
SearchRestrictionMixin.init_search_restriction_mixin(self)
SavedSearchBoxMixin.init_saved_seach_box_mixin(self)
# ###################### Library view ########################
LibraryViewMixin.init_library_view_mixin(self, db)
SearchBoxMixin.init_search_box_mixin(self) # Requires current_db
self.library_view.model().count_changed_signal.connect(
self.iactions['Choose Library'].count_changed)
if not gprefs.get('quick_start_guide_added', False):
try:
add_quick_start_guide(self.library_view)
except:
import traceback
traceback.print_exc()
for view in ('library', 'memory', 'card_a', 'card_b'):
v = getattr(self, '%s_view' % view)
v.selectionModel().selectionChanged.connect(self.update_status_bar)
v.model().count_changed_signal.connect(self.update_status_bar)
self.library_view.model().count_changed()
self.bars_manager.database_changed(self.library_view.model().db)
self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
type=Qt.QueuedConnection)
# ########################## Tags Browser ##############################
TagBrowserMixin.init_tag_browser_mixin(self, db)
self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection)
# ######################## Search Restriction ##########################
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
# ########################## Cover Flow ################################
CoverFlowMixin.init_cover_flow_mixin(self)
self._calculated_available_height = min(max_available_height()-15,
self.height())
self.resize(self.width(), self._calculated_available_height)
self.build_context_menus()
for ac in list(self.iactions.values()):
try:
ac.gui_layout_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
if config['autolaunch_server']:
self.start_content_server()
self.read_settings()
self.finalize_layout()
self.bars_manager.start_animation()
self.set_window_title()
for ac in list(self.iactions.values()):
try:
ac.initialization_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
register_keyboard_shortcuts()
self.keyboard.finalize()
if show_gui:
# Note this has to come after restoreGeometry() because of
# https://bugreports.qt.io/browse/QTBUG-56831
self.show()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray:
self.hide_windows()
self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
self.save_layout_state()
# Collect cycles now
gc.collect()
QApplication.instance().shutdown_signal_received.connect(self.quit)
if show_gui and self.gui_debug is not None:
QTimer.singleShot(10, self.show_gui_debug_msg)
self.iactions['Connect Share'].check_smartdevice_menus()
QTimer.singleShot(1, self.start_smartdevice)
QTimer.singleShot(100, self.update_toggle_to_tray_action)
def show_gui_debug_msg(self):
info_dialog(self, _('Debug mode'), '<p>' +
_('You have started calibre in debug mode. After you '
'quit calibre, the debug log will be available in '
'the file: %s<p>The '
'log will be displayed automatically.')%self.gui_debug, show=True)
def esc(self, *args):
self.search.clear()
def shift_esc(self):
self.current_view().setFocus(Qt.OtherFocusReason)
def ctrl_esc(self):
self.apply_virtual_library()
self.current_view().setFocus(Qt.OtherFocusReason)
def start_smartdevice(self):
message = None
if self.device_manager.get_option('smartdevice', 'autostart'):
try:
message = self.device_manager.start_plugin('smartdevice')
except:
message = 'start smartdevice unknown exception'
prints(message)
import traceback
traceback.print_exc()
if message:
if not self.device_manager.is_running('Wireless Devices'):
error_dialog(self, _('Problem starting the wireless device'),
_('The wireless device driver had problems starting. '
'It said "%s"')%message, show=True)
self.iactions['Connect Share'].set_smartdevice_action_state()
def start_content_server(self, check_started=True):
from calibre.srv.embedded import Server
if not gprefs.get('server3_warning_done', False):
gprefs.set('server3_warning_done', True)
if os.path.exists(os.path.join(config_dir, 'server.py')):
try:
os.remove(os.path.join(config_dir, 'server.py'))
except EnvironmentError:
pass
warning_dialog(self, _('Content server changed!'), _(
'calibre 3 comes with a completely re-written content server.'
' As such any custom configuration you have for the content'
' server no longer applies. You should check and refresh your'
' settings in Preferences->Sharing->Sharing over the net'), show=True)
self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
self.content_server.state_callback = Dispatcher(
self.iactions['Connect Share'].content_server_state_changed)
if check_started:
self.content_server.start_failure_callback = \
Dispatcher(self.content_server_start_failed)
self.content_server.start()
def handle_changes_from_server(self, library_path, change_event):
if DEBUG:
prints('Received server change event: {} for {}'.format(change_event, library_path))
if self.library_broker.is_gui_library(library_path):
self.server_changes.put((library_path, change_event))
self.server_change_notification_timer.start()
def handle_changes_from_server_debounced(self):
if self.shutting_down:
return
changes = []
while True:
try:
library_path, change_event = self.server_changes.get_nowait()
except Empty:
break
if self.library_broker.is_gui_library(library_path):
changes.append(change_event)
if changes:
handle_changes(changes, self)
def content_server_start_failed(self, msg):
self.content_server = None
error_dialog(self, _('Failed to start Content server'),
_('Could not start the Content server. Error:\n\n%s')%msg,
show=True)
def resizeEvent(self, ev):
MainWindow.resizeEvent(self, ev)
self.search.setMaximumWidth(self.width()-150)
def create_spare_pool(self, *args):
if self._spare_pool is None:
num = min(detect_ncpus(), int(config['worker_limit']/2.0))
self._spare_pool = Pool(max_workers=num, name='GUIPool')
def spare_pool(self):
ans, self._spare_pool = self._spare_pool, None
QTimer.singleShot(1000, self.create_spare_pool)
return ans
def do_proceed(self, func, payload):
if callable(func):
func(payload)
def no_op(self, *args):
pass
def system_tray_icon_activated(self, r=False):
if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False):
if self.isVisible():
if self.isMinimized():
self.showNormal()
else:
self.hide_windows()
else:
self.show_windows()
if self.isMinimized():
self.showNormal()
@property
def is_minimized_to_tray(self):
return getattr(self, '__systray_minimized', False)
def ask_a_yes_no_question(self, title, msg, det_msg='',
show_copy_button=False, ans_when_user_unavailable=True,
skip_dialog_name=None, skipped_value=True):
if self.is_minimized_to_tray:
return ans_when_user_unavailable
return question_dialog(self, title, msg, det_msg=det_msg,
show_copy_button=show_copy_button,
skip_dialog_name=skip_dialog_name,
skip_dialog_skipped_value=skipped_value)
def update_toggle_to_tray_action(self, *args):
if hasattr(self, 'toggle_to_tray_action'):
self.toggle_to_tray_action.setText(
_('Hide main window') if self.isVisible() else _('Show main window'))
def hide_windows(self):
for window in QApplication.topLevelWidgets():
if isinstance(window, (MainWindow, QDialog)) and \
window.isVisible():
window.hide()
setattr(window, '__systray_minimized', True)
self.update_toggle_to_tray_action()
def show_windows(self, *args):
for window in QApplication.topLevelWidgets():
if getattr(window, '__systray_minimized', False):
window.show()
setattr(window, '__systray_minimized', False)
self.update_toggle_to_tray_action()
def test_server(self, *args):
if self.content_server is not None and \
self.content_server.exception is not None:
error_dialog(self, _('Failed to start Content server'),
str(self.content_server.exception)).exec_()
@property
def current_db(self):
return self.library_view.model().db
def refresh_all(self):
m = self.library_view.model()
m.db.data.refresh(clear_caches=False, do_search=False)
self.saved_searches_changed(recount=False)
m.resort()
m.research()
self.tags_view.recount()
def another_instance_wants_to_talk(self):
try:
msg = self.listener.queue.get_nowait()
except Empty:
return
if msg.startswith('launched:'):
import json
try:
argv = json.loads(msg[len('launched:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
if DEBUG:
error_dialog(self, 'Invalid message',
'Received an invalid message from other calibre instance.'
' Do you have multiple versions of calibre installed?',
det_msg='Invalid msg: %r' % msg, show=True)
argv = ()
if isinstance(argv, (list, tuple)) and len(argv) > 1:
files = [os.path.abspath(p) for p in argv[1:] if not os.path.isdir(p) and os.access(p, os.R_OK)]
if files:
self.iactions['Add Books'].add_filesystem_book(files)
self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive)
self.show_windows()
self.raise_()
self.activateWindow()
elif msg.startswith('refreshdb:'):
m = self.library_view.model()
m.db.new_api.reload_from_db()
self.refresh_all()
elif msg.startswith('shutdown:'):
self.quit(confirm_quit=False)
elif msg.startswith('bookedited:'):
parts = msg.split(':')[1:]
try:
book_id, fmt, library_id = parts[:3]
book_id = int(book_id)
m = self.library_view.model()
db = m.db.new_api
if m.db.library_id == library_id and db.has_id(book_id):
db.format_metadata(book_id, fmt, allow_cache=False, update_db=True)
db.update_last_modified((book_id,))
m.refresh_ids((book_id,))
except Exception:
import traceback
traceback.print_exc()
else:
print(msg)
def current_view(self):
'''Convenience method that returns the currently visible view '''
idx = self.stack.currentIndex()
if idx == 0:
return self.library_view
if idx == 1:
return self.memory_view
if idx == 2:
return self.card_a_view
if idx == 3:
return self.card_b_view
def booklists(self):
return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
def library_moved(self, newloc, copy_structure=False, allow_rebuild=False):
if newloc is None:
return
with self.library_broker:
default_prefs = None
try:
olddb = self.library_view.model().db
if copy_structure:
default_prefs = olddb.prefs
except:
olddb = None
if copy_structure and olddb is not None and default_prefs is not None:
default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata()
db = self.library_broker.prepare_for_gui_library_change(newloc)
if db is None:
try:
db = LibraryDatabase(newloc, default_prefs=default_prefs)
except apsw.Error:
if not allow_rebuild:
raise
import traceback
repair = question_dialog(self, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
'The rebuild may not be completely successful.')
% force_unicode(newloc, filesystem_encoding),
det_msg=traceback.format_exc()
)
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase(newloc, default_prefs=default_prefs)
else:
return
else:
return
self.library_path = newloc
prefs['library_path'] = self.library_path
self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device)
self.library_view.set_database(db)
self.tags_view.set_database(db, self.alter_tb)
self.library_view.model().set_book_on_device_func(self.book_on_device)
self.status_bar.clear_message()
self.search.clear()
self.saved_search.clear()
self.book_details.reset_info()
# self.library_view.model().count_changed()
db = self.library_view.model().db
self.iactions['Choose Library'].count_changed(db.count())
self.set_window_title()
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed(recount=False) # reload the search restrictions combo box
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
for action in list(self.iactions.values()):
action.library_changed(db)
self.library_broker.gui_library_changed(db, olddb)
if self.device_connected:
self.set_books_in_library(self.booklists(), reset=True)
self.refresh_ondevice()
self.memory_view.reset()
self.card_a_view.reset()
self.card_b_view.reset()
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
self.library_view.set_current_row(0)
# Run a garbage collection now so that it does not freeze the
# interface later
gc.collect()
def set_window_title(self):
db = self.current_db
restrictions = [x for x in (db.data.get_base_restriction_name(),
db.data.get_search_restriction_name()) if x]
restrictions = ' :: '.join(restrictions)
font = QFont()
if restrictions:
restrictions = ' :: ' + restrictions
font.setBold(True)
font.setItalic(True)
self.virtual_library.setFont(font)
title = '{0} - || {1}{2} ||'.format(
__appname__, self.iactions['Choose Library'].library_name(), restrictions)
self.setWindowTitle(title)
def location_selected(self, location):
'''
Called when a location icon is clicked (e.g. Library)
'''
page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3
self.stack.setCurrentIndex(page)
self.book_details.reset_info()
for x in ('tb', 'cb'):
splitter = getattr(self, x+'_splitter')
splitter.button.setEnabled(location == 'library')
for action in list(self.iactions.values()):
action.location_selected(location)
if location == 'library':
self.virtual_library_menu.setEnabled(True)
self.highlight_only_button.setEnabled(True)
self.vl_tabs.setEnabled(True)
else:
self.virtual_library_menu.setEnabled(False)
self.highlight_only_button.setEnabled(False)
self.vl_tabs.setEnabled(False)
# Reset the view in case something changed while it was invisible
self.current_view().reset()
self.set_number_of_books_shown()
self.update_status_bar()
def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None):
if not hasattr(self, '_modeless_dialogs'):
self._modeless_dialogs = []
minz = self.is_minimized_to_tray
if self.isVisible():
for x in list(self._modeless_dialogs):
if not x.isVisible():
self._modeless_dialogs.remove(x)
try:
if 'calibre.ebooks.DRMError' in job.details:
if not minz:
from calibre.gui2.dialogs.drm_error import DRMErrorMessage
d = DRMErrorMessage(self, _('Cannot convert') + ' ' +
job.description.split(':')[-1].partition('(')[-1][:-1])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s')%title
msg += '<p>'+_('''
Many older e-book reader devices are incapable of displaying
EPUB files that have internal components over a certain size.
Therefore, when converting to EPUB, calibre automatically tries
to split up the EPUB into smaller sized pieces. For some
files that are large undifferentiated blocks of text, this
splitting fails.
<p>You can <b>work around the problem</b> by either increasing the
maximum split size under <i>EPUB output</i> in the conversion dialog,
or by turning on Heuristic Processing, also in the conversion
dialog. Note that if you make the maximum split size too large,
your e-book reader may have trouble with the EPUB.
''')
if not minz:
d = error_dialog(self, _('Conversion Failed'), msg,
det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details:
if not minz:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s') % title
idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:')
msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')),
job.details[idx:].partition(':')[2].strip())
d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.web.feeds.input.RecipeDisabled' in job.details:
if not minz:
msg = job.details
msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):]
msg = msg.partition(':')[-1]
d = error_dialog(self, _('Recipe Disabled'),
'<p>%s</p>'%msg)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details:
if not minz:
import json
payload = job.details.rpartition(
'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1]
payload = json.loads('{' + payload.partition('{')[-1])
d = {'info':info_dialog, 'warn':warning_dialog,
'error':error_dialog}.get(payload['level'],
error_dialog)
d = d(self, payload['title'],
'<p>%s</p>'%payload['msg'],
det_msg=payload['det_msg'])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
except:
pass
if job.killed:
return
try:
prints(job.details, file=sys.stderr)
except:
pass
if not minz:
self.job_error_dialog.show_error(dialog_title,
_('<b>Failed</b>')+': '+str(job.description),
det_msg=job.details, retry_func=retry_func)
def read_settings(self):
geometry = config['main_window_geometry']
if geometry is not None:
self.restoreGeometry(geometry)
self.read_layout_settings()
def write_settings(self):
with gprefs: # Only write to gprefs once
config.set('main_window_geometry', self.saveGeometry())
dynamic.set('sort_history', self.library_view.model().sort_history)
self.save_layout_state()
self.stack.tb_widget.save_state()
def quit(self, checked=True, restart=False, debug_on_restart=False,
confirm_quit=True):
if self.shutting_down:
return
if confirm_quit and not self.confirm_quit():
return
try:
self.shutdown()
except:
pass
self.restart_after_quit = restart
self.debug_on_restart = debug_on_restart
QApplication.instance().quit()
def donate(self, *args):
open_url(QUrl('https://calibre-ebook.com/donate'))
def confirm_quit(self):
if self.job_manager.has_jobs():
msg = _('There are active jobs. Are you sure you want to quit?')
if self.job_manager.has_device_jobs():
msg = '<p>'+__appname__ + \
_(''' is communicating with the device!<br>
Quitting may cause corruption on the device.<br>
Are you sure you want to quit?''')+'</p>'
if not question_dialog(self, _('Active jobs'), msg):
return False
if self.proceed_question.questions:
msg = _('There are library updates waiting. Are you sure you want to quit?')
if not question_dialog(self, _('Library updates waiting'), msg):
return False
from calibre.db.delete_service import has_jobs
if has_jobs():
msg = _('Some deleted books are still being moved to the Recycle '
'Bin, if you quit now, they will be left behind. Are you '
'sure you want to quit?')
if not question_dialog(self, _('Active jobs'), msg):
return False
return True
def shutdown(self, write_settings=True):
self.shutting_down = True
self.show_shutdown_message()
self.server_change_notification_timer.stop()
from calibre.customize.ui import has_library_closed_plugins
if has_library_closed_plugins():
self.show_shutdown_message(
_('Running database shutdown plugins. This could take a few seconds...'))
self.grid_view.shutdown()
db = None
try:
db = self.library_view.model().db
cf = db.clean
except:
pass
else:
cf()
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata())
db.commit_dirty_cache()
db.prefs.write_serialized(prefs['library_path'])
for action in list(self.iactions.values()):
if not action.shutting_down():
return
if write_settings:
self.write_settings()
self.check_messages_timer.stop()
if hasattr(self, 'update_checker'):
self.update_checker.shutdown()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
self.device_manager.keep_going = False
self.auto_adder.stop()
# Do not report any errors that happen after the shutdown
# We cannot restore the original excepthook as that causes PyQt to
# call abort() on unhandled exceptions
import traceback
def eh(t, v, tb):
try:
traceback.print_exception(t, v, tb, file=sys.stderr)
except:
pass
sys.excepthook = eh
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
self.library_view.model().close()
try:
try:
if self.content_server is not None:
# If the Content server has any sockets being closed then
# this can take quite a long time (minutes). Tell the user that it is
# happening.
self.show_shutdown_message(
_('Shutting down the Content server. This could take a while...'))
s = self.content_server
self.content_server = None
s.exit()
except:
pass
except KeyboardInterrupt:
pass
self.hide_windows()
if self._spare_pool is not None:
self._spare_pool.shutdown()
from calibre.db.delete_service import shutdown
shutdown()
time.sleep(2)
self.istores.join()
return True
def run_wizard(self, *args):
if self.confirm_quit():
self.run_wizard_b4_shutdown = True
self.restart_after_quit = True
try:
self.shutdown(write_settings=False)
except:
pass
QApplication.instance().quit()
def closeEvent(self, e):
if self.shutting_down:
return
self.write_settings()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible():
if not dynamic['systray_msg'] and not isosx:
info_dialog(self, 'calibre', 'calibre '+
_('will keep running in the system tray. To close it, '
'choose <b>Quit</b> in the context menu of the '
'system tray.'), show_copy_button=False).exec_()
dynamic['systray_msg'] = True
self.hide_windows()
e.ignore()
else:
if self.confirm_quit():
try:
self.shutdown(write_settings=False)
except:
import traceback
traceback.print_exc()
e.accept()
else:
e.ignore()
class InventoryModule(BaseInventoryPlugin, Constructable):
NAME = 'azure.azcollection.azure_rm'
def __init__(self):
super(InventoryModule, self).__init__()
self._serializer = Serializer()
self._deserializer = Deserializer()
self._hosts = []
self._filters = None
# FUTURE: use API profiles with defaults
self._compute_api_version = '2017-03-30'
self._network_api_version = '2015-06-15'
self._default_header_parameters = {
'Content-Type': 'application/json; charset=utf-8'
}
self._request_queue = Queue()
self.azure_auth = None
self._batch_fetch = False
def verify_file(self, path):
'''
:param loader: an ansible.parsing.dataloader.DataLoader object
:param path: the path to the inventory config file
:return the contents of the config file
'''
if super(InventoryModule, self).verify_file(path):
if re.match(r'.{0,}azure_rm\.y(a)?ml$', path):
return True
# display.debug("azure_rm inventory filename must end with 'azure_rm.yml' or 'azure_rm.yaml'")
return False
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path)
self._read_config_data(path)
if self.get_option('use_contrib_script_compatible_sanitization'):
self._sanitize_group_name = self._legacy_script_compatible_group_sanitization
self._batch_fetch = self.get_option('batch_fetch')
self._legacy_hostnames = self.get_option('plain_host_names')
self._filters = self.get_option(
'exclude_host_filters') + self.get_option('default_host_filters')
try:
self._credential_setup()
self._get_hosts()
except Exception:
raise
def _credential_setup(self):
auth_options = dict(
auth_source=self.get_option('auth_source'),
profile=self.get_option('profile'),
subscription_id=self.get_option('subscription_id'),
client_id=self.get_option('client_id'),
secret=self.get_option('secret'),
tenant=self.get_option('tenant'),
ad_user=self.get_option('ad_user'),
password=self.get_option('password'),
cloud_environment=self.get_option('cloud_environment'),
cert_validation_mode=self.get_option('cert_validation_mode'),
api_profile=self.get_option('api_profile'),
adfs_authority_url=self.get_option('adfs_authority_url'))
self.azure_auth = AzureRMAuth(**auth_options)
self._clientconfig = AzureRMRestConfiguration(
self.azure_auth.azure_credentials, self.azure_auth.subscription_id,
self.azure_auth._cloud_environment.endpoints.resource_manager)
self._client = ServiceClient(self._clientconfig.credentials,
self._clientconfig)
def _enqueue_get(self, url, api_version, handler, handler_args=None):
if not handler_args:
handler_args = {}
self._request_queue.put_nowait(
UrlAction(url=url,
api_version=api_version,
handler=handler,
handler_args=handler_args))
def _enqueue_vm_list(self, rg='*'):
if not rg or rg == '*':
url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines'
else:
url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines'
url = url.format(subscriptionId=self._clientconfig.subscription_id,
rg=rg)
self._enqueue_get(url=url,
api_version=self._compute_api_version,
handler=self._on_vm_page_response)
def _enqueue_vmss_list(self, rg=None):
if not rg or rg == '*':
url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets'
else:
url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets'
url = url.format(subscriptionId=self._clientconfig.subscription_id,
rg=rg)
self._enqueue_get(url=url,
api_version=self._compute_api_version,
handler=self._on_vmss_page_response)
def _get_hosts(self):
for vm_rg in self.get_option('include_vm_resource_groups'):
self._enqueue_vm_list(vm_rg)
for vmss_rg in self.get_option('include_vmss_resource_groups'):
self._enqueue_vmss_list(vmss_rg)
if self._batch_fetch:
self._process_queue_batch()
else:
self._process_queue_serial()
constructable_config_strict = boolean(
self.get_option('fail_on_template_errors'))
constructable_config_compose = self.get_option('hostvar_expressions')
constructable_config_groups = self.get_option('conditional_groups')
constructable_config_keyed_groups = self.get_option('keyed_groups')
for h in self._hosts:
inventory_hostname = self._get_hostname(h)
if self._filter_host(inventory_hostname, h.hostvars):
continue
self.inventory.add_host(inventory_hostname)
# FUTURE: configurable default IP list? can already do this via hostvar_expressions
self.inventory.set_variable(
inventory_hostname, "ansible_host",
next(
chain(h.hostvars['public_ipv4_addresses'],
h.hostvars['private_ipv4_addresses']), None))
for k, v in iteritems(h.hostvars):
# FUTURE: configurable hostvar prefix? Makes docs harder...
self.inventory.set_variable(inventory_hostname, k, v)
# constructable delegation
self._set_composite_vars(constructable_config_compose,
h.hostvars,
inventory_hostname,
strict=constructable_config_strict)
self._add_host_to_composed_groups(
constructable_config_groups,
h.hostvars,
inventory_hostname,
strict=constructable_config_strict)
self._add_host_to_keyed_groups(constructable_config_keyed_groups,
h.hostvars,
inventory_hostname,
strict=constructable_config_strict)
# FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host
def _filter_host(self, inventory_hostname, hostvars):
self.templar.available_variables = hostvars
for condition in self._filters:
# FUTURE: should warn/fail if conditional doesn't return True or False
conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format(
condition)
try:
if boolean(self.templar.template(conditional)):
return True
except Exception as e:
if boolean(self.get_option('fail_on_template_errors')):
raise AnsibleParserError(
"Error evaluating filter condition '{0}' for host {1}: {2}"
.format(condition, inventory_hostname, to_native(e)))
continue
return False
def _get_hostname(self, host):
# FUTURE: configurable hostname sources
return host.default_inventory_hostname
def _process_queue_serial(self):
try:
while True:
item = self._request_queue.get_nowait()
resp = self.send_request(item.url, item.api_version)
item.handler(resp, **item.handler_args)
except Empty:
pass
def _on_vm_page_response(self, response, vmss=None):
next_link = response.get('nextLink')
if next_link:
self._enqueue_get(url=next_link,
api_version=self._compute_api_version,
handler=self._on_vm_page_response)
if 'value' in response:
for h in response['value']:
# FUTURE: add direct VM filtering by tag here (performance optimization)?
self._hosts.append(
AzureHost(h,
self,
vmss=vmss,
legacy_name=self._legacy_hostnames))
def _on_vmss_page_response(self, response):
next_link = response.get('nextLink')
if next_link:
self._enqueue_get(url=next_link,
api_version=self._compute_api_version,
handler=self._on_vmss_page_response)
# FUTURE: add direct VMSS filtering by tag here (performance optimization)?
for vmss in response['value']:
url = '{0}/virtualMachines'.format(vmss['id'])
# VMSS instances look close enough to regular VMs that we can share the handler impl...
self._enqueue_get(url=url,
api_version=self._compute_api_version,
handler=self._on_vm_page_response,
handler_args=dict(vmss=vmss))
# use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip
#
def _process_queue_batch(self):
while True:
batch_requests = []
batch_item_index = 0
batch_response_handlers = dict()
try:
while batch_item_index < 100:
item = self._request_queue.get_nowait()
name = str(uuid.uuid4())
query_parameters = {'api-version': item.api_version}
req = self._client.get(item.url, query_parameters)
batch_requests.append(
dict(httpMethod="GET", url=req.url, name=name))
batch_response_handlers[name] = item
batch_item_index += 1
except Empty:
pass
if not batch_requests:
break
batch_resp = self._send_batch(batch_requests)
key_name = None
if 'responses' in batch_resp:
key_name = 'responses'
elif 'value' in batch_resp:
key_name = 'value'
else:
raise AnsibleError(
"didn't find expected key responses/value in batch response"
)
for idx, r in enumerate(batch_resp[key_name]):
status_code = r.get('httpStatusCode')
returned_name = r['name']
result = batch_response_handlers[returned_name]
if status_code != 200:
# FUTURE: error-tolerant operation mode (eg, permissions)
raise AnsibleError(
"a batched request failed with status code {0}, url {1}"
.format(status_code, result.url))
# FUTURE: store/handle errors from individual handlers
result.handler(r['content'], **result.handler_args)
def _send_batch(self, batched_requests):
url = '/batch'
query_parameters = {'api-version': '2015-11-01'}
body_obj = dict(requests=batched_requests)
body_content = self._serializer.body(body_obj, 'object')
header = {'x-ms-client-request-id': str(uuid.uuid4())}
header.update(self._default_header_parameters)
request = self._client.post(url, query_parameters)
initial_response = self._client.send(request, header, body_content)
# FUTURE: configurable timeout?
poller = ARMPolling(timeout=2)
poller.initialize(
client=self._client,
initial_response=initial_response,
deserialization_callback=lambda r: self._deserializer('object', r))
poller.run()
return poller.resource()
def send_request(self, url, api_version):
query_parameters = {'api-version': api_version}
req = self._client.get(url, query_parameters)
resp = self._client.send(req,
self._default_header_parameters,
stream=False)
resp.raise_for_status()
content = resp.content
return json.loads(content)
@staticmethod
def _legacy_script_compatible_group_sanitization(name):
# note that while this mirrors what the script used to do, it has many issues with unicode and usability in python
regex = re.compile(r"[^A-Za-z0-9\_\-]")
return regex.sub('_', name)
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you
give it as they come in, but in parallel.
Uses a single "daddy" thread to manage a fleet of child processes.
Communication with the daddy thread happens via two queues: one queue of
jobs waiting to be run (the input queue), and one queue of jobs that are
finished/stopped and need to be returned by getUpdatedBatchJob (the output
queue).
When the batch system is shut down, the daddy thread is stopped.
If running in debug-worker mode, jobs are run immediately as they are sent
to the batch system, in the sending thread, and the daddy thread is not
run. But the queues are still used.
"""
@classmethod
def supportsAutoDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
self.config = config
# Limit to the smaller of the user-imposed limit and what we actually
# have on this machine for each resource.
#
# If we don't have up to the limit of the resource (and the resource
# isn't the inlimited sentinel), warn.
if maxCores > self.numCores:
if maxCores != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough cores! User limited to %i but we only have %i.',
maxCores, self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
if maxMemory != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough memory! User limited to %i bytes but we only have %i bytes.',
maxMemory, self.physicalMemory)
maxMemory = self.physicalMemory
workdir = Toil.getLocalWorkflowDir(
config.workflowID, config.workDir
) # config.workDir may be None; this sets a real directory
self.physicalDisk = toil.physicalDisk(workdir)
if maxDisk > self.physicalDisk:
if maxDisk != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough disk space! User limited to %i bytes but we only have %i bytes.',
maxDisk, self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores,
maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
if config.badWorker > 0 and config.debugWorker:
# We can't throw SIGUSR1 at the worker because it is also going to
# be the leader and/or test harness.
raise RuntimeError(
"Cannot use badWorker and debugWorker together; "
"worker would have to kill the leader")
self.debugWorker = config.debugWorker
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs = {}
"""
:type: dict[str,toil.job.JobDescription]
"""
# A queue of jobs waiting to be executed. Consumed by the daddy thread.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the daddy thread.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs = {}
"""
:type: dict[str,Info]
"""
# These next two are only used outside debug-worker mode
# A dict mapping PIDs to Popen objects for running jobs.
# Jobs that don't fork are executed one at a time in the main thread.
self.children = {}
"""
:type: dict[int,subprocess.Popen]
"""
# A dict mapping child PIDs to the Job IDs they are supposed to be running.
self.childToJob = {}
"""
:type: dict[int,str]
"""
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(int(self.maxCores / self.minCores),
'cores')
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory')
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk')
# If we can't schedule something, we fill this in with a reason why
self.schedulingStatusMessage = None
# We use this event to signal shutdown
self.shuttingDown = Event()
# A thread in charge of managing all our child processes.
# Also takes care of resource accounting.
self.daddyThread = None
# If it breaks it will fill this in
self.daddyException = None
if self.debugWorker:
log.debug('Started in worker debug mode.')
else:
self.daddyThread = Thread(target=self.daddy, daemon=True)
self.daddyThread.start()
log.debug('Started in normal mode.')
def daddy(self):
"""
Be the "daddy" thread.
Our job is to look at jobs from the input queue.
If a job fits in the available resources, we allocate resources for it
and kick off a child process.
We also check on our children.
When a child finishes, we reap it, release its resources, and put its
information in the output queue.
"""
try:
log.debug('Started daddy thread.')
while not self.shuttingDown.is_set():
# Main loop
while not self.shuttingDown.is_set():
# Try to start as many jobs as we can try to start
try:
# Grab something from the input queue if available.
args = self.inputQueue.get_nowait()
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
coreFractions = int(jobCores / self.minCores)
# Try to start the child
result = self._startChild(jobCommand, jobID,
coreFractions, jobMemory,
jobDisk, environment)
if result is None:
# We did not get the resources to run this job.
# Requeue last, so we can look at the next job.
# TODO: Have some kind of condition the job can wait on,
# but without threads (queues for jobs needing
# cores/memory/disk individually)?
self.inputQueue.put(args)
break
# Otherwise it's a PID if it succeeded, or False if it couldn't
# start. But we don't care either way here.
except Empty:
# Nothing to run. Stop looking in the queue.
break
# Now check on our children.
for done_pid in self._pollForDoneChildrenIn(self.children):
# A child has actually finished.
# Clean up after it.
self._handleChild(done_pid)
# Then loop again: start and collect more jobs.
# TODO: It would be good to be able to wait on a new job or a finished child, whichever comes first.
# For now we just sleep and loop.
time.sleep(0.01)
# When we get here, we are shutting down.
for popen in self.children.values():
# Kill all the children, going through popen to avoid signaling re-used PIDs.
popen.kill()
for popen in self.children.values():
# Reap all the children
popen.wait()
# Then exit the thread.
return
except Exception as e:
log.critical('Unhandled exception in daddy thread: %s',
traceback.format_exc())
# Pass the exception back to the main thread so it can stop the next person who calls into us.
self.daddyException = e
raise
def _checkOnDaddy(self):
if self.daddyException is not None:
# The daddy thread broke and we cannot do our job
log.critical(
'Propagating unhandled exception in daddy thread to main thread'
)
exc = self.daddyException
self.daddyException = None
raise exc
def _pollForDoneChildrenIn(self, pid_to_popen):
"""
See if any children represented in the given dict from PID to Popen
object have finished.
Return a collection of their PIDs.
Guarantees that each child's exit code will be gettable via wait() on
the child's Popen object (i.e. does not reap the child, unless via
Popen).
"""
# We keep our found PIDs in a set so we can work around waitid showing
# us the same one repeatedly.
ready = set()
# Find the waitid function
waitid = getattr(os, 'waitid', None)
if callable(waitid):
# waitid exists (not Mac)
while True:
# Poll for any child to have exit, but don't reap it. Leave reaping
# to the Popen.
# TODO: What if someone else in Toil wants to do this syscall?
# TODO: Is this one-notification-per-done-child with WNOHANG? Or
# can we miss some? Or do we see the same one repeatedly until it
# is reaped?
try:
siginfo = waitid(os.P_ALL, -1,
os.WEXITED | os.WNOWAIT | os.WNOHANG)
except ChildProcessError:
# This happens when there is nothing to wait on right now,
# instead of the weird C behavior of overwriting a field in
# a pointed-to struct.
siginfo = None
if siginfo is not None and siginfo.si_pid in pid_to_popen and siginfo.si_pid not in ready:
# Something new finished
ready.add(siginfo.si_pid)
else:
# Nothing we own that we haven't seen before has finished.
return ready
else:
# On Mac there's no waitid and no way to wait and not reap.
# Fall back on polling all the Popen objects.
# To make this vaguely efficient we have to return done children in
# batches.
for pid, popen in pid_to_popen.items():
if popen.poll() is not None:
# Process is done
ready.add(pid)
log.debug('Child %d has stopped', pid)
# Return all the done processes we found
return ready
def _runDebugJob(self, jobCommand, jobID, environment):
"""
Run the jobCommand right now, in the current thread.
May only be called in debug-worker mode.
Assumes resources are available.
"""
assert self.debugWorker
# TODO: It is not possible to kill running jobs in forkless mode,
# because they are run immediately in the main thread.
info = Info(time.time(), None, None, killIntended=False)
self.runningJobs[jobID] = info
if jobCommand.startswith("_toil_worker "):
# We can actually run in this thread
jobName, jobStoreLocator, jobStoreID = jobCommand.split()[
1:] # Parse command
jobStore = Toil.resumeJobStore(jobStoreLocator)
toil_worker.workerScript(
jobStore,
jobStore.config,
jobName,
jobStoreID,
redirectOutputToLogFile=not self.debugWorker
) # Call the worker
else:
# Run synchronously. If starting or running the command fails, let the exception stop us.
subprocess.check_call(jobCommand,
shell=True,
env=dict(os.environ, **environment))
self.runningJobs.pop(jobID)
if not info.killIntended:
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=0,
wallTime=time.time() - info.time,
exitReason=None))
def getSchedulingStatusMessage(self):
# Implement the abstractBatchSystem's scheduling status message API
return self.schedulingStatusMessage
def _setSchedulingStatusMessage(self, message):
"""
If we can't run a job, we record a short message about why not. If the
leader wants to know what is up with us (for example, to diagnose a
deadlock), it can ask us for the message.
"""
self.schedulingStatusMessage = message
def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk,
environment):
"""
Start a child process for the given job.
Allocate its required resources and save it and save it in our bookkeeping structures.
If the job is started, returns its PID.
If the job fails to start, reports it as failed and returns False.
If the job cannot get the resources it needs to start, returns None.
"""
# We fill this in if we manage to actually start the child.
popen = None
# This is when we started working on the job.
startTime = time.time()
# See if we can fit the job in our resource pools right now.
if self.coreFractions.acquireNow(coreFractions):
# We got some cores
if self.memory.acquireNow(jobMemory):
# We got some memory
if self.disk.acquireNow(jobDisk):
# We got the final resource, disk.
# Actually run the job.
# When it finishes we will release what it was using.
# So it is important to not lose track of the child process.
try:
# Launch the job
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(
os.environ,
**environment))
except Exception:
# If the job can't start, make sure we release resources now
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.error('Could not start job %s: %s', jobID,
traceback.format_exc())
# Report as failed.
self.outputQueue.put(
UpdatedBatchJobInfo(
jobID=jobID,
exitStatus=EXIT_STATUS_UNAVAILABLE_VALUE,
wallTime=0,
exitReason=None))
# Free resources
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
# Complain it broke.
return False
else:
# If the job did start, record it
self.children[popen.pid] = popen
# Make sure we can look it up by PID later
self.childToJob[popen.pid] = jobID
# Record that the job is running, and the resources it is using
info = Info(startTime,
popen, (coreFractions, jobMemory, jobDisk),
killIntended=False)
self.runningJobs[jobID] = info
log.debug('Launched job %s as child %d', jobID,
popen.pid)
# Report success starting the job
# Note that if a PID were somehow 0 it would look like False
assert popen.pid != 0
return popen.pid
else:
# We can't get disk, so free cores and memory
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self._setSchedulingStatusMessage(
'Not enough disk to run job %s' % jobID)
else:
# Free cores, since we can't get memory
self.coreFractions.release(coreFractions)
self._setSchedulingStatusMessage(
'Not enough memory to run job %s' % jobID)
else:
self._setSchedulingStatusMessage('Not enough cores to run job %s' %
jobID)
# If we get here, we didn't succeed or fail starting the job.
# We didn't manage to get the resources.
# Report that.
return None
def _handleChild(self, pid):
"""
Handle a child process PID that has finished.
The PID must be for a child job we started.
Not thread safe to run at the same time as we are making more children.
Remove the child from our bookkeeping structures and free its resources.
"""
# Look up the child
popen = self.children[pid]
jobID = self.childToJob[pid]
info = self.runningJobs[jobID]
# Unpack the job resources
(coreFractions, jobMemory, jobDisk) = info.resources
# Clean up our records of the job.
self.runningJobs.pop(jobID)
self.childToJob.pop(pid)
self.children.pop(pid)
# See how the child did, and reap it.
statusCode = popen.wait()
if statusCode != 0 and not info.killIntended:
log.error("Got exit code %i (indicating failure) "
"from job %s.", statusCode, self.jobs[jobID])
if not info.killIntended:
# Report if the job failed and we didn't kill it.
# If we killed it then it shouldn't show up in the queue.
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=statusCode,
wallTime=time.time() - info.time,
exitReason=None))
# Free up the job's resources.
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.debug('Child %d for job %s succeeded', pid, jobID)
def issueBatchJob(self, jobDesc):
"""Adds the command and resources to a queue to be run."""
self._checkOnDaddy()
# Round cores to minCores and apply scale.
# Make sure to give minCores even if asked for 0 cores, or negative or something.
cores = max(
math.ceil(jobDesc.cores * self.scale / self.minCores) *
self.minCores, self.minCores)
# Don't do our own assertions about job size vs. our configured size.
# The abstract batch system can handle it.
self.checkResourceRequest(jobDesc.memory,
cores,
jobDesc.disk,
job_name=jobDesc.jobName,
detail=f'Scale is set to {self.scale}.')
log.debug(
f"Issuing the command: {jobDesc.command} with "
f"memory: {jobDesc.memory}, cores: {cores}, disk: {jobDesc.disk}")
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobDesc.command
if self.debugWorker:
# Run immediately, blocking for return.
# Ignore resource requirements; we run one job at a time
self._runDebugJob(jobDesc.command, jobID, self.environment.copy())
else:
# Queue the job for later
self.inputQueue.put(
(jobDesc.command, jobID, cores, jobDesc.memory, jobDesc.disk,
self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs):
"""Kills jobs by ID."""
self._checkOnDaddy()
log.debug('Killing jobs: {}'.format(jobIDs))
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
if info.popen is not None:
log.debug('Send kill to PID %s', info.popen.pid)
info.popen.kill()
log.debug('Sent kill to PID %s', info.popen.pid)
else:
# No popen if running in forkless mode currently
assert self.debugWorker
log.critical("Can't kill job: %s in debug mode" % jobID)
while jobID in self.runningJobs:
pass
def getIssuedBatchJobIDs(self):
"""Just returns all the jobs that have been run, but not yet returned as updated."""
self._checkOnDaddy()
return list(self.jobs.keys())
def getRunningBatchJobIDs(self):
self._checkOnDaddy()
now = time.time()
return {
jobID: now - info.time
for jobID, info in list(self.runningJobs.items())
}
def shutdown(self):
"""
Cleanly terminate and join daddy thread.
"""
if self.daddyThread is not None:
# Tell the daddy thread to stop.
self.shuttingDown.set()
# Wait for it to stop.
self.daddyThread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""Returns a tuple of a no-longer-running job, the return value of its process, and its runtime, or None."""
self._checkOnDaddy()
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
self.jobs.pop(item.jobID)
log.debug("Ran jobID: %s with exit value: %i", item.jobID,
item.exitStatus)
return item
@classmethod
def setOptions(cls, setOption):
setOption("scale", default=1)
class _PrefetchingIter(object):
def __init__(self, dataloader, dataloader_it, num_threads=None):
self.queue = Queue(1)
self.dataloader_it = dataloader_it
self.dataloader = dataloader
self.num_threads = num_threads
self.use_thread = dataloader.use_prefetch_thread
self.use_alternate_streams = dataloader.use_alternate_streams
self.device = self.dataloader.device
if self.use_alternate_streams and self.device.type == 'cuda':
self.stream = torch.cuda.Stream(device=self.device)
else:
self.stream = None
self._shutting_down = False
if self.use_thread:
self._done_event = threading.Event()
thread = threading.Thread(
target=_prefetcher_entry,
args=(dataloader_it, dataloader, self.queue, num_threads,
self.stream, self._done_event),
daemon=True)
thread.start()
self.thread = thread
def __iter__(self):
return self
def _shutdown(self):
# Sometimes when Python is exiting complicated operations like
# self.queue.get_nowait() will hang. So we set it to no-op and let Python handle
# the rest since the thread is daemonic.
# PyTorch takes the same solution.
if PYTHON_EXIT_STATUS is True or PYTHON_EXIT_STATUS is None:
return
if not self._shutting_down:
try:
self._shutting_down = True
self._done_event.set()
try:
self.queue.get_nowait() # In case the thread is blocking on put().
except: # pylint: disable=bare-except
pass
self.thread.join()
except: # pylint: disable=bare-except
pass
def __del__(self):
if self.use_thread:
self._shutdown()
def _next_non_threaded(self):
batch = next(self.dataloader_it)
batch = recursive_apply(batch, restore_parent_storage_columns, self.dataloader.graph)
batch, feats, stream_event = _prefetch(batch, self.dataloader, self.stream)
return batch, feats, stream_event
def _next_threaded(self):
try:
batch, feats, stream_event, exception = self.queue.get(timeout=prefetcher_timeout)
except Empty:
raise RuntimeError(
f'Prefetcher thread timed out at {prefetcher_timeout} seconds.')
if batch is None:
self.thread.join()
if exception is None:
raise StopIteration
exception.reraise()
return batch, feats, stream_event
def __next__(self):
batch, feats, stream_event = \
self._next_non_threaded() if not self.use_thread else self._next_threaded()
batch = recursive_apply_pair(batch, feats, _assign_for)
if stream_event is not None:
stream_event.wait()
return batch
async def get_item(self, queue: Queue):
return queue.get_nowait()
def search_overseer_thread(args, method, new_location_queue, pause_bit,
encryption_lib_path, db_updates_queue, wh_queue):
log.info('Search overseer starting')
search_items_queue = Queue()
threadStatus = {}
threadStatus['Overseer'] = {
'message': 'Initializing',
'type': 'Overseer',
'method': 'Hex Grid' if method == 'hex' else 'Spawn Point'
}
if (args.print_status):
log.info('Starting status printer thread')
t = Thread(target=status_printer,
name='status_printer',
args=(threadStatus, search_items_queue, db_updates_queue,
wh_queue))
t.daemon = True
t.start()
# Create a search_worker_thread per account
log.info('Starting search worker threads')
for i, account in enumerate(args.accounts):
log.debug('Starting search worker thread %d for user %s', i,
account['username'])
workerId = 'Worker {:03}'.format(i)
threadStatus[workerId] = {
'type': 'Worker',
'message': 'Creating thread...',
'success': 0,
'fail': 0,
'noitems': 0,
'skip': 0,
'user': account['username']
}
t = Thread(target=search_worker_thread,
name='search-worker-{}'.format(i),
args=(args, account, search_items_queue, pause_bit,
encryption_lib_path, threadStatus[workerId],
db_updates_queue, wh_queue))
t.daemon = True
t.start()
'''
For hex scanning, we can generate the full list of scan points well
in advance. When then can queue them all up to be searched as fast
as the threads will allow.
With spawn point scanning (sps) we can come up with the order early
on, and we can populate the entire queue, but the individual threads
will need to wait until the point is available (and ensure it is not
to late as well).
'''
# A place to track the current location
current_location = False
# Used to tell SPS to scan for all CURRENT pokemon instead
# of, like during a normal loop, just finding the next one
# which will appear (since you've already scanned existing
# locations in the prior loop)
# Needed in a first loop and pausing/changing location.
sps_scan_current = True
# The real work starts here but will halt on pause_bit.set()
while True:
# paused; clear queue if needed, otherwise sleep and loop
while pause_bit.is_set():
if not search_items_queue.empty():
try:
while True:
search_items_queue.get_nowait()
except Empty:
pass
threadStatus['Overseer']['message'] = 'Scanning is paused'
sps_scan_current = True
time.sleep(1)
# If a new location has been passed to us, get the most recent one
if not new_location_queue.empty():
log.info('New location caught, moving search grid')
sps_scan_current = True
try:
while True:
current_location = new_location_queue.get_nowait()
except Empty:
pass
# We (may) need to clear the search_items_queue
if not search_items_queue.empty():
try:
while True:
search_items_queue.get_nowait()
except Empty:
pass
# If there are no search_items_queue either the loop has finished (or been
# cleared above) -- either way, time to fill it back up
if search_items_queue.empty():
log.debug('Search queue empty, restarting loop')
# locations = [((lat, lng, alt), ts_appears, ts_leaves),...]
if method == 'hex':
locations = get_hex_location_list(args, current_location)
else:
locations = get_sps_location_list(args, current_location,
sps_scan_current)
sps_scan_current = False
if len(locations) == 0:
log.warning('Nothing to scan!')
threadStatus['Overseer']['message'] = 'Queuing steps'
for step, step_location in enumerate(locations, 1):
log.debug('Queueing step %d @ %f/%f/%f', step,
step_location[0][0], step_location[0][1],
step_location[0][2])
search_args = (step, step_location[0], step_location[1],
step_location[2])
search_items_queue.put(search_args)
else:
nextitem = search_items_queue.queue[0]
threadStatus['Overseer'][
'message'] = 'Processing search queue, next item is {:6f},{:6f}'.format(
nextitem[1][0], nextitem[1][1])
# If times are specified, print the time of the next queue item, and how many seconds ahead/behind realtime
if nextitem[2]:
threadStatus['Overseer']['message'] += ' @ {}'.format(
time.strftime('%H:%M:%S', time.localtime(nextitem[2])))
if nextitem[2] > now():
threadStatus['Overseer'][
'message'] += ' ({}s ahead)'.format(nextitem[2] -
now())
else:
threadStatus['Overseer'][
'message'] += ' ({}s behind)'.format(now() -
nextitem[2])
# Now we just give a little pause here
time.sleep(1)
class FFMPEGNVR(Thread):
nvr_list: List[object] = []
def __init__(self,
config,
detector,
detector_queue,
post_processors,
mqtt_queue=None):
Thread.__init__(self)
self.setup_loggers(config)
self._logger.debug("Initializing NVR thread")
# Use FFMPEG to read from camera. Used for reading/recording
self.camera = FFMPEGCamera(config)
self._mqtt = MQTT(config, mqtt_queue)
self.config = config
self.kill_received = False
self.camera_grabber = None
self._objects_in_fov = []
self._labels_in_fov = []
self._reported_label_count = {}
self.idle_frames = 0
self._motion_frames = 0
self._motion_detected = False
self._motion_only_frames = 0
self._motion_max_timeout_reached = False
self.detector = detector
self._post_processors = post_processors
self._object_decoder_queue = Queue(maxsize=2)
self._motion_decoder_queue = Queue(maxsize=2)
motion_queue = Queue(maxsize=2)
self.object_return_queue = Queue(maxsize=2)
self.motion_return_queue = Queue(maxsize=2)
if config.motion_detection.trigger_detector:
self.camera.scan_for_motion.set()
self.camera.scan_for_objects.clear()
else:
self.camera.scan_for_objects.set()
self.camera.scan_for_motion.clear()
self._object_filters = {}
for object_filter in config.object_detection.labels:
self._object_filters[object_filter.label] = Filter(object_filter)
self._zones = []
for zone in config.camera.zones:
self._zones.append(
Zone(
zone,
self.camera.resolution,
config,
self._mqtt.mqtt_queue,
post_processors,
))
# Motion detector class.
if config.motion_detection.timeout or config.motion_detection.trigger_detector:
self.motion_detector = MotionDetection(config,
self.camera.resolution)
self.motion_thread = Thread(
target=self.motion_detector.motion_detection,
args=(motion_queue, ))
self.motion_thread.daemon = True
self.motion_thread.start()
self.motion_decoder = Thread(
target=self.camera.decoder,
args=(
self._motion_decoder_queue,
motion_queue,
config.motion_detection.width,
config.motion_detection.height,
),
)
self.motion_decoder.daemon = True
self.motion_decoder.start()
self.object_decoder = Thread(
target=self.camera.decoder,
args=(
self._object_decoder_queue,
detector_queue,
detector.model_width,
detector.model_height,
),
)
self.object_decoder.daemon = True
self.object_decoder.start()
self.start_camera()
# Initialize recorder
self._trigger_recorder = False
self._start_recorder = False
self.recorder = FFMPEGRecorder(config, self.detector.detection_lock,
mqtt_queue)
self.nvr_list.append({config.camera.mqtt_name: self})
self._logger.debug("NVR thread initialized")
def setup_loggers(self, config):
self._logger = logging.getLogger(__name__ + "." +
config.camera.name_slug)
if getattr(config.camera.logging, "level", None):
self._logger.setLevel(config.camera.logging.level)
self._motion_logger = logging.getLogger(__name__ + "." +
config.camera.name_slug +
".motion")
if getattr(config.motion_detection.logging, "level", None):
self._motion_logger.setLevel(config.motion_detection.logging.level)
elif getattr(config.camera.logging, "level", None):
self._motion_logger.setLevel(config.camera.logging.level)
self._object_logger = logging.getLogger(__name__ + "." +
config.camera.name_slug +
".object")
if getattr(config.object_detection.logging, "level", None):
self._object_logger.setLevel(config.object_detection.logging.level)
elif getattr(config.camera.logging, "level", None):
self._object_logger.setLevel(config.camera.logging.level)
def on_connect(self, client):
"""Called when MQTT connection is established"""
subscriptions = self._mqtt.on_connect(client)
self.recorder.on_connect(client)
for zone in self._zones:
zone.on_connect(client)
# We subscribe to the switch topic to toggle camera on/off
subscriptions[self._mqtt.devices["switch"].command_topic].append(
self.toggle_camera)
return subscriptions
def toggle_camera(self, message):
if message.payload.decode() == "ON":
self.start_camera()
return
if message.payload.decode() == "OFF":
self.stop_camera()
return
def start_camera(self):
if not self.camera_grabber or not self.camera_grabber.is_alive():
self._logger.debug("Starting camera")
self.camera_grabber = Thread(
target=self.camera.capture_pipe,
args=(
self.config.object_detection.interval,
self._object_decoder_queue,
self.object_return_queue,
self.config.motion_detection.interval,
self._motion_decoder_queue,
self.motion_return_queue,
),
)
self.camera_grabber.daemon = True
self.camera_grabber.start()
def stop_camera(self):
self._logger.debug("Stopping camera")
self.camera.release()
self.camera_grabber.join()
if self.recorder.is_recording:
self.recorder.stop_recording()
def event_over(self):
if self._trigger_recorder or any(zone.trigger_recorder
for zone in self._zones):
self._motion_max_timeout_reached = False
self._motion_only_frames = 0
return False
if self.config.motion_detection.timeout and self.motion_detected:
# Only allow motion to keep event active for a specified period of time
if self._motion_only_frames >= (
self.camera.stream.fps *
self.config.motion_detection.max_timeout):
if not self._motion_max_timeout_reached:
self._motion_max_timeout_reached = True
self._logger.debug(
"Motion has stalled recorder for longer than max_timeout, "
"event considered over anyway")
return True
self._motion_only_frames += 1
return False
return True
def start_recording(self, frame):
recorder_thread = Thread(
target=self.recorder.start_recording,
args=(frame, self.objects_in_fov, self.camera.resolution),
)
recorder_thread.start()
if (self.config.motion_detection.timeout
and not self.camera.scan_for_motion.is_set()):
self.camera.scan_for_motion.set()
self._logger.info("Starting motion detector")
def stop_recording(self):
if self.idle_frames % self.camera.stream.fps == 0:
self._logger.info("Stopping recording in: {}".format(
int(self.config.recorder.timeout -
(self.idle_frames / self.camera.stream.fps))))
if self.idle_frames >= (self.camera.stream.fps *
self.config.recorder.timeout):
if not self.config.motion_detection.trigger_detector:
self.camera.scan_for_motion.clear()
self._logger.info("Pausing motion detector")
self.recorder.stop_recording()
def get_processed_object_frame(self):
""" Returns a frame along with its detections which has been processed
by the object detector """
try:
return self.object_return_queue.get_nowait()["frame"]
except Empty:
return None
def filter_fov(self, frame):
objects_in_fov = []
labels_in_fov = []
self._trigger_recorder = False
for obj in frame.objects:
if self._object_filters.get(obj.label) and self._object_filters[
obj.label].filter_object(obj):
obj.relevant = True
objects_in_fov.append(obj)
labels_in_fov.append(obj.label)
if self._object_filters[obj.label].triggers_recording:
self._trigger_recorder = True
# Send detection to configured post processors
if self._object_filters[obj.label].post_processor:
send_to_post_processor(
self._logger,
self.config,
self._post_processors,
self._object_filters[obj.label].post_processor,
frame,
obj,
)
self.objects_in_fov = objects_in_fov
self.labels_in_fov = labels_in_fov
@property
def objects_in_fov(self):
return self._objects_in_fov
@objects_in_fov.setter
def objects_in_fov(self, objects):
if objects == self._objects_in_fov:
return
if self._mqtt.mqtt_queue:
attributes = {}
attributes["objects"] = [obj.formatted for obj in objects]
self._mqtt.devices["object_detected"].publish(
bool(objects), attributes)
self._objects_in_fov = objects
@property
def labels_in_fov(self):
return self._labels_in_fov
@labels_in_fov.setter
def labels_in_fov(self, labels):
self._labels_in_fov, self._reported_label_count = report_labels(
labels,
self._labels_in_fov,
self._reported_label_count,
self._mqtt.mqtt_queue,
self._mqtt.devices,
)
def filter_zones(self, frame):
for zone in self._zones:
zone.filter_zone(frame)
def get_processed_motion_frame(self):
""" Returns a frame along with its motion contours which has been processed
by the motion detector """
try:
return self.motion_return_queue.get_nowait()["frame"]
except Empty:
return None
def filter_motion(self, motion_contours):
_motion_found = bool(
motion_contours.max_area > self.config.motion_detection.area)
if _motion_found:
self._motion_frames += 1
self._motion_logger.debug("Consecutive frames with motion: {}, "
"max area size: {}".format(
self._motion_frames,
motion_contours.max_area))
if self._motion_frames >= self.config.motion_detection.frames:
if not self.motion_detected:
self.motion_detected = True
return
else:
self._motion_frames = 0
if self.motion_detected:
self.motion_detected = False
@property
def motion_detected(self):
return self._motion_detected
@motion_detected.setter
def motion_detected(self, motion_detected):
self._motion_detected = motion_detected
self._motion_logger.debug(
"Motion detected" if motion_detected else "Motion stopped")
if self._mqtt.mqtt_queue:
self._mqtt.devices["motion_detected"].publish(motion_detected)
def process_object_event(self):
if self._trigger_recorder or any(zone.trigger_recorder
for zone in self._zones):
if not self.recorder.is_recording:
self._start_recorder = True
def process_motion_event(self):
if self.motion_detected:
if (self.config.motion_detection.trigger_detector
and not self.camera.scan_for_objects.is_set()):
self.camera.scan_for_objects.set()
self._logger.debug("Starting object detector")
elif (self.camera.scan_for_objects.is_set()
and not self.recorder.is_recording
and self.config.motion_detection.trigger_detector):
self._logger.debug("Not recording, pausing object detector")
self.camera.scan_for_objects.clear()
def update_status_sensor(self):
status = "unknown"
if self.recorder.is_recording:
status = "recording"
elif self.camera.scan_for_objects.is_set():
status = "scanning_for_objects"
elif self.camera.scan_for_motion.is_set():
status = "scanning_for_motion"
attributes = {}
attributes["last_recording_start"] = self.recorder.last_recording_start
attributes["last_recording_end"] = self.recorder.last_recording_end
if (status != self._mqtt.status_state
or attributes != self._mqtt.status_attributes):
self._mqtt.status_attributes = attributes
self._mqtt.status_state = status
def run(self):
""" Main thread. It handles starting/stopping of recordings and
publishes to MQTT if object is detected. Speed is determined by FPS"""
self._logger.debug("Waiting for first frame")
self.camera.frame_ready.wait()
self._logger.debug("First frame received")
self.idle_frames = 0
while not self.kill_received:
self.update_status_sensor()
self.camera.frame_ready.wait()
# Filter returned objects
processed_object_frame = self.get_processed_object_frame()
if processed_object_frame:
# Filter objects in the FoV
self.filter_fov(processed_object_frame)
# Filter objects in each zone
self.filter_zones(processed_object_frame)
if self._object_logger.level == LOG_LEVELS["DEBUG"]:
if self.config.object_detection.log_all_objects:
objs = [
obj.formatted
for obj in processed_object_frame.objects
]
self._object_logger.debug(f"All objects: {objs}")
else:
objs = [obj.formatted for obj in self.objects_in_fov]
self._object_logger.debug(f"Objects: {objs}")
# Filter returned motion contours
processed_motion_frame = self.get_processed_motion_frame()
if processed_motion_frame:
# self._logger.debug(processed_motion_frame.motion_contours)
self.filter_motion(processed_motion_frame.motion_contours)
self.process_object_event()
self.process_motion_event()
if (processed_object_frame or processed_motion_frame
) and self.config.camera.publish_image:
self._mqtt.publish_image(
processed_object_frame,
processed_motion_frame,
self._zones,
self.camera.resolution,
)
# If we are recording and no object is detected
if self._start_recorder:
self._start_recorder = False
self.start_recording(processed_object_frame)
elif self.recorder.is_recording and self.event_over():
self.idle_frames += 1
self.stop_recording()
continue
self.idle_frames = 0
self._logger.info("Exiting NVR thread")
def stop(self):
self._logger.info("Stopping NVR thread")
self.kill_received = True
# Stop frame grabber
self.camera.release()
self.camera_grabber.join()
# Stop potential recording
if self.recorder.is_recording:
self.recorder.stop_recording()
