class ThreadPool(object):
"""
Nonblock Pool of threads consuming tasks from a queue
"""
def __init__(self, numThreads):
self.numThreads = numThreads
self.workers = []
self.tasks = Queue(numThreads)
for _ in range(numThreads):
self.workers.append(Worker(self.tasks))
def addTask(self, func, *args, **kargs):
"""
Add a task to the queue
if Queue is full already when adding task, return false
"""
try:
self.tasks.put_nowait((func, args, kargs))
return True
except Full:
LOG.error("Thread pool of size %s is full" % self.numThreads)
return False
def getWorkerRunningTime(self):
""" get info for all threads """
runningTime = []
for _, worker in enumerate(self.workers):
runningTime.append("%.2fms" % (1000 * worker.getRunningTime()))
return runningTime
def waitCompletion(self):
"""Wait for completion of all the tasks in the queue"""
self.tasks.join()
def main():
f = init_bloom_filter()
print "bloom filter's count:%s" % f.count
user_accessed_set = init_user_access()
print "user accessed set:%s" % user_accessed_set
from zhihu_thread import MyThread
threads = []
queue = Queue()
url = USER_URL.format("jie-28")
user_seeds = User(url).generate_user_seeds(int(math.ceil(THREAD_COUNT)), user_accessed_set)
for user_seed in user_seeds:
queue.put_nowait(user_seed)
print "Start, user seeds:%s " % user_seeds
import threading
lock = threading.Lock()
bf_lock = threading.Lock()
for i in range(THREAD_COUNT):
t = MyThread(consume, (lock, bf_lock, f, user_accessed_set, queue, i, GRAPH_DEEP_LEVEL),
consume.__name__)
threads.append(t)
for t in threads:
t.start()
for t in threads:
t.join()
print "End, queue's size:%s" % queue.qsize()
print "All Done"
def bfs(self, src):
self.colors = []
self.distances = []
self.predecessors = []
for u in xrange(self.graphlen):
self.colors.append(WHITE)
self.distances.append(0)
self.predecessors.append(None)
self.colors[src] = GRAY
Q = Queue()
Q.put_nowait(src)
while not Q.empty():
u = Q.get_nowait()
if not (u in self.graph):
continue
for v in self.graph[u]:
if self.colors[v] == WHITE:
self.colors[v] = GRAY
self.distances[v] = self.distances[u]+1
self.predecessors[v] = u
Q.put_nowait(v)
self.colors[u] = BLACK
class Carrier(object):
""" スレッド間のデータ受け渡しとイベント通知を行う """
def __init__(self, name):
self.name = name
self.queue = Queue()
self.event = Event()
def handover(self, packet):
""" データをキューに入れる. 配送先のイベントを起こす """
self.queue.put_nowait(packet)
self.event.set()
self.event.clear()
def pickup(self):
""" 配達したデータを受け取ってもらう """
return self.queue.get_nowait()
def wake(self):
""" イベントを起こす """
self.event.set()
def clear(self):
""" キューを空っぽにする """
while not self.empty():
self.pickup()
def empty(self):
""" キューが空かどうかを返す """
return self.queue.empty()
def sleep(self):
""" イベントを眠らせる """
self.event.wait()
def main(argv):
global input_file
global tenant
global number
try:
opts, args = getopt.getopt(argv, "hn:i:t:", ["help", "number=", "input=", "tenant="])
except getopt.GetoptError:
usage()
sys.exit(2)
if len(opts) == 0:
logger_main.info("Will use default parameter: python {file_name} -i {input_file} -t {tenant} -n {number}".format(
file_name=sys.argv[0], input_file=input_file, tenant=tenant, number=number))
for opt, arg in opts:
if opt in ("-n", "--number"):
number = int(arg)
elif opt in ("-i", "--input"):
input_file = arg
elif opt in ("-t", "--tenant"):
tenant = arg
elif opt in ("-h", "--help"):
usage()
exit()
else:
usage()
exit(1)
f = open(input_file)
lines = f.readlines()
option_set = ';'.join(lines).replace('\n', '').split(';')
from threading import Thread
from Queue import Queue
#initialize a queue
sql_queue = Queue()
#initialize a thread to process DB insertion
db_thread = Thread(target=do_insertion, args=(sql_queue,))
db_thread.start()
i = 0
sql_segment_buffer = ''
while i < number:
i += 1
one_row = generate_one_row(option_set, tenant)
if i % 1000 == 0:
sql_segment_buffer += one_row + ';'
sql_queue.put_nowait(sql_segment_buffer)
sql_segment_buffer = ''
logger_main.info("Generate the {index}th insert SQL".format(index=i))
else:
sql_segment_buffer += one_row + ','
logger_main.info("Finish SQL generation")
sql_queue.put_nowait(POISON)
db_thread.join()
class HttpPool(object):
def __init__(
self,
threads_count,
fail_op,
log,
):
self._tasks = Queue()
self._results = Queue()
for i in xrange(threads_count):
thread.start_new_thread(get_remote_data, (self._tasks,
self._results, fail_op, log))
def add_task(self, tid, url):
task = {'id': tid, 'url': url}
try:
self._tasks.put_nowait(task)
except Full:
return False
return True
def get_results(self):
results = []
while True:
try:
res = self._results.get_nowait()
except Empty:
break
results.append(res)
return results
class MongoDBSubscriber:
def __init__(self, collection, topics, timestamp):
self.messages = Queue()
self.collection = collection
self.topics = list(topics)
self.timestamp = timestamp
def getNext(self):
self.cursor = self.collection.find({'topic': {'$in': self.topics}, 'timestamp': {'$gte': self.timestamp}})
for data in self.cursor:
self.messages.put_nowait(data['message'])
self.timestamp = datetime.now()
if self.messages.qsize() == 0:
return None
else:
return self.messages.get(block=False, timeout=None)
def getAll(self):
self.cursor = self.collection.find({'topic': {'$in': self.topics}, 'timestamp': {'$gte': self.timestamp}})
for data in self.cursor:
self.messages.put_nowait(data['message'])
self.timestamp = datetime.now()
items = []
maxItemsToRetreive = self.messages.qsize()
for numOfItemsRetrieved in range(0, maxItemsToRetreive):
try:
if numOfItemsRetrieved == maxItemsToRetreive:
break
items.append(self.messages.get_nowait())
except Empty, e:
break
return items
class Publisher(object):
def __init__(self, port):
super(Publisher, self).__init__()
self.address = 'tcp://*:%s' % port
self.context = zmq.Context()
self.messages = Queue()
def publish(self, command, *args):
message = {
'command': command,
'args': args
}
self.messages.put_nowait(message)
def _deliver(self):
self.socket = self.context.socket(zmq.PUB)
self.socket.bind(self.address)
time.sleep(0.5)
while True:
while not self.messages.empty():
self.socket.send_pyobj(self.messages.get_nowait())
time.sleep(0.05)
def run(self):
t = threading.Thread(target=self._deliver, name='pub_server')
t.daemon = True
t.start()
def unbind(self):
self.socket.unbind(self.address)
class TaskProcessor(object):
"""
Collect tasks and run with multiprocessing.Process
"""
def __init__(self, processes, process_timeout, result_queue_handler):
"""
:param processes: nun processes
:type processes: int
:param process_timeout: number of seconds for process timeout
:type process_timeout: int, float
:param result_queue_handler: handler for result queue
"""
self._processes = processes if processes > 0 else cpu_count()
self._current = []
self._queue = TaskQueue()
self._process_timeout = process_timeout
self._result_queue_handler = result_queue_handler
def _is_release(self):
if len(self._current) < self._processes:
return True
for process in self._current:
if not process.is_alive():
process.terminate()
process.join()
self._current.remove(process)
return True
return False
def add_task(self, target, args=None, kwargs=None):
self._queue.put_nowait((target, args or tuple(), kwargs or dict()))
def serve(self):
while not self._queue.empty():
try:
waiting_for(self._is_release, timeout=self._process_timeout, sleep=0.01)
except TimeoutException:
self.destroy()
raise
target, args, kwargs = self._queue.get_nowait()
process = Process(target=target, args=args, kwargs=kwargs)
process.start()
self._current.append(process)
self._result_queue_handler.handle()
def destroy(self):
"""
Kill all processes
"""
for process in self._current:
process.terminate()
def close(self):
for process in self._current:
process.join()
class ParserThread(Thread):
"""Parse the http results in parallel with getting them from the server
"""
def __init__(self, parser):
self.parser = parser
self.q = Queue()
self.ptime = 0
self.rawlength = 0
Thread.__init__(self)
def run(self):
self.result = []
while True:
next = self.q.get(True)
if next == None:
break
(url, item) = next
# print "parsing result for", url
item.seek(0)
tic = time.time()
val = self.parser(item.read())
self.ptime += time.time() - tic
self.rawlength += item.tell()
self.result.append((url, val))
def add(self, url, pval):
self.q.put_nowait((url, pval))
def finish(self):
self.q.put_nowait(None)
self.join()
return self.result
class Lock(object):
def __init__(self, io_loop=None):
self._lock = TLock()
self._queue = Queue()
self.io_loop = io_loop
@gen.engine
def run(self, callback, *args, **kwargs):
yield gen.Task(self.acquire)
callback(*args, **kwargs)
self.release()
@gen.engine
def run_async(self, callback, *args, **kwargs):
yield gen.Task(self.acquire)
yield gen.Task(callback, *args, **kwargs)
self.release()
def acquire(self, callback, *args, **kwargs):
if self._lock.acquire(False):
callback(*args, **kwargs)
else:
self._queue.put_nowait((callback, args, kwargs))
def release(self):
self._lock.release()
try:
callback, args, kwargs = self._queue.get_nowait()
if self.io_loop:
self.io_loop.add_callback(lambda: self.acquire(callback, *args, **kwargs))
else:
self.acquire(callback, *args, **kwargs)
except Empty:
pass
class Transceiver(object):
def __init__(self):
from Queue import Queue
print 'Initialising queue.'
self.queue = Queue(1)
def _get_name(self):
from threading import currentThread
return currentThread().getName()
def enqueue(self, msg):
from Queue import Full
if not isinstance(msg, Message):
raise Exception('%s: Das Fak !' % self._get_name())
try:
#self.queue.put(msg,block=True,timeout=1.0)
self.queue.put_nowait(msg)
print '%s: Enqueue %s.' % (self._get_name(),msg)
return True
except Full:
print '%s: Queue busy. Dropping %s.' % (self._get_name(),msg)
return False
def _process(self, msg):
raise Exception('Not implemented.')
def __call__(self, *args, **kwargs):
from Queue import Empty
while True:
try:
msg = self.queue.get(block=True)
except Empty:
continue
self._process(msg)
class TransmitterDevice(i2cMock.I2CDevice):
MAX_CONTENT_SIZE = 235
BUFFER_SIZE = 40
def __init__(self):
super(TransmitterDevice, self).__init__(0x62)
self.log = logging.getLogger("Transmitter")
self._reset = Event()
self._hwreset = Event()
self._buffer = Queue(TransmitterDevice.BUFFER_SIZE)
@i2cMock.command([0xAA])
def _reset(self):
self.log.info("Reset")
self._reset.set()
@i2cMock.command([0xAB])
def _hwreset(self):
print "hardware reset"
self._hwreset.set()
@i2cMock.command([0x10])
def _send_frame(self, *data):
self.log.info("Send frame %s", data)
self._buffer.put_nowait(data)
return [TransmitterDevice.BUFFER_SIZE - self._buffer.qsize()]
def wait_for_reset(self, timeout=None):
return self._reset.wait(timeout)
def get_message_from_buffer(self, timeout=None):
return self._buffer.get(timeout=timeout)
class DeviceConnectionObject(Foundation.NSObject):
def initWithHost_port_(self, host, port):
self = self.init()
if self is None: return None
self.client = DeviceClient(host, port)
self.commandQueue = Queue()
return self
def queueSize(self):
return self.commandQueue.qsize()
def appendCommand_(self, command):
self.willChangeValueForKey_('queueSize')
self.commandQueue.put_nowait([str(item) for item in command])
self.didChangeValueForKey_('queueSize')
def tickLoop(self):
asyncore.loop(timeout=0, count=1)
self.fillSendBuffer()
def disconnect(self):
self.client.close()
self.client = None
def fillSendBuffer(self):
if self.client.buffer or self.commandQueue.empty(): return
self.willChangeValueForKey_('queueSize')
command = self.commandQueue.get_nowait()
self.client.buffer = msgpack.packb(command)
self.didChangeValueForKey_('queueSize')
class HttpPool(object):
def __init__(self, threads_count, fail_op, log):
self._tasks = Queue()
self._results = Queue()
for i in xrange(threads_count):
thread.start_new_thread(get_remote_data,
(self._tasks, self._results, fail_op, log))
def add_task(self, tid, host, url, params, headers = {}, method = 'GET', timeout = None):
task = {
'id' : tid,
'conn_args' : {'host' : host} if timeout is None else {'host' : host, 'timeout' : timeout},
'headers' : headers,
'url' : url,
'params' : params,
'method' : method,
}
try:
self._tasks.put_nowait(task)
except Full:
return False
return True
def get_results(self):
results = []
while True:
try:
res = self._results.get_nowait()
except Empty:
break
results.append(res)
return results
class Pool(object):
def __init__(self):
self.__queue_ = Queue()
self.__thread_ = Thread(group=None, target = self.__thread_func_)
self.__started = False
def invoke(self, d, func, args, kwargs):
self.__queue_.put_nowait((d, func, args, kwargs))
def start(self):
if not self.__started:
self.__started = True
self.__thread_.start()
return defer.succeed(True)
def stop(self):
self.__started = False
def __thread_func_(self):
while self.__started:
(d, func, args, kwargs) = self.__queue_.get(True)
try:
result = func(*args, **kwargs)
reactor.callFromThread(d.callback, *(result,))
except Exception, e:
reactor.callFromThread(d.errback, Failure(e, Exception))
class Link:
def __init__(self, socket_obj, connection):
self._connection = connection
self._socket = socket_obj
self._read_buffer = bytearray()
self._write_queue = Queue()
def read(self):
result = bytearray('')
while True:
read_count = self._connection.recv_into(self._read_buffer, BUFFER_SIZE)
if read_count > 0:
result.extend(self._read_buffer[:read_count - 1])
else:
break
return bytearray
def send(self, data):
self._write_queue.put_nowait(data)
self._socket.notify_send_data(self._connection.fileno())
def write(self):
while self._write_queue.not_empty():
data = self._write_queue.get_nowait()
self._connection.sendall(data)
def close(self):
self._connection.close()
class IWRCBot():
def __init__(self, site, safe = True):
self.other_ns = re.compile(u'14\[\[07(' + u'|'.join(site.namespaces()) + u')')
interwiki.globalvar.autonomous = True
self.site = site
self.queue = Queue()
self.processed = []
self.safe = safe
# Start 20 threads
for i in range(20):
t = threading.Thread(target=self.worker)
t.setDaemon(True)
t.start()
def worker(self):
bot = interwiki.InterwikiBot()
while True:
# Will wait until one page is available
bot.add(self.queue.get())
bot.queryStep()
self.queue.task_done()
def addQueue(self, name):
if self.other_ns.match(name):
return
if self.safe:
if name in self.processed:
return
self.processed.append(name)
page = pywikibot.Page(self.site, name)
# the Queue has for now an unlimited size,
# it is a simple atomic append(), no need to acquire a semaphore
self.queue.put_nowait(page)
class MyStore(object):
def __init__(self):
"""initializes a Store"""
self.store = Queue(maxsize=100)
self.size = (24, 80)
self.typesOfFractals = ['julia', 'mandelbrot', 'test']
def put(self,item):
"""puts an item into the queue"""
try:
self.store.put_nowait(item)
except:
pass
return True
def get(self):
"""gets an item out of the store"""
try:
return self.store.get_nowait()
except:
return False
def set_size(self, height, width):
"""sets a size for the console"""
self.size = (int(height), int(width))
return True
def get_size(self):
"""Gets the console size"""
return self.size
def pick_type(self):
"""pick the type of fractal to generate"""
return random.choice(self.typesOfFractals)
class ConnectionPool(object):
def __init__(self, source_connection, min=2, max=None, preload=True):
self.source_connection = source_connection
self.min = min
self.max = max
self.preload = preload
self.source_connection.pool = self
self._connections = Queue()
self._dirty = deque()
self._connections.put(self.source_connection)
for i in range(min - 1):
self._connections.put_nowait(self._new_connection())
def acquire(self, block=False, timeout=None, connect_timeout=None):
try:
conn = self._connections.get(block=block, timeout=timeout)
except QueueEmpty:
conn = self._new_connection()
self._dirty.append(conn)
if connect_timeout is not None:
conn.connect_timeout = connect_timeout
return conn
def release(self, connection):
self._dirty.remove(connection)
self._connections.put_nowait(connection)
def _new_connection(self):
if len(self._dirty) >= self.max:
raise ConnectionLimitExceeded(self.max)
return copy(self.source_connection)
def batch_futures(self, session, statement_generator):
concurrency = 10
futures = Queue(maxsize=concurrency)
number_of_timeouts = 0
for i, statement in enumerate(statement_generator):
if i > 0 and i % (concurrency - 1) == 0:
# clear the existing queue
while True:
try:
futures.get_nowait().result()
except (OperationTimedOut, WriteTimeout):
ex_type, ex, tb = sys.exc_info()
number_of_timeouts += 1
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
time.sleep(1)
except Empty:
break
future = session.execute_async(statement)
futures.put_nowait(future)
while True:
try:
futures.get_nowait().result()
except (OperationTimedOut, WriteTimeout):
ex_type, ex, tb = sys.exc_info()
number_of_timeouts += 1
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
time.sleep(1)
except Empty:
break
return number_of_timeouts
class Protocol(RPCProtocol):
noisy = True
def __init__(self, *args, **kwargs):
self.queue = Queue(maxsize=10000)
self.thread_stop = False
self.thread = threading.Thread(target=self._thread_loop)
self.thread.start()
RPCProtocol.__init__(self, *args, **kwargs)
def _thread_loop(self):
while not self.thread_stop:
time.sleep(0.002) # dont hog cpu
for entry in storjnode.util.empty_queue(self.queue):
self.call(entry["sender"], entry["value"] + 1)
def stop(self):
self.thread_stop = True
self.thread.join()
def rpc_call(self, sender, value):
try:
print("Got {0} from {1}".format(value, sender))
self.queue.put_nowait({"sender": sender, "value": value})
return True
except Full:
print("Queue full, f**k!")
return False
class FM_Controller(object):
def __init__(self):
self.agent = FM_API()
self.player = Player()
qsize = 10
self.queue = Queue(qsize)
def load(self, url):
obj = pydl(url, progress_bar = False)
obj.start(blocking = True)
return obj.get_dest()
def refresh_list(self, sid = None):
if not sid:
songs = self.agent.get_playlist()
else:
songs = self.agent.update_playlist(sid)
for song in songs:
try:
self.queue.put_nowait(item)
except Full:
break
def load_d(self):
while True:
song = self.queue.get()
song['url'] = self.load(song['url'])
self.player.add()
class ThreadedWrapperHandler(WrapperHandler):
"""This handled uses a single background thread to dispatch log records
to a specific other handler using an internal queue. The idea is that if
you are using a handler that requires some time to hand off the log records
(such as the mail handler) and would block your request, you can let
Logbook do that in a background thread.
The threaded wrapper handler will automatically adopt the methods and
properties of the wrapped handler. All the values will be reflected:
>>> twh = ThreadedWrapperHandler(TestHandler())
>>> from logbook import WARNING
>>> twh.level_name = 'WARNING'
>>> twh.handler.level_name
'WARNING'
"""
_direct_attrs = frozenset(['handler', 'queue', 'controller'])
def __init__(self, handler, maxsize=0):
WrapperHandler.__init__(self, handler)
self.queue = ThreadQueue(maxsize)
self.controller = TWHThreadController(self)
self.controller.start()
def close(self):
self.controller.stop()
self.handler.close()
def emit(self, record):
try:
self.queue.put_nowait(record)
except Full:
# silently drop
pass
class Source:
def __init__(self, task_list, results, timeout, verbose = False):
self.tasks = Queue()
for task in task_list:
self.tasks.put_nowait(task)
self.results = results
self.timeout = timeout
self.verbose = verbose
def start(self, worker_count):
t0 = datetime.now()
sink = Sink(self.results)
self.workers = [ Worker(_+1, self.tasks, sink, self.timeout, self.verbose) for _ in range(worker_count) ]
if self.verbose:
print('[P] Starting workers.')
for w in self.workers:
w.t0 = t0
w.start()
ans = self.join_workers()
if self.verbose:
print('[P] Finished.')
return ans
def join_workers(self):
try:
for w in self.workers:
w.join(20000)
return True
except KeyboardInterrupt:
for w in self.workers:
w.stop = True
return False
def generate_items():
global items, symbols, goto_table
item_queue = Queue()
# 将第一个产生式[S->S']的闭包加进items
start_production = productions[0]
start_item = Item(0)
start_item.insert_item_line(
ItemLine(start_production.pid, 0, Symbol('$', 1)))
start_item = closure(start_item)
items.append(start_item)
item_queue.put_nowait(start_item)
while not item_queue.empty():
current_item = item_queue.get_nowait() # 弹出一个Item
for s in symbols:
next_item = goto(current_item, s)
# 检查next_item是否为空（即无条目）以及items中是否已经存在，这里应该要自己写判断函数
if next_item.item_lines:
tmp = is_item_exist(next_item)
if not tmp:
items.append(next_item)
item_queue.put_nowait(next_item)
else:
next_item = tmp
# 添加到goto表中
s_id = current_item.item_id
if s_id not in goto_table.keys():
goto_table[s_id] = {}
goto_table[s_id][s.value] = next_item.item_id # 建立边
class ProtonClient(object):
def __init__(self, reactor, connection, connector, session):
self.connector = connector
self.connection = connection
self.session = session
self.sender = None
self.links = []
self._inbox = None
self._outbox = Queue()
self._reactor = reactor
def _pushIncomingMessage(self, msg):
try:
self._inbox.put_nowait((self, msg))
except AttributeError:
# Inbox not set
pass
def _popPendingMessage(self):
return self._outbox.get_nowait()
def setInbox(self, queue):
self._inbox = queue
def send(self, msg):
self._outbox.put_nowait(msg)
self._reactor._activate(self.connector,
proton.PN_CONNECTOR_WRITABLE)
def close(self):
#TODO
pass
class AsyncPublisher(object):
def __init__(self, callback):
self.queue = Queue()
self.callback = callback
self._stop_sentinel = None
def start(self):
self.thread = threading.Thread(target=self._process_queue)
self.thread.setDaemon(True)
self.thread.start()
def on_entity(self, entity):
self.queue.put(entity)
def stop(self):
self.queue.put_nowait(self._stop_sentinel)
self.thread.join()
def _process_queue(self):
while True:
try:
entity = self.queue.get()
if entity is self._stop_sentinel:
return
self.callback(entity)
except KeyError:
pass
class Pipe(object):
"""Define a thread-safe pipe object to transfer data between filters."""
def __init__(self):
"""Create a Queue and an event to synchronize the filters."""
self.queue = Queue()
self.register = threading.Event()
def open_register(self):
"""Declare that the pipe is open. Simply sets the register event."""
self.register.set()
def close_register(self):
"""Declare that the pipe is closed. Simply clears the register event."""
self.register.clear()
def push(self, data_packet):
"""Insert data into the pipe."""
self.queue.put_nowait(data_packet)
def pull(self):
"""Return data from the pipe."""
return self.queue.get_nowait()
def has_flow(self):
"""Return True if there are data items in the pipe."""
return not self.queue.empty()
def is_open(self):
"""Return True if the register event is set."""
return self.register.is_set()
class ThreadPool(object):
def __init__(self, workers=10):
self.queue = Queue()
self.workers = []
self.tasks = []
for worker in xrange(workers):
self.workers.append(Worker(self.queue))
def add(self, ident, func, args=None, kwargs=None):
if args is None:
args = ()
if kwargs is None:
kwargs = {}
task = (ident, func, args, kwargs)
self.tasks.append(ident)
self.queue.put_nowait(task)
def join(self):
for worker in self.workers:
worker.start()
results = defaultdict(list)
for worker in self.workers:
worker.join()
for k, v in worker.results.iteritems():
results[k].extend(v)
return results
class UIAHandler(COMObject):
_com_interfaces_=[IUIAutomationEventHandler,IUIAutomationFocusChangedEventHandler,IUIAutomationPropertyChangedEventHandler,IUIAutomationNotificationEventHandler]
def __init__(self):
super(UIAHandler,self).__init__()
self.MTAThreadInitEvent=threading.Event()
self.MTAThreadInitException=None
self.MTAThreadQueue=Queue()
self.MTAThread=threading.Thread(target=self.MTAThreadFunc)
self.MTAThread.daemon=True
self.MTAThread.start()
self.MTAThreadInitEvent.wait(2)
if self.MTAThreadInitException:
raise self.MTAThreadInitException
def terminate(self):
MTAThreadHandle=HANDLE(windll.kernel32.OpenThread(winKernel.SYNCHRONIZE,False,self.MTAThread.ident))
self.MTAThreadQueue.put_nowait(None)
#Wait for the MTA thread to die (while still message pumping)
if windll.user32.MsgWaitForMultipleObjects(1,byref(MTAThreadHandle),False,200,0)!=0:
log.debugWarning("Timeout or error while waiting for UIAHandler MTA thread")
windll.kernel32.CloseHandle(MTAThreadHandle)
del self.MTAThread
def MTAThreadFunc(self):
try:
oledll.ole32.CoInitializeEx(None,comtypes.COINIT_MULTITHREADED)
isUIA8=False
try:
self.clientObject=CoCreateInstance(CUIAutomation8._reg_clsid_,interface=IUIAutomation,clsctx=CLSCTX_INPROC_SERVER)
isUIA8=True
except (COMError,WindowsError,NameError):
self.clientObject=CoCreateInstance(CUIAutomation._reg_clsid_,interface=IUIAutomation,clsctx=CLSCTX_INPROC_SERVER)
if isUIA8:
# #8009: use appropriate interface based on highest supported interface.
# #8338: made easier by traversing interfaces supported on Windows 8 and later in reverse.
for interface in reversed(CUIAutomation8._com_interfaces_):
try:
self.clientObject=self.clientObject.QueryInterface(interface)
break
except COMError:
pass
# Windows 10 RS5 provides new performance features for UI Automation including event coalescing and connection recovery.
# Enable all of these where available.
if isinstance(self.clientObject,IUIAutomation6):
self.clientObject.CoalesceEvents=CoalesceEventsOptions_Enabled
self.clientObject.ConnectionRecoveryBehavior=ConnectionRecoveryBehaviorOptions_Enabled
log.info("UIAutomation: %s"%self.clientObject.__class__.__mro__[1].__name__)
self.windowTreeWalker=self.clientObject.createTreeWalker(self.clientObject.CreateNotCondition(self.clientObject.CreatePropertyCondition(UIA_NativeWindowHandlePropertyId,0)))
self.windowCacheRequest=self.clientObject.CreateCacheRequest()
self.windowCacheRequest.AddProperty(UIA_NativeWindowHandlePropertyId)
self.UIAWindowHandleCache={}
self.baseTreeWalker=self.clientObject.RawViewWalker
self.baseCacheRequest=self.windowCacheRequest.Clone()
import UIAHandler
self.ItemIndex_PropertyId=NVDAHelper.localLib.registerUIAProperty(byref(ItemIndex_Property_GUID),u"ItemIndex",1)
self.ItemCount_PropertyId=NVDAHelper.localLib.registerUIAProperty(byref(ItemCount_Property_GUID),u"ItemCount",1)
for propertyId in (UIA_FrameworkIdPropertyId,UIA_AutomationIdPropertyId,UIA_ClassNamePropertyId,UIA_ControlTypePropertyId,UIA_ProviderDescriptionPropertyId,UIA_ProcessIdPropertyId,UIA_IsTextPatternAvailablePropertyId,UIA_IsContentElementPropertyId,UIA_IsControlElementPropertyId):
self.baseCacheRequest.addProperty(propertyId)
self.baseCacheRequest.addPattern(UIA_TextPatternId)
self.rootElement=self.clientObject.getRootElementBuildCache(self.baseCacheRequest)
self.reservedNotSupportedValue=self.clientObject.ReservedNotSupportedValue
self.ReservedMixedAttributeValue=self.clientObject.ReservedMixedAttributeValue
self.pendingForegroundUIAElement=None
self.currentForegroundUIAElement=None
self.clientObject.AddFocusChangedEventHandler(self.baseCacheRequest,self)
#self.clientObject.AddPropertyChangedEventHandler(self.rootElement,TreeScope_Subtree,self.baseCacheRequest,self,UIAPropertyIdsToNVDAEventNames.keys())
for x in UIAEventIdsToNVDAEventNames.iterkeys():
self.clientObject.addAutomationEventHandler(x,self.rootElement,TreeScope_Subtree,self.baseCacheRequest,self)
# #7984: add support for notification event (IUIAutomation5, part of Windows 10 build 16299 and later).
if isinstance(self.clientObject, IUIAutomation5):
self.clientObject.AddNotificationEventHandler(self.rootElement,TreeScope_Subtree,self.baseCacheRequest,self)
except Exception as e:
self.MTAThreadInitException=e
finally:
self.MTAThreadInitEvent.set()
while True:
func=self.MTAThreadQueue.get()
if func:
try:
func()
except:
log.error("Exception in function queued to UIA MTA thread",exc_info=True)
else:
break
self.clientObject.RemoveAllEventHandlers()
def _onForegroundChange(self):
pendingForegroundUIAElement=self.pendingForegroundUIAElement
if pendingForegroundUIAElement==self.currentForegroundUIAElement:
return
if self.currentForegroundUIAElement:
try:
self.clientObject.removePropertyChangedEventHandler(self.currentForegroundUIAElement,self)
except COMError:
# The old UIAElement died as the window was closed.
# The system should forget the old event registration itself.
pass
try:
self.clientObject.AddPropertyChangedEventHandler(pendingForegroundUIAElement,TreeScope_Subtree,self.baseCacheRequest,self,UIAPropertyIdsToNVDAEventNames.keys())
except COMError:
log.error("Could not register for UIA property change events for new foreground")
self.currentForegroundUIAElement=None
else:
self.currentForegroundUIAElement=pendingForegroundUIAElement
def onForegroundChange(self,hwnd):
try:
self.pendingForegroundUIAElement=self.clientObject.ElementFromHandle(hwnd)
except COMError:
log.debugWarning("Could not get a UIAElement from new foreground window")
return
# Event registration/unregistration must be always done from the MTA thread, otherwise deadlocks can occur with our UI.
self.MTAThreadQueue.put_nowait(self._onForegroundChange)
def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID):
if not self.MTAThreadInitEvent.isSet():
# UIAHandler hasn't finished initialising yet, so just ignore this event.
return
if eventID==UIA_MenuOpenedEventId and eventHandler.isPendingEvents("gainFocus"):
# We don't need the menuOpened event if focus has been fired,
# as focus should be more correct.
return
NVDAEventName=UIAEventIdsToNVDAEventNames.get(eventID,None)
if not NVDAEventName:
return
if not self.isNativeUIAElement(sender):
return
window=self.getNearestWindowHandle(sender)
if window and not eventHandler.shouldAcceptEvent(NVDAEventName,windowHandle=window):
return
import NVDAObjects.UIA
obj=NVDAObjects.UIA.UIA(UIAElement=sender)
if (
not obj
or (NVDAEventName=="gainFocus" and not obj.shouldAllowUIAFocusEvent)
or (NVDAEventName=="liveRegionChange" and not obj._shouldAllowUIALiveRegionChangeEvent)
):
return
focus=api.getFocusObject()
if obj==focus:
obj=focus
eventHandler.queueEvent(NVDAEventName,obj)
def IUIAutomationFocusChangedEventHandler_HandleFocusChangedEvent(self,sender):
if not self.MTAThreadInitEvent.isSet():
# UIAHandler hasn't finished initialising yet, so just ignore this event.
return
if not self.isNativeUIAElement(sender):
return
import NVDAObjects.UIA
if isinstance(eventHandler.lastQueuedFocusObject,NVDAObjects.UIA.UIA):
lastFocus=eventHandler.lastQueuedFocusObject.UIAElement
# Ignore duplicate focus events.
# It seems that it is possible for compareElements to return True, even though the objects are different.
# Therefore, don't ignore the event if the last focus object has lost its hasKeyboardFocus state.
if self.clientObject.compareElements(sender,lastFocus) and lastFocus.currentHasKeyboardFocus:
return
window=self.getNearestWindowHandle(sender)
if window and not eventHandler.shouldAcceptEvent("gainFocus",windowHandle=window):
return
obj=NVDAObjects.UIA.UIA(UIAElement=sender)
if not obj or not obj.shouldAllowUIAFocusEvent:
return
eventHandler.queueEvent("gainFocus",obj)
def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sender,propertyId,newValue):
# #3867: For now manually force this VARIANT type to empty to get around a nasty double free in comtypes/ctypes.
# We also don't use the value in this callback.
newValue.vt=VT_EMPTY
if not self.MTAThreadInitEvent.isSet():
# UIAHandler hasn't finished initialising yet, so just ignore this event.
return
NVDAEventName=UIAPropertyIdsToNVDAEventNames.get(propertyId,None)
if not NVDAEventName:
return
if not self.isNativeUIAElement(sender):
return
window=self.getNearestWindowHandle(sender)
if window and not eventHandler.shouldAcceptEvent(NVDAEventName,windowHandle=window):
return
import NVDAObjects.UIA
obj=NVDAObjects.UIA.UIA(UIAElement=sender)
if not obj:
return
focus=api.getFocusObject()
if obj==focus:
obj=focus
eventHandler.queueEvent(NVDAEventName,obj)
def IUIAutomationNotificationEventHandler_HandleNotificationEvent(self,sender,NotificationKind,NotificationProcessing,displayString,activityId):
if not self.MTAThreadInitEvent.isSet():
# UIAHandler hasn't finished initialising yet, so just ignore this event.
return
import NVDAObjects.UIA
obj=NVDAObjects.UIA.UIA(UIAElement=sender)
if not obj:
# Sometimes notification events can be fired on a UIAElement that has no windowHandle and does not connect through parents back to the desktop.
# There is nothing we can do with these.
return
eventHandler.queueEvent("UIA_notification",obj, notificationKind=NotificationKind, notificationProcessing=NotificationProcessing, displayString=displayString, activityId=activityId)
def _isUIAWindowHelper(self,hwnd):
# UIA in NVDA's process freezes in Windows 7 and below
processID=winUser.getWindowThreadProcessID(hwnd)[0]
if windll.kernel32.GetCurrentProcessId()==processID:
return False
import NVDAObjects.window
windowClass=NVDAObjects.window.Window.normalizeWindowClassName(winUser.getClassName(hwnd))
# For certain window classes, we always want to use UIA.
if windowClass in goodUIAWindowClassNames:
return True
# allow the appModule for the window to also choose if this window is good
# An appModule should be able to override bad UIA class names as prescribed by core
appModule=appModuleHandler.getAppModuleFromProcessID(processID)
if appModule and appModule.isGoodUIAWindow(hwnd):
return True
# There are certain window classes that just had bad UIA implementations
if windowClass in badUIAWindowClassNames:
return False
if windowClass=="NetUIHWND":
parentHwnd=winUser.getAncestor(hwnd,winUser.GA_ROOT)
# #2816: Outlook 2010 auto complete does not fire enough UIA events, IAccessible is better.
# #4056: Combo boxes in Office 2010 Options dialogs don't expose a name via UIA, but do via MSAA.
if winUser.getClassName(parentHwnd) in {"Net UI Tool Window","NUIDialog"}:
return False
# allow the appModule for the window to also choose if this window is bad
if appModule and appModule.isBadUIAWindow(hwnd):
return False
# Ask the window if it supports UIA natively
res=windll.UIAutomationCore.UiaHasServerSideProvider(hwnd)
if res:
# the window does support UIA natively, but
# Microsoft Word should not use UIA unless we can't inject or the user explicitly chose to use UIA with Microsoft word
if windowClass=="_WwG" and not (config.conf['UIA']['useInMSWordWhenAvailable'] or not appModule.helperLocalBindingHandle):
return False
return bool(res)
def isUIAWindow(self,hwnd):
now=time.time()
v=self.UIAWindowHandleCache.get(hwnd,None)
if not v or (now-v[1])>0.5:
v=self._isUIAWindowHelper(hwnd),now
self.UIAWindowHandleCache[hwnd]=v
return v[0]
def getNearestWindowHandle(self,UIAElement):
if hasattr(UIAElement,"_nearestWindowHandle"):
# Called previously. Use cached result.
return UIAElement._nearestWindowHandle
try:
processID=UIAElement.cachedProcessID
except COMError:
return None
appModule=appModuleHandler.getAppModuleFromProcessID(processID)
# WDAG (Windows Defender application Guard) UIA elements should be treated as being from a remote machine, and therefore their window handles are completely invalid on this machine.
# Therefore, jump all the way up to the root of the WDAG process and use that window handle as it is local to this machine.
if appModule.appName==WDAG_PROCESS_NAME:
condition=UIAUtils.createUIAMultiPropertyCondition({UIA_ClassNamePropertyId:[u'ApplicationFrameWindow',u'CabinetWClass']})
walker=self.clientObject.createTreeWalker(condition)
else:
# Not WDAG, just walk up to the nearest valid windowHandle
walker=self.windowTreeWalker
try:
new=walker.NormalizeElementBuildCache(UIAElement,self.windowCacheRequest)
except COMError:
return None
try:
window=new.cachedNativeWindowHandle
except COMError:
window=None
# Cache for future use to improve performance.
UIAElement._nearestWindowHandle=window
return window
def isNativeUIAElement(self,UIAElement):
#Due to issues dealing with UIA elements coming from the same process, we do not class these UIA elements as usable.
#It seems to be safe enough to retreave the cached processID, but using tree walkers or fetching other properties causes a freeze.
try:
processID=UIAElement.cachedProcessId
except COMError:
return False
if processID==windll.kernel32.GetCurrentProcessId():
return False
# Whether this is a native element depends on whether its window natively supports UIA.
windowHandle=self.getNearestWindowHandle(UIAElement)
if windowHandle:
if self.isUIAWindow(windowHandle):
return True
if winUser.getClassName(windowHandle)=="DirectUIHWND" and "IEFRAME.dll" in UIAElement.cachedProviderDescription and UIAElement.currentClassName in ("DownloadBox", "accessiblebutton", "DUIToolbarButton", "PushButton"):
# This is the IE 9 downloads list.
# #3354: UiaHasServerSideProvider returns false for the IE 9 downloads list window,
# so we'd normally use MSAA for this control.
# However, its MSAA implementation is broken (fires invalid events) if UIA is initialised,
# whereas its UIA implementation works correctly.
# Therefore, we must use UIA here.
return True
return False
def create():
"""
Creates a new RbacRoleProfile.
A role profile is a cache which speeds up child/parent lookups for
roles.
"""
logger.debug("Creating RbacRoleProfile")
if settings.USE_TZ:
currentTime = datetime.utcnow().replace(tzinfo=utc)
else:
currentTime = datetime.now()
adj_list = {}
bulk_list = []
pairs = set()
with transaction.atomic():
#create an adjacency list of the role hierarchy
for parent, child in RbacRole.objects.exclude(
children=None).values_list('id', 'children'):
if parent in adj_list:
adj_list[parent].append(child)
else:
adj_list[parent] = [
child,
]
#Search for all child nodes which can be reached from parent through
# a breadth-first-search.
#Instead of coloring we're using a dict which keeps track of the
# discovered nodes (@see: BFS)
for parent in adj_list:
child_queue = Queue()
found = {}
for child in adj_list[parent]:
child_queue.put_nowait(child)
found[child] = True
while not child_queue.empty():
node = child_queue.get_nowait()
pair = (parent, node)
if pair not in pairs:
pairs.add(pair)
else:
continue
bulk_list.append(
RbacRoleProfile(parent_id=parent,
child_id=node,
touch_date=currentTime,
create_date=currentTime))
if node in adj_list:
for child in adj_list[node]:
if child not in found:
child_queue.put_nowait(child)
#clear previous cache
RbacRoleProfile.objects.all().delete()
RbacRoleProfile.objects.bulk_create(bulk_list)
logger.debug("Finished creating RbacRoleProfile")
class HabitatUploader(object):
'''
Queued Habitat Telemetry Uploader class
This performs uploads to the Habitat servers, and also handles generation of flight documents.
Incoming telemetry packets are fed into queue, which is checked regularly.
If a new callsign is sighted, a payload document is created in the Habitat DB.
The telemetry data is then converted into a UKHAS-compatible format, before being added to queue to be
uploaded as network speed permits.
If an upload attempt times out, the packet is discarded.
If the queue fills up (probably indicating no network connection, and a fast packet downlink rate),
it is immediately emptied, to avoid upload of out-of-date packets.
Note that this uploader object is intended to handle telemetry from multiple sondes
'''
# We require the following fields to be present in the incoming telemetry dictionary data
REQUIRED_FIELDS = [
'frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq',
'freq_float', 'datetime_dt'
]
def __init__(self,
user_callsign='N0CALL',
user_position=None,
user_antenna="",
payload_callsign_override=None,
synchronous_upload_time=30,
callsign_validity_threshold=5,
upload_queue_size=16,
upload_timeout=10,
upload_retries=5,
upload_retry_interval=0.25,
user_position_update_rate=6,
inhibit=False):
""" Initialise a Habitat Uploader object.
Args:
user_callsign (str): Callsign of the uploader.
user_position (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated,
is used to plot the listener's position on the Habitat map, both when this class is initialised, and
when a new sonde ID is observed.
payload_callsign_override (str): Override the payload callsign in the uploaded sentence with this value.
WARNING: This will horribly break the tracker map if multiple sondes are uploaded under the same callsign.
USE WITH CAUTION!!!
synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
and also acts as decimation on the number of sentences uploaded to Habitat.
callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
upload_queue_size (int): Maximum umber of sentences to keep in the upload queue. If the queue is filled,
it will be emptied (discarding the queue contents).
upload_timeout (int): Timeout (Seconds) when performing uploads to Habitat. Default: 10 seconds.
upload_retries (int): Retry an upload up to this many times. Default: 5
upload_retry_interval (int): Time interval between upload retries. Default: 0.25 seconds.
user_position_update_rate (int): Time interval between automatic station position updates, hours.
Set to 6 hours by default, updating any more often than this is not really useful.
inhibit (bool): Inhibit all uploads. Mainly intended for debugging.
"""
self.user_callsign = user_callsign
self.user_position = user_position
self.user_antenna = user_antenna
self.payload_callsign_override = payload_callsign_override
self.upload_timeout = upload_timeout
self.upload_retries = upload_retries
self.upload_retry_interval = upload_retry_interval
self.upload_queue_size = upload_queue_size
self.synchronous_upload_time = synchronous_upload_time
self.callsign_validity_threshold = callsign_validity_threshold
self.inhibit = inhibit
self.user_position_update_rate = user_position_update_rate
# Our two Queues - one to hold sentences to be upload, the other to temporarily hold
# input telemetry dictionaries before they are converted and processed.
self.habitat_upload_queue = Queue(upload_queue_size)
self.input_queue = Queue()
# Dictionary where we store sorted telemetry data for upload when required.
# Elements will be named after payload IDs, and will contain:
# 'count' (int): Number of times this callsign has been observed. Uploads will only occur when
# this number rises above callsign_validity_threshold.
# 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires,
# this queue will be dumped, and the most recent telemetry uploaded.
# 'habitat_document' (bool): Indicates if a habitat document has been created for this payload ID.
# 'listener_updated' (bool): Indicates if the listener position has been updated for the start of this ID's flight.
self.observed_payloads = {}
# Record of when we last uploaded a user station position to Habitat.
self.last_user_position_upload = 0
# Lock for dealing with telemetry uploads.
self.upload_lock = Lock()
# Start the uploader thread.
self.upload_thread_running = True
self.upload_thread = Thread(target=self.habitat_upload_thread)
self.upload_thread.start()
# Start the input queue processing thread.
self.input_processing_running = True
self.input_thread = Thread(target=self.process_queue)
self.input_thread.start()
self.timer_thread_running = True
self.timer_thread = Thread(target=self.upload_timer)
self.timer_thread.start()
def user_position_upload(self):
""" Upload the the station position to Habitat. """
if self.user_position is not None:
_success = uploadListenerPosition(self.user_callsign,
self.user_position[0],
self.user_position[1],
version=auto_rx_version,
antenna=self.user_antenna)
self.last_user_position_upload = time.time()
return _success
else:
return False
def habitat_upload(self, sentence):
''' Upload a UKHAS-standard telemetry sentence to Habitat
Args:
sentence (str): The UKHAS-standard telemetry sentence to upload.
'''
if self.inhibit:
self.log_info("Upload inhibited.")
return
# Generate payload to be uploaded
_sentence_b64 = b64encode(sentence.encode(
'ascii')) # Encode to ASCII to be able to perform B64 encoding...
_date = datetime.datetime.utcnow().isoformat("T") + "Z"
_user_call = self.user_callsign
_data = {
"type": "payload_telemetry",
"data": {
"_raw": _sentence_b64.decode(
'ascii'
) # ... but decode back to a string to enable JSON serialisation.
},
"receivers": {
_user_call: {
"time_created": _date,
"time_uploaded": _date,
},
},
}
# The URL to upload to.
_url = HABITAT_URL + "habitat/_design/payload_telemetry/_update/add_listener/%s" % sha256(
_sentence_b64).hexdigest()
# Delay for a random amount of time between 0 and upload_retry_interval*2 seconds.
time.sleep(random.random() * self.upload_retry_interval * 2.0)
_retries = 0
# When uploading, we have three possible outcomes:
# - Can't connect. No point immediately re-trying in this situation.
# - The packet is uploaded successfuly (201 / 403)
# - There is a upload conflict on the Habitat DB end (409). We can retry and it might work.
while _retries < self.upload_retries:
# Run the request.
try:
headers = {"User-Agent": "autorx-" + auto_rx_version}
_req = requests.put(_url,
data=json.dumps(_data),
timeout=self.upload_timeout,
headers=headers)
except Exception as e:
self.log_error("Upload Failed: %s" % str(e))
break
if _req.status_code == 201 or _req.status_code == 403:
# 201 = Success, 403 = Success, sentence has already seen by others.
self.log_info("Uploaded sentence to Habitat successfully: %s" %
sentence.strip())
_upload_success = True
break
elif _req.status_code == 409:
# 409 = Upload conflict (server busy). Sleep for a moment, then retry.
self.log_debug("Upload conflict.. retrying.")
time.sleep(random.random() * self.upload_retry_interval)
_retries += 1
else:
self.log_error(
"Error uploading to Habitat. Status Code: %d %s." %
(_req.status_code, _req.text))
break
if _retries == self.upload_retries:
self.log_error("Upload conflict not resolved with %d retries." %
self.upload_retries)
return
def habitat_upload_thread(self):
''' Handle uploading of packets to Habitat '''
self.log_debug("Started Habitat Uploader Thread.")
while self.upload_thread_running:
if self.habitat_upload_queue.qsize() > 0:
# If the queue is completely full, jump to the most recent telemetry sentence.
if self.habitat_upload_queue.qsize() == self.upload_queue_size:
while not self.habitat_upload_queue.empty():
sentence = self.habitat_upload_queue.get()
self.log_warning(
"Uploader queue was full - possible connectivity issue."
)
else:
# Otherwise, get the first item in the queue.
sentence = self.habitat_upload_queue.get()
# Attempt to upload it.
self.habitat_upload(sentence)
else:
# Wait for a short time before checking the queue again.
time.sleep(0.1)
self.log_debug("Stopped Habitat Uploader Thread.")
def handle_telem_dict(self, telem, immediate=False):
# Try and convert it to a UKHAS sentence
try:
_sentence = sonde_telemetry_to_sentence(telem)
except Exception as e:
self.log_error("Error converting telemetry to sentence - %s" %
str(e))
return
_callsign = "RS_" + telem['id']
# Wait for the upload_lock to be available, to ensure we don't end up with
# race conditions resulting in multiple payload docs being created.
self.upload_lock.acquire()
# Create a habitat document if one does not already exist:
if not self.observed_payloads[telem['id']]['habitat_document']:
# Check if there has already been telemetry from this ID observed on Habhub
_document_exists = check_callsign(_callsign)
# If so, we don't need to create a new document
if _document_exists:
self.observed_payloads[telem['id']]['habitat_document'] = True
else:
# Otherwise, we attempt to create a new document.
if self.inhibit:
# If we have an upload inhibit, don't create a payload doc.
_created = True
else:
_created = initPayloadDoc(
_callsign,
description="Meteorology Radiosonde",
frequency=telem['freq_float'])
if _created:
self.observed_payloads[
telem['id']]['habitat_document'] = True
else:
self.log_error("Error creating payload document!")
self.upload_lock.release()
return
if immediate:
self.log_info(
"Performing immediate upload for first telemetry sentence of %s."
% telem['id'])
self.habitat_upload(_sentence)
else:
# Attept to add it to the habitat uploader queue.
try:
self.habitat_upload_queue.put_nowait(_sentence)
except Exception as e:
self.log_error("Error adding sentence to queue: %s" % str(e))
self.upload_lock.release()
def upload_timer(self):
""" Add packets to the habitat upload queue if it is time for us to upload. """
while self.timer_thread_running:
if int(time.time()) % self.synchronous_upload_time == 0:
# Time to upload!
for _id in self.observed_payloads.keys():
# If no data, continue...
if self.observed_payloads[_id]['data'].empty():
continue
else:
# Otherwise, dump the queue and keep the latest telemetry.
while not self.observed_payloads[_id]['data'].empty():
_telem = self.observed_payloads[_id]['data'].get()
self.handle_telem_dict(_telem)
# Sleep a second so we don't hit the synchronous upload time again.
time.sleep(1)
else:
# Not yet time to upload, wait for a bit.
time.sleep(0.1)
def process_queue(self):
""" Process packets from the input queue.
This thread handles packets from the input queue (provided by the decoders)
Packets are sorted by ID, and a dictionary entry is created.
"""
while self.input_processing_running:
# Process everything in the queue.
while self.input_queue.qsize() > 0:
# Grab latest telem dictionary.
_telem = self.input_queue.get_nowait()
_id = _telem['id']
if _id not in self.observed_payloads:
# We haven't seen this ID before, so create a new dictionary entry for it.
self.observed_payloads[_id] = {
'count': 1,
'data': Queue(),
'habitat_document': False,
'listener_updated': False
}
self.log_debug(
"New Payload %s. Not observed enough to allow upload."
% _id)
# However, we don't yet add anything to the queue for this payload...
else:
# We have seen this payload before!
# Increment the 'seen' counter.
self.observed_payloads[_id]['count'] += 1
# If we have seen this particular ID enough times, add the data to the ID's queue.
if self.observed_payloads[_id][
'count'] >= self.callsign_validity_threshold:
# If this is the first time we have observed this payload, update the listener position.
if (self.observed_payloads[_id]['listener_updated']
== False) and (self.user_position is not None):
self.observed_payloads[_id][
'listener_updated'] = self.user_position_upload(
)
# Because receiving balloon telemetry appears to be a competition, immediately upload the
# first valid position received.
self.handle_telem_dict(_telem, immediate=True)
else:
# Otherwise, add the telemetry to the upload queue
self.observed_payloads[_id]['data'].put(_telem)
else:
self.log_debug(
"Payload ID %s not observed enough to allow upload."
% _id)
# If we haven't uploaded our station position recently, re-upload it.
if (time.time() - self.last_user_position_upload
) > self.user_position_update_rate * 3600:
self.user_position_upload()
time.sleep(0.1)
def add(self, telemetry):
""" Add a dictionary of telemetry to the input queue.
Args:
telemetry (dict): Telemetry dictionary to add to the input queue.
"""
# Check the telemetry dictionary contains the required fields.
for _field in self.REQUIRED_FIELDS:
if _field not in telemetry:
self.log_error("JSON object missing required field %s" %
_field)
return
# Add it to the queue if we are running.
if self.input_processing_running:
self.input_queue.put(telemetry)
else:
self.log_error("Processing not running, discarding.")
def close(self):
''' Shutdown uploader and processing threads. '''
self.log_debug("Waiting for threads to close...")
self.input_processing_running = False
self.timer_thread_running = False
self.upload_thread_running = False
# Wait for all threads to close.
if self.upload_thread is not None:
self.upload_thread.join()
if self.timer_thread is not None:
self.timer_thread.join()
if self.input_thread is not None:
self.input_thread.join()
def log_debug(self, line):
""" Helper function to log a debug message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.debug("Habitat - %s" % line)
def log_info(self, line):
""" Helper function to log an informational message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.info("Habitat - %s" % line)
def log_error(self, line):
""" Helper function to log an error message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.error("Habitat - %s" % line)
def log_warning(self, line):
""" Helper function to log a warning message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.warning("Habitat - %s" % line)
class LaborThread(Thread):
""""""
#----------------------------------------------------------------------
def __init__(self, result_queue, master, clean_mod=True, *args, **kargs):
"""Constructor"""
Thread.__init__(self,
name='ThreadPool-Labor-' + uuid1().hex,
*args,
**kargs)
self._master = master
self._clean_mod = clean_mod
self._result_queue = result_queue
self._startworkingflag_ = True
self._task_queue = Queue(1)
self._count_lock = Lock()
#----------------------------------------------------------------------
def get_result_queue(self):
""""""
return self._result_queue
#----------------------------------------------------------------------
def get_task_queue(self):
""""""
return self._task_queue
#----------------------------------------------------------------------
def feed(self, function, *vargs, **kwargs):
""""""
try:
self._task_queue.put_nowait(tuple([function, vargs, kwargs]))
return True
except Full:
#format_exc()
return False
#----------------------------------------------------------------------
def run(self):
""""""
while self._startworkingflag_:
#pprint('Running')
try:
_task = self._task_queue.get(timeout=3)
result = {}
result['from'] = self.name
result['state'] = False
result['result'] = None
result['current_task'] = _task.__str__()
result['exception'] = tuple()
try:
ret = self._process_task(_task)
result['state'] = True
result['result'] = ret
#self._result_queue.put(result)
except Exception as e:
result['state'] = False
result['result'] = None
exception_i = (str(type(e)), str(e))
result['exception'] = exception_i
finally:
if self._clean_mod:
_result = {}
_result['state'] = result['state']
_result['result'] = result['result']
result = _result
self._result_queue.put(result)
self._count_lock.acquire()
self._master._executed_task_count = \
self._master._executed_task_count + 1
self._count_lock.release()
except Empty:
pass
#----------------------------------------------------------------------
def _process_task(self, task):
""""""
try:
ret = task[0](*task[1], **task[2])
return ret
except Exception as e:
raise e
#----------------------------------------------------------------------
def stop(self):
""""""
#self.stop()
self._startworkingflag_ = False
#----------------------------------------------------------------------
def __del__(self):
""""""
self.stop()
#----------------------------------------------------------------------
def _exception_process(self):
""""""
class VelbusDev:
"""
Velbus domogik plugin
"""
def __init__(self, log, cb_send_xpl, cb_send_trig, stop):
""" Init object
@param log : log instance
@param cb_send_xpl : callback
@param cb_send_trig : callback
@param stop :
"""
self._log = log
self._callback = cb_send_xpl
self._cb_send_trig = cb_send_trig
self._stop = stop
self._dev = None
self._devtype = 'serial'
self._nodes = {}
# Queue for writing packets to Rfxcom
self.write_rfx = Queue()
# Thread to process queue
write_process = threading.Thread(None, self.write_daemon,
"write_packets_process", (), {})
write_process.start()
def open(self, device, devicetype):
""" Open (opens the device once)
@param device : the device string to open
"""
self._devtype = devicetype
try:
self._log.info("Try to open VELBUS: %s" % device)
if devicetype == 'socket':
addr = device.split(':')
addr = (addr[0], int(addr[1]))
self._dev = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._dev.connect(addr)
else:
self._dev = serial.Serial(device, 38400, timeout=0)
self._log.info("VELBUS opened")
except:
error = "Error while opening Velbus : %s. Check if it is the good device or if you have the good permissions on it." % device
raise VelbusException(error)
def close(self):
""" Close the open device
"""
self._log.info("Close VELBUS")
try:
self._dev.close()
except:
error = "Error while closing device"
raise VelbusException(error)
def scan(self):
self._log.info("Starting the bus scan")
for add in range(0, 255):
self.send_moduletyperequest(add)
self._log.info("Bus scan finished")
def send_shutterup(self, address, channel):
""" Send shutter up message
"""
data = chr(0x05) + self._blinchannel_to_byte(channel) + chr(
0x00) + chr(0x00) + chr(0x00)
self.write_packet(address, data)
def send_shutterdown(self, address, channel):
""" Send shutter down message
"""
data = chr(0x06) + self._blinchannel_to_byte(channel) + chr(
0x00) + chr(0x00) + chr(0x00)
self.write_packet(address, data)
def send_level(self, address, channel, level):
""" Set the level for a device
if relay => level can only be 0 or 100
if dimmer => level can be anything from 0 to 100
"""
address = int(address)
self._log.debug("received set_level for {0}".format(address))
if address in self._nodes.keys():
mtype = self._nodes[address]
else:
self._log.error(
"Request to set a level on a device, but the device is not known. address {0}"
.format(address))
return
try:
ltype = MODULE_TYPES[mtype]["subtype"]
except KeyError:
self._log.error(
"Request to set a level on a device, but the subtype is not known. mtype {0}"
.format(mtype))
return
if ltype == "DIMMER":
""" Send dimemr value
- speed = 1 second
"""
level = (255 / 100) * level
data = chr(0x07) + self._channels_to_byte(channel) + chr(
int(level)) + chr(0x00) + chr(0x01)
self.write_packet(address, data)
elif int(level) == 255 and ltype == "RELAY":
""" Send relay on message
"""
data = chr(0x02) + self._channels_to_byte(channel)
self.write_packet(address, data)
elif int(level) == 0 and ltype == "RELAY":
""" Send relay off message
"""
data = chr(0x01) + self._channels_to_byte(channel)
self.write_packet(address, data)
else:
self._log.error(
"This methode should only be called for dimmers or relays and with level 0 to 255"
)
return
def send_moduletyperequest(self, address):
""" Request module type
"""
self.write_packet(address, None)
def write_packet(self, address, data):
""" put a packet in the write queu
"""
self._log.debug("put packet for {0} in send queue ({1})".format(
address, data))
self.write_rfx.put_nowait({"address": address, "data": data})
def write_daemon(self):
""" handle the queu
"""
self._log.info("write deamon")
while not self._stop.isSet():
res = self.write_rfx.get(block=True)
self._log.debug("start sending packet to {0}".format(
hex(int(res["address"]))))
# start (8bit)
packet = chr(0x0F)
# priority (8bit, F8=high, FB=low)
if res["data"] == None:
packet += chr(0xFB)
else:
packet += chr(0xF8)
# address (8bit)
packet += chr(int(res["address"]))
if res["data"] == None:
# module type request
packet += chr(0x40)
else:
packet += chr(len(res["data"]))
# data
packet += res["data"]
# checksum (8bit)
packet += self._checksum(packet)
# end byte (8bit)
packet += chr(0x04)
self._log.debug(packet.encode('hex'))
# send
if self._devtype == 'socket':
self._dev.send(packet)
else:
self._dev.write(packet)
# sleep for 60ms
self._stop.wait(0.06)
def listen(self, stop):
""" Listen thread for incomming VELBUS messages
"""
self._log.info("Start listening VELBUS")
# infinite
try:
while not stop.isSet():
self.read()
except:
error = "Error while reading velbus device (disconnected ?) : %s" % traceback.format_exc(
)
print(error)
self._log.error(error)
return
def read(self):
""" Read data from the velbus line
"""
if self._devtype == 'socket':
data = self._dev.recv(9999)
else:
data = self._dev.read(9999)
if len(data) >= 6:
if ord(data[0]) == 0x0f:
size = ord(data[3]) & 0x0F
self._parser(data[0:6 + size])
def _checksum(self, data):
"""
Calculate the velbus checksum
"""
assert isinstance(data, str)
__checksum = 0
for data_byte in data:
__checksum += ord(data_byte)
__checksum = -(__checksum % 256) + 256
try:
__checksum = chr(__checksum)
except ValueError:
__checksum = chr(0)
return __checksum
def _parser(self, data):
"""
parse the velbus packet
"""
assert isinstance(data, str)
assert len(data) > 0
assert len(data) >= 6
assert ord(data[0]) == 0x0f
self._log.debug("starting parser: %s" % data.encode('hex'))
if len(data) > 14:
self._log.warning(
"Velbus message: maximum %s bytes, this one is %s",
str(14, str(len(data))))
return
if ord(data[-1]) != 0x04:
self._log.warning("Velbus message: end byte not correct")
return data
if ord(data[1]) != 0xfb and ord(data[1]) != 0xf8:
self._log.warning("Velbus message: unrecognized priority")
return
data_size = ord(data[3]) & 0x0F
if data_size + 6 != len(data):
self._log.warning(
"length of data size does not match actual length of message")
return
if not self._checksum(data[:-2]) == data[-2]:
self._log.warning("Packet has no valid checksum")
return
if data_size > 0:
if ord(data[4]) in MSG_TYPES:
# lookup the module type
try:
if ord(data[2]) in self._nodes.keys():
mtype = self._nodes[ord(data[2])]
else:
mtype = None
except KeyError:
mtype = None
if mtype:
self._log.debug(
"Received message with type: '%s' address: %s module: %s(%s)"
% (MSG_TYPES[ord(data[4])], ord(
data[2]), MODULE_TYPES[mtype]['id'], mtype))
else:
self._log.debug(
"Received message with type: '%s' address: %s module: UNKNOWN"
% (MSG_TYPES[ord(data[4])], ord(data[2])))
# first try the module specifick parser
parsed = False
if mtype:
try:
methodcall = getattr(
self,
"_process_{0}_{1}".format(ord(data[4]), mtype))
methodcall(data)
parsed = True
except AttributeError:
self._log.debug(
"Messagetype module specifick parser not implemented"
)
if not parsed:
try:
methodcall = getattr(
self, "_process_{0}".format(ord(data[4])))
methodcall(data)
except AttributeError:
self._log.debug("Messagetype unimplemented {0}".format(
ord(data[4])))
else:
self._log.warning(
"Received message with unknown type {0}".format(
ord(data[4])))
else:
if (ord(data[3]) & 0x40 == 0x40):
self._log.debug("Received module type request")
else:
self._log.warning(
"zero sized message received without rtr set")
# procee the velbus received messages
# format will beL
# _process_<messageId> => general parser for this messagetype
# _process_<messageId>_<moduleType> => parser specifickly for this module type
def _process_255(self, data):
"""
Process a 255 Message
Node type => send out as answer on a module_type_request
"""
naddress = ord(data[2])
ntype = ord(data[5])
self._log.info(
"Found node with address {0} and module_type {1}".format(
str(naddress), MODULE_TYPES[ntype]['id']))
self._nodes[naddress] = ntype
def _process_251_8(self, data):
"""
Process a 251 Message
Specifickly for VMB4RY
Switch status => send out when a relay is switched
"""
naddress = ord(data[2])
#chan = self._blinchannel_to_byte(data[5])
status = ord(data[7])
#self._callback("lighting.device",
# {"device" : device,
# "level" : level})
def _process_251(self, data):
"""
Process a 251 Message
Switch status => send out when a relay is switched
"""
for channel in self._byte_to_channels(data[5]):
address = str(ord(data[2]))
channel = str(channel)
level = -1
if (ord(data[7]) & 0x03) == 0:
level = 0
if (ord(data[7]) & 0x03) == 1:
level = 255
if level != -1:
self._callback(
"lighting.device", {
"device": str(ord(data[2])),
"channel": str(channel),
"level": level
})
def _process_238(self, data):
"""
Process a 251 Message
Dimmer status => send out when the dimmer status is changed
"""
level = -1
level = (100 / 255) * ord(data[7])
if level != -1:
self._callback(
"lighting.device", {
"device": str(ord(data[2])),
"channel": str(ord(data[5]) - 1),
"level": level
})
def _process_190(self, data):
device = ord(data[2])
channel = (ord(data[5]) & 3) + 1
pulses = (ord(data[5]) >> 2) * 100
counter = (ord(data[6]) << 24) + (ord(data[7]) << 16) + (
ord(data[8]) << 8) + ord(data[9])
kwh = float(float(counter) / pulses)
delay = (ord(data[10]) << 8) + ord(data[11])
watt = float((1000 * 1000 * 3600) / (delay * pulses))
# transmit kwh
self._callback(
"sensor.basic", {
"device": device,
"channel": channel,
"type": "energy",
"units": "kWh",
"current": str(kwh)
})
# transmit watt
self._callback(
"sensor.basic", {
"device": device,
"channel": channel,
"type": "power",
"units": "W",
"current": str(watt)
})
def _process_184(self, data):
"""
Process a 184 Message
Dimmer channel status => send out when the dimmer status is changed
"""
for channel in self._byte_to_channels(data[5]):
level = -1
level = (100 / 255) * ord(data[7])
if level != -1:
self._callback(
"lighting.device", {
"device": str(ord(data[2])),
"channel": str(channel),
"level": level
})
def _process_0(self, data):
"""
Process a 0 Message
switch status => send out when an input (switch changed)
HIGH = just pressed
LOW = just released
LONG = long pressed
"""
device = str(ord(data[2]))
chanpres = self._byte_to_channels(data[5])
chanrel = self._byte_to_channels(data[6])
chanlpres = self._byte_to_channels(data[7])
for chan in chanpres:
self._callback(
"sensor.basic", {
"device": str(device),
"channel": str(chan),
"type": "input",
"current": "HIGH"
})
for chan in chanlpres:
self._callback(
"sensor.basic", {
"device": str(device),
"channel": str(chan),
"type": "input",
"current": "LONG"
})
for chan in chanrel:
self._callback(
"sensor.basic", {
"device": str(device),
"channel": str(chan),
"type": "input",
"current": "LOW"
})
def _process_236(self, data):
"""
Process a 236 Message
blind channel status => send out when the blind status changes
chan <X> <status>
chan: 00000011=1, 00001100=2
status: 0=off, 1=chan 1 up, 2=chan 1 down, 4=chan 2 up, 8=chan 2 down
foreach _byte_to_blindchannel(data[5])
status _byte_to_channels data[7]
"""
chan = self._blinchannel_to_byte(data[5])
device = str(ord(data[2]))
status = self._byte_to_channels(data[7])
command = []
if chan == 1:
if 1 in status:
command = "up"
if 2 in status:
command = "down"
elif chan == 2:
if 4 in status:
command = "up"
if 8 in status:
command = "down"
else:
command = "off"
if command == "":
self._callback("shutter.device", {
"device": device + "-" + chan,
"command": command
})
def _process_230(self, data):
"""
Process a 230 message Temperature Sensor Temperature
Databyte 2 => High byte current sensor temperature
Databyte 3 => Low byte of current temperature sensor in two's complement format
Resolution: 0.0625 degree celcius
"""
device = str(ord(data[2]))
cur = ord(data[5]) << 8
cur = ((cur | ord(data[6])) / 32) * 0.0625
low = ord(data[7]) << 8
low = ((low | ord(data[8])) / 32) * 0.0625
high = ord(data[9]) << 8
high = ((high | ord(data[10])) / 32) * 0.0625
self._callback(
"sensor.basic", {
"device": device,
"type": "temp",
"units": "c",
"current": str(cur),
"lowest": str(low),
"highest": str(high)
})
# Some convert procs
def _channels_to_byte(self, chan):
"""
Convert a channel to a byte
only works for one channel at a time
"""
return chr((1 << (int(chan) - 1)))
def _byte_to_channels(self, byte):
"""
Convert a byte to a channel list
"""
assert isinstance(byte, str)
assert len(byte) == 1
byte = ord(byte)
result = []
for offset in range(0, 8):
if byte & (1 << offset):
result.append(offset + 1)
return result
def _blinchannel_to_byte(self, channel):
"""
Convert a channel 1 or 2 to its correct byte
"""
assert isinstance(channel, int)
if channel == 1:
return chr(0x03)
else:
return chr(0x0C)
def _byte_to_blindchannel(self, byte):
"""
Convert a byte to its channel
"""
if byte == chr(0x03):
return 1
else:
return 2
class JsonRpcServer(object):
log = logging.getLogger("jsonrpc.JsonRpcServer")
"""
Creates new JsonrRpcServer by providing a bridge, timeout in seconds
which defining how often we should log connections stats and thread
factory.
"""
def __init__(self, bridge, timeout, threadFactory=None):
self._bridge = bridge
self._workQueue = Queue()
self._threadFactory = threadFactory
self._timeout = timeout
self._next_report = monotonic_time() + self._timeout
self._counter = 0
def queueRequest(self, req):
self._workQueue.put_nowait(req)
"""
Aggregates number of requests received by vdsm. Each request from
a batch is added separately. After time defined by timeout we log
number of requests.
"""
def _attempt_log_stats(self):
self._counter += 1
if monotonic_time() > self._next_report:
self.log.info('%s requests processed during %s seconds',
self._counter, self._timeout)
self._next_report += self._timeout
self._counter = 0
def _serveRequest(self, ctx, req):
self._attempt_log_stats()
mangledMethod = req.method.replace(".", "_")
logLevel = logging.DEBUG
if mangledMethod in ('Host_getVMList', 'Host_getAllVmStats',
'Host_getStats', 'StorageDomain_getStats',
'VM_getStats', 'Host_fenceNode'):
logLevel = logging.TRACE
self.log.log(logLevel, "Calling '%s' in bridge with %s", req.method,
req.params)
try:
method = getattr(self._bridge, mangledMethod)
except AttributeError:
if req.isNotification():
return
ctx.requestDone(
JsonRpcResponse(None, JsonRpcMethodNotFoundError(), req.id))
return
try:
params = req.params
self._bridge.register_server_address(ctx.address)
if isinstance(req.params, list):
res = method(*params)
else:
res = method(**params)
self._bridge.unregister_server_address()
except JsonRpcError as e:
ctx.requestDone(JsonRpcResponse(None, e, req.id))
except Exception as e:
self.log.exception("Internal server error")
ctx.requestDone(
JsonRpcResponse(None, JsonRpcInternalError(str(e)), req.id))
else:
res = True if res is None else res
self.log.log(logLevel, "Return '%s' in bridge with %s", req.method,
res)
ctx.requestDone(JsonRpcResponse(res, None, req.id))
@traceback(on=log.name)
def serve_requests(self):
while True:
obj = self._workQueue.get()
if obj is None:
break
client, addr, msg = obj
self._parseMessage(client, addr, msg)
def _parseMessage(self, client, addr, msg):
ctx = _JsonRpcServeRequestContext(client, addr)
try:
rawRequests = json.loads(msg)
except:
ctx.addResponse(JsonRpcResponse(None, JsonRpcParseError(), None))
ctx.sendReply()
return
if isinstance(rawRequests, list):
# Empty batch request
if len(rawRequests) == 0:
ctx.addResponse(
JsonRpcResponse(
None,
JsonRpcInvalidRequestError('request batch is empty',
rawRequests), None))
ctx.sendReply()
return
else:
# From this point on we know it's always a list
rawRequests = [rawRequests]
# JSON Parsed handling each request
requests = []
for rawRequest in rawRequests:
try:
req = JsonRpcRequest.fromRawObject(rawRequest)
requests.append(req)
except JsonRpcError as err:
ctx.addResponse(JsonRpcResponse(None, err, None))
except:
ctx.addResponse(
JsonRpcResponse(None, JsonRpcInternalError(), None))
ctx.setRequests(requests)
# No request was built successfully or is only notifications
if ctx.counter == 0:
ctx.sendReply()
for request in requests:
self._runRequest(ctx, request)
def _runRequest(self, ctx, request):
if self._threadFactory is None:
self._serveRequest(ctx, request)
else:
try:
self._threadFactory(partial(self._serveRequest, ctx, request))
except Exception as e:
self.log.exception("could not allocate request thread")
ctx.requestDone(
JsonRpcResponse(None, JsonRpcInternalError(str(e)),
request.id))
def stop(self):
self.log.info("Stopping JsonRPC Server")
self._workQueue.put_nowait(None)
class DBPool(object):
'''''一个数据库连接池'''
def initPool(self, maxActive=5, maxWait=None, init_size=1, db_type="mysql", **config):
'''初始化数据库连接池
'''
log.msg("__init__ Pool..")
self.__freeConns = Queue(maxActive)
self.maxWait = maxWait
self.db_type = db_type
self.config = config
if init_size > maxActive:
init_size = maxActive
for i in range(init_size):
self.free(self._create_conn())
self.nowconn = None
def __del__(self):
log.msg("__del__ Pool..")
self.release()
def release(self):
'''''释放资源，关闭池中的所有连接'''
log.msg("release Pool..")
while self.__freeConns and not self.__freeConns.empty():
con = self.get()
con.release()
self.__freeConns = None
def _create_conn(self):
'''''创建连接 '''
if self.db_type in DBCS:
return MySQLdb.connect(**self.config);
def get(self, timeout=None):
'''''获取一个连接
@param timeout:超时时间
'''
if timeout is None:
timeout = self.maxWait
conn = None
if self.__freeConns.empty():#如果容器是空的，直接创建一个连接
conn = self._create_conn()
else:
conn = self.__freeConns.get(timeout=timeout)
conn.pool = self
return conn
def cursor(self,cursorclass = None):
'''通配接口'''
conn = self.get()
self.nowconn = conn
ucur = UCursor( conn, cursorclass)
return ucur
def commit(self):
'''提交'''
try:
self.nowconn.commit()
except Exception as e:
log.err(e.message)
def rollback(self):
'''事务回滚
'''
try:
self.nowconn.rollback()
except Exception as e:
log.err(e.message)
def free(self, conn):
'''''将一个连接放回池中
@param conn: 连接对象
'''
conn.pool = None
if(self.__freeConns.full()):#如果当前连接池已满，直接关闭连接
conn.release()
return self.__freeConns.put_nowait(conn)
def execSql(self,sqlstr):
'''执行数据库的写操作(插入,修改,删除)
@param sqlstr: str 需要执行的sql语句
'''
try:
conn = self.get(5)
cursor = conn.cursor()
count = cursor.execute(sqlstr)
conn.commit()
cursor.close()
conn.close()
if count>0:
return True
return False
except Exception,err:
log.err(err)
conn.close()
return None#通过放回NONE在远程调用上抛出异常
class SerialAnalyzerThread():
"""
SerialAnalyzerThread: Thread to analyze every lines that are read from
Serial port.
The mechanism is similar to LogCatAnalyzerThread
"""
def __init__(self, logger):
# Analyzer thread stop condition
self.__start_analyze = False
# Messages to trigger
self.__messages_to_trigger = {}
# Message to received
self.__message_to_receive = None
self.__message_received = None
self.__is_message_received = False
# Lock object
self.__lock_message_triggered = threading.RLock()
self.__lock_message_received = threading.RLock()
# Internal buffer
self.__queue = Queue()
self.__analyzer_thread = None
# Logger
self._logger = logger
def stop(self):
"""
Stop the Thread
"""
self.__start_analyze = False
if self.__analyzer_thread is not None:
try:
self.__analyzer_thread.join(5)
except Exception: # pylint: disable=W0703
pass
finally:
del self.__analyzer_thread
self.__analyzer_thread = None
return
def start(self):
"""
Start the thread
"""
self.__analyzer_thread = threading.Thread(target=self.__run)
self.__analyzer_thread.name = "SerialAnalyzerThread"
self.__analyzer_thread.daemon = True
self.__analyzer_thread.start()
def push(self, line):
"""
Method to receive the line that are read from the serial port.
This method is used by the SerialReaderThread
:type line: String
:param line: Line read from the serial port
"""
self.__queue.put_nowait(line)
def __run(self):
"""
Overloaded method that contains the instructions to run
when the thread is started
"""
self.__start_analyze = True
while self.__start_analyze:
while not self.__queue.empty():
try:
line = self.__queue.get_nowait()
if len(line) > 0:
self.__analyse_line(line.rstrip('\r\n'))
except Empty:
pass
time.sleep(1)
def __analyse_line(self, line):
"""
Sub method to analyse every line read by the SerialReaderThread
and store them if they match one of the trigger message
:type line: String
:param line: Line read from the serial port
"""
# For each line to analyze
# for line in lines:
# Check all messages to be triggered
self.__lock_message_triggered.acquire()
for trig_message in self.__messages_to_trigger:
if line.find(trig_message) != -1:
# Message received, store log line
self.__messages_to_trigger[trig_message].append(line)
self.__lock_message_triggered.release()
# Check message to be received
self.__lock_message_received.acquire()
if self.__message_to_receive is not None:
if line.find(self.__message_to_receive) != -1:
self.__message_received.append(line)
self.__is_message_received = True
self.__lock_message_received.release()
def add_trigger_messages(self, messages):
""" Trigger a list of messages
:type messages: Array
:param messages: messages to be triggered
"""
for message in messages:
self.add_trigger_message(message)
def add_trigger_message(self, message):
""" Trigger a message
:type message: string
:param message: message to be triggered
"""
self.__lock_message_triggered.acquire()
self.__messages_to_trigger[message] = list()
self.__lock_message_triggered.release()
def remove_trigger_message(self, message):
""" Remove a triggered message
:type message: string
:param message: message to be removed
"""
if message in self.__messages_to_trigger:
self.__lock_message_triggered.acquire()
del self.__messages_to_trigger[message]
self.__lock_message_triggered.release()
def is_message_received(self, message, timeout):
""" Check if a message is received
:type message: string
:param message: message that we look for
:type timeout: int
:param timeout: time limit where we expect to receive the message
:return: Array of message received, empty array if nothing
:rtype: list
"""
self.__lock_message_received.acquire()
self.__is_message_received = False
self.__message_received = list()
self.__message_to_receive = message
self.__lock_message_received.release()
time_count = 0
while (not self.__is_message_received) and (time_count <= timeout):
time.sleep(1)
time_count += 1
self.__is_message_received = False
self.__message_to_receive = None
return self.__message_received
def get_message_triggered_status(self, message):
""" Get the status of a message triggered
:type message: string
:param message: message triggered
:rtype: list of string
:return: Array of message received, empty array if nothing
"""
if message in self.__messages_to_trigger:
return self.__messages_to_trigger[message]
else:
return None
def reset_trigger_message(self, message):
""" Reset triggered message
:type message: string
:param message: message to be reseted
"""
if message in self.__messages_to_trigger:
self.__lock_message_received.acquire()
self.__messages_to_trigger[message] = list()
self.__lock_message_received.release()
class ThreadPool(object):
def __init__(self,
parse_inst,
save_inst=None,
proxieser=None,
fetch_inst=None,
max_count_parsave=100,
max_count_proxies=100):
self._number_dict = {
TPEnum.TASKS_RUNNING: 0, # the count of tasks which are running
TPEnum.URL_FETCH_NOT:
0, # the count of urls which haven't been fetched
TPEnum.URL_FETCH_SUCC:
0, # the count of urls which have been fetched successfully
TPEnum.URL_FETCH_FAIL:
0, # the count of urls which have been fetched failed
TPEnum.URL_FETCH_COUNT:
0, # the count of urls which appeared in self._queue_fetch
TPEnum.HTM_PARSE_NOT:
0, # the count of urls which haven't been parsed
TPEnum.HTM_PARSE_SUCC:
0, # the count of urls which have been parsed successfully
TPEnum.HTM_PARSE_FAIL:
0, # the count of urls which have been parsed failed
TPEnum.ITEM_SAVE_NOT:
0, # the count of urls which haven't been saved
TPEnum.ITEM_SAVE_SUCC:
0, # the count of urls which have been saved successfully
TPEnum.ITEM_SAVE_FAIL:
0, # the count of urls which have been saved failed
TPEnum.PROXIES_LEFT: 0, # the count of proxies which are avaliable
TPEnum.PROXIES_FAIL:
0, # the count of proxies which banned by website
}
self._lock = threading.Lock() # the lock which self._number_dict needs
self._url_filter = UrlFilter() #URL
self._thread_stop_flag = False # default: False, stop flag of threads
self._fetcher_number = 0 # default: 0, fetcher number in thread pool
self._max_count_parsave = max_count_parsave # maximum count of items which in parse queue or save queue
self._max_count_proxies = max_count_proxies # maximum count of items which in proxies queue
self._queue_fetch = Queue()
self._queue_parse = Queue()
self._queue_saver = Queue()
self._queue_proxies = Queue(
) # {"http": "http://auth@ip:port", "https": "https://auth@ip:port"}
self._inst_fetcher = fetch_inst if fetch_inst else Fetcher()
self._inst_parse = parse_inst
self._inst_saver = save_inst if save_inst else Saver()
self._inst_proxieser = proxieser if proxieser else Proxieser()
self._thread_fetch_list = []
self._thread_parse = None
self._thread_save = None
self._thread_proxieser = None
self._thread_moniter = MoniterThread("MoniterThread", self)
return
## ===================================================================================================================
def start_working(self, root_url, fetcher_num):
self._fetcher_number = fetcher_num
self._thread_stop_flag = False
self.add_a_task(TPEnum.URL_FETCH, (root_url, None, 0))
logging.info(
"ThreadPool starts working: urls_count=%s, fetcher_num=%s",
self.get_number_dict(TPEnum.URL_FETCH_NOT), fetcher_num)
self._thread_fetch_list = [
FetchThread("FetchThread %d" % i, self._inst_fetcher, self)
for i in xrange(fetcher_num)
]
self._thread_parse = ParseThread("ParseThread", self._inst_parse, self)
self._thread_save = SaveThread("SaveThread", self._inst_saver, self)
self._thread_proxieser = ProxiesThread("ProxiesThread",
self._inst_proxieser, self)
if self._thread_moniter:
self._thread_moniter.setDaemon(True)
self._thread_moniter.start()
for thread in self._thread_fetch_list:
thread.setDaemon(True)
thread.start()
if self._thread_parse:
self._thread_parse.setDaemon(True)
self._thread_parse.start()
if self._thread_save:
self._thread_save.setDaemon(True)
self._thread_save.start()
logging.info("ThreadPool starts working: success")
return
def wait_for_finish(self):
self._thread_stop_flag = True
for thread in filter(lambda x: x.is_alive(), self._thread_fetch_list):
thread.join()
if self._thread_parse and self._thread_parse.is_alive():
self._thread_parse.join()
if self._thread_save and self._thread_save.is_alive():
self._thread_save.join()
if self._thread_proxieser and self._thread_proxieser.is_alive():
self._thread_proxieser.join()
if self._thread_monitor and self._thread_monitor.is_alive():
self._thread_monitor.join()
logging.info("ThreadPool has finished")
return None
## ===================================================================================================================
def get_thread_stop_flag(self):
return self._thread_stop_flag
def get_fetcher_number(self):
return self._fetcher_number
def get_number_dict(self, key=None):
return self._number_dict[key] if key else self._number_dict
def update_number_dict(self, key, value):
self._lock.acquire()
self._number_dict[key] += value
self._lock.release()
return None
def is_all_tasks_done(self):
return False if self._number_dict[TPEnum.TASKS_RUNNING] or self._number_dict[TPEnum.URL_FETCH_NOT] or \
self._number_dict[TPEnum.HTM_PARSE_NOT] or self._number_dict[TPEnum.ITEM_SAVE_NOT] else True
## ===================================================================================================================
def add_a_task(self, task_name, task):
if task_name == TPEnum.URL_FETCH and (
(not self._url_filter) or self._url_filter.check_and_add(task[0])):
self._queue_fetch.put_nowait(task)
self.update_number_dict(TPEnum.URL_FETCH_NOT, +1)
self.update_number_dict(TPEnum.COUNTER, +1)
elif task_name == TPEnum.HTM_PARSE and self._thread_parse:
self._queue_parse.put_nowait(task)
self.update_number_dict(TPEnum.HTM_PARSE_NOT, +1)
elif task_name == TPEnum.ITEM_SAVE and self._thread_save:
self._queue_saver.put_nowait(task)
self.update_number_dict(TPEnum.ITEM_SAVE_NOT, +1)
elif (task_name == TPEnum.PROXIES) and self._thread_proxieser:
self._queue_proxies.put_nowait(task)
self.update_number_dict(TPEnum.PROXIES_LEFT, +1)
return None
def get_a_task(self, task_name):
task = None
if task_name == TPEnum.PROXIES:
task = self._queue_proxies.get(block=True, timeout=5)
self.update_number_dict(TPEnum.PROXIES_LEFT, -1)
return task
if task_name == TPEnum.URL_FETCH:
task = self._queue_fetch.get(block=True, timeout=5)
self.update_number_dict(TPEnum.URL_FETCH_NOT, -1)
elif task_name == TPEnum.HTM_PARSE:
task = self._queue_parse.get(block=True, timeout=5)
self.update_number_dict(TPEnum.HTM_PARSE_NOT, -1)
elif task_name == TPEnum.ITEM_SAVE:
task = self._queue_saver.get(block=True, timeout=10)
self.update_number_dict(TPEnum.ITEM_SAVE_NOT, -1)
self.update_number_dict(TPEnum.TASKS_RUNNING, +1)
return task
def finish_a_task(self, task_name):
if task_name == TPEnum.PROXIES:
self._queue_proxies.task_done()
return
if task_name == TPEnum.URL_FETCH:
self._queue_fetch.task_done()
elif task_name == TPEnum.HTM_PARSE:
self._queue_parse.task_done()
elif task_name == TPEnum.ITEM_SAVE:
self._queue_saver.task_done()
self.update_number_dict(TPEnum.TASKS_RUNNING, -1)
return None
class SMSd(object):
'''
Very basic SMS relay
Receive, acknowledge and forward SMS-RP messages
'''
#
# verbosity level: list of log types to display when calling
# self._log(logtype, msg)
DEBUG = ('ERR', 'WNG', 'INF', 'DBG')
#
TRACK_PDU = True
#
# time resolution for consuming the queue for TP msg
QUEUE_TO = 0.1
#
# SMS relay phone number
RP_OA = {'Type': 1, 'NumberingPlan': 1, 'Num': '1234'}
#
# TP settings for sending handcrafted SMS DELIVER to UEs
TP_OA = {'Type': 1, 'NumberingPlan': 1, 'Num': '12341234'}
TP_PID = {'Format': 0, 'Telematic': {'Telematic': 0, 'Protocol': 0}}
TP_DCS = {'Group': 0, 'Charset': 0, 'Class': 0}
#
# timezone for TP_SCTS information (float)
TIMEZONE = 0.0
#
# CorenetServer reference, for checking UE MSISDN and sending MT-SMS
Server = None
def __init__(self):
self._pdu = []
# dict with dicts of ongoing RP transactions indexed by RP ref and
# ongoing TP transactions indexed by TP msg ref
# indexed by UE msisdn
self.Proc = {}
# dict with lists of RP-DATA and TP procedures in error, indexed by UE msisdn
self.Err = {}
#
# set 2 queues to process / forward or inject TP messages within a background thread
self._forward_q = Queue()
self._inject_q = Queue()
self._forwarding = True
self._forward_t = threadit(self.forward)
self._log('INF', 'SMS relay started')
def _log(self, logtype='DBG', msg=''):
# logtype: 'ERR', 'WNG', 'INF', 'DBG'
if logtype in self.DEBUG:
log('[%s] [SMSd] %s' % (logtype, msg))
def stop(self):
if self._forwarding:
self._forwarding = False
self._forward_t.join()
def forward(self):
# consume the queue
while self._forwarding:
try:
tp_msg, num = self._forward_q.get_nowait()
except Empty:
try:
tp_msg, num = self._inject_q.get_nowait()
except Empty:
sleep(self.QUEUE_TO)
else:
self.send_tp(tp_msg, num)
else:
self.process_tp(tp_msg, num)
def init_ue(self, num):
self.Proc[num] = {
'RP': {}, # dict of ongoing RP procedures at the RP layer
'TP': {} # dict of ongoing TP procedures at the TP layer
}
self.Err[num] = {
'RP': [], # list of RP procedures in error
'TP': [] # list of TP procedures in error
}
def process_rp(self, rp_msg, num):
"""process an RP message `rp_msg' sent by a UE with a given MSISDN `num',
returns an RP ACK or ERROR if rp_msg is DATA or SMMA
None if rp_msg is ACK or ERROR
"""
if not isinstance(rp_msg, NAS.SMS_RP):
self._log('WNG', 'process_rp: invalid rp_msg')
return None
#
if self.TRACK_PDU:
self._pdu.append((time(), 'UL', rp_msg))
#
if num not in self.Proc:
self.init_ue(num)
#
if rp_msg._name == 'RP_DATA_MO':
# this will return an RP_ACK or RP_ERR
ret = self._process_rp_data(rp_msg, num)
elif rp_msg._name == 'RP_SMMA':
# this will return an RP_ACK or RP_ERR
ret = self._process_rp_smma(rp_msg, num)
elif rp_msg._name in ('RP_ACK_MO', 'RP_ERROR_MO'):
# check the ref together with num
ret = self._process_rp_ack_err(rp_msg, num)
else:
self._log('WNG', 'process_rp: invalid message %r' % rp_msg)
ret = None
#
if ret and self.TRACK_PDU:
self._pdu.append((time(), 'DL', ret))
return ret
def _process_rp_data(self, rp_msg, num):
ref = rp_msg[2].get_val()
rp_procs = self.Proc[num]['RP']
rp_procs[ref] = (rp_msg, None)
#
# check RP orig / dest address
if rp_msg[3][0].get_val() > 0:
rp_orig = rp_msg[3][1]
self._log(
'WNG',
'process_rp_data: non-empty originator address, %r' % rp_orig)
# invalid mandatory information
del rp_procs[ref]
return NAS.RP_ERROR_MT(val={
'Ref': ind[1],
'RPCause': {
'Value': 96
}
})
#
if rp_msg[4][0].get_val() > 0:
rp_dest = rp_msg[4][1]
if rp_dest['Num'].decode() != self.RP_OA['Num']:
self._log('INF',
'process_rp_data: destination address, %r' % rp_dest)
else:
self._log('WNG', 'process_rp_data: empty destination address')
# invalid mandatory information
del rp_procs[ref]
return NAS.RP_ERROR_MT(val={
'Ref': ind[1],
'RPCause': {
'Value': 96
}
})
#
if not isinstance(rp_msg[5][1], NAS.SMS_TP):
self._log('WNG',
'process_rp_data: invalid TP data, %r' % tp_msg[5])
# invalid mandatory information
del rp_procs[ref]
return NAS.RP_ERROR_MT(val={
'Ref': ind[1],
'RPCause': {
'Value': 96
}
})
#
# process TP in the background thread
self._insert_tp(rp_msg[5][1], num)
# acknowledge RP
rp_ack = NAS.RP_ACK_MT(val={'Ref': ref})
del rp_procs[ref]
return rp_ack
def _process_rp_smma(self, rp_msg, num):
ref = rp_msg[2].get_val()
self._log('INF',
'process_rp_smma: procedure ref (%s, %i)' % (num, ref))
return NAS.RP_ACK_MT(val={'Ref': ref})
def _process_rp_ack_err(self, rp_msg, num):
rp_msg_name = rp_msg._name[:-3].replace('_', '-')
ref = rp_msg[2].get_val()
rp_procs = self.Proc[num]['RP']
if ref in rp_procs:
rp_req, tp_ref = rp_procs[ref]
rp_ud = rp_msg['RPUserData']
if not rp_ud.get_trans() and isinstance(rp_ud[2], NAS.SMS_TP):
# SMS_DELIVER_REPORT_RP_ACK/ERROR provided
if rp_msg._name == 'RP_ACK_MO':
# TP status 0: Short message transaction completed - Short message received by the SME
stat = 0
else:
# TP status 64: Permanent error, SC is not making any more transfer attempts - Remote procedure error
stat = 64
self._report_status(rp_req, tp_ref, stat)
# TODO: check if it requires an RP-ACK back
# delete the RP procedure
del rp_procs[ref]
if rp_msg_name == 'RP-ACK':
self._log(
'DBG',
'process_rp_ack_err: procedure ref (%s, %i) completed' %
(num, ref))
else:
self.Err[num]['RP'].append(rp_req)
self._log('INF', 'process_rp_ack_err: procedure ref (%s, %i) in error with cause %r'\
% (num, ref, rp_msg[3][1]))
else:
self._log(
'INF', 'process_rp_ack_err: procedure ref (%s, %i) unknown' %
(num, ref))
return None
def _report_status(self, rp_req, tp_ref, stat=64):
# when a downlink RP-DATA fails within CorenetServer (-> discard_rp())
# or receiving an RP-ACK/ERROR-MO with TP data (SMS-DELIVER-REPORT-RP-ACK/ERROR)
# we need to start an SMS-STATUS-REPORT toward to original sender
# 1) reassociate to the SMS SUBMIT of the initial sender
try:
tp_oa = rp_req[5][1]['TP_OA']['Num'].decode()
except:
self._log(
'WNG',
'report_status: unable to retrieve the TP originating address')
else:
if tp_oa in self.Proc:
tp_procs = self.Proc[tp_oa]['TP']
if tp_ref in tp_procs:
tp_req, atime = self.Proc[tp_oa]['TP'][tp_ref]
# 2) send a status report to the initial sender and delete the TP transaction
del self.Proc[tp_oa]['TP'][tp_ref]
tp_stat = self._create_tp_stat_rep(tp_req, tp_oa, atime,
stat)
self._inject_tp(tp_stat, tp_oa)
self._log(
'DBG', 'report_status: delete TP procedure (%s, %i)' %
(tp_oa, tp_ref))
return
# no status report was requested, hence we just pass our way
self._log(
'DBG',
'report_status: no SMS SUBMIT requiring status report for %s' %
tp_oa)
def _insert_tp(self, tp_msg, num):
"""put the tp_msg within the forwarding queue,
and let the forwarding thread take care of it
"""
try:
self._forward_q.put_nowait((tp_msg, num))
except Full as err:
self._log('ERR', 'insert_tp: TP forwarding queue is full (%i), deleting it, %s'\
% (self._forward_q.qsize(), err))
self._forward_q = Queue()
def process_tp(self, tp_msg, num):
"""process a TP message `tp_msg' sent by a UE with a given MSISDN `num'
"""
if tp_msg._name == 'SMS_SUBMIT':
# should forward TP user data in an SMS DELIVER to the TP dest
self._process_tp_submit(tp_msg, num)
elif tp_msg._name == 'SMS_COMMAND':
# correspond to an MS invoking an operation within the SMS-Center
self._process_tp_cmd(tp_msg, num)
else:
# SMS_DELIVER_REPORT_RP_ACK and SMS_DELIVER_REPORT_RP_ERROR
# are processed within _process_rp_ack_err()
self._log('WNG', 'process_tp: invalid message %r' % tp_msg)
return None
def _process_tp_submit(self, tp_msg, num):
atime = localtime()
if tp_msg[0].get_val():
# the sender UE requests a status report as a result of the SMS DELIVER process
tp_ref = tp_msg[6].get_val()
else:
tp_ref = None
#
# check TP dest addr
num_dest = tp_msg[7]['Num'].decode()
if num_dest in self.Server.MSISDN:
imsi = self.Server.MSISDN[num_dest]
else:
# unknown msisdn
# status 65: incompatible dest
self._log('INF',
'process_tp_submit: destination unknown, %s' % num_dest)
if tp_ref:
tp_stat = self._create_tp_stat_rep(tp_msg, num, atime, stat=65)
self.send_tp(tp_stat, num)
return
#
if imsi in self.Server.UE:
ued = self.Server.UE[imsi]
else:
# UE never attached
# status 34: no response from SME
self._log('INF',
'process_tp_submit: destination offline, %s' % num_dest)
if tp_ref:
tp_stat = self._create_tp_stat_rep(tp_msg, num, atime, stat=34)
self.send_tp(tp_stat, num)
return
#
# build tp_deliver
if tp_ref is not None:
# keep track of the SMS SUBMIT for further status report
self.Proc[num]['TP'][tp_ref] = (tp_msg, atime)
tp_del = self._create_tp_deliver(tp_msg, num, atime)
self.send_tp(tp_del, num_dest, report_ref=tp_ref)
def _process_tp_cmd(self, tp_msg, num):
self._log('INF', 'process_tp_cmd: CDL %i, CD 0x%s'\
% (tp_msg['TP_CDL'].get_val(),
hexlify(tp_msg['TP_CD'].get_val()).decode('ascii')))
atime = localtime()
if tp_msg[0].get_val():
# the sender UE requests a status report of the result of the SMS COMMAND process
tp_stat = self._create_tp_stat_rep(tp_msg, num, atime, stat=0)
self.send_tp(tp_stat, num_dest)
def _create_tp_stat_rep(self, tp_msg, num, atime, stat=64):
tp_srq = 1 if isinstance(tp_msg, NAS.SMS_COMMAND) else 0
tp_mr = tp_msg[6].get_val()
tp_ra = {'Type': 1, 'NumberingPlan': 1, 'Num': num}
tp_scts = (atime, self.TIMEZONE)
if 0 <= stat <= 255:
tp_stat = stat
else:
tp_stat = 64
#
tp_stat = NAS.SMS_STATUS_REPORT(
val={
'TP_SRQ': tp_srq,
'TP_MR': tp_mr,
'TP_RA': tp_ra,
'TP_SCTS': tp_scts,
'TP_ST': stat
})
tp_stat['TP_PI'].set_trans(True)
self._set_tp_scts(tp_stat['TP_DT'])
return tp_stat
def _create_tp_deliver(self, tp_msg, num, atime):
tp_sri = tp_msg[0].get_val()
tp_udhi = tp_msg[1].get_val()
tp_oa = {'Type': 1, 'NumberingPlan': 1, 'Num': num}
tp_pid = tp_msg[8].get_val()
tp_dcs = tp_msg[9].get_val()
tp_msg_ud = tp_msg['TP_UD']
if tp_udhi:
tp_udh = tp_msg_ud[1][1].get_val()
else:
tp_udh = None
tp_ud = tp_msg_ud[2].get_val()
#
tp_del = NAS.SMS_DELIVER(
val={
'TP_SRI': tp_sri,
'TP_UDHI': tp_udhi,
'TP_OA': tp_oa,
'TP_PID': tp_pid,
'TP_DCS': tp_dcs,
'TP_UD': {
'UDH': {
'UDH': tp_udh
},
'UD': tp_ud
}
})
self._set_tp_scts(tp_del['TP_SCTS'])
return tp_del
def _set_tp_scts(self, tp_scts):
if tp_scts.get_len() == 7:
T = localtime()
tp_scts.encode(localtime(), tz=self.TIMEZONE)
else:
self._log('WNG', 'set_tp_scts: custom timestamping unhandled')
def _inject_tp(self, tp_msg, num):
"""put the tp_msg within the injection queue,
and let the forwarding thread take care of it
"""
try:
self._inject_q.put_nowait((tp_msg, num))
except Full as err:
self._log('ERR', 'inject_tp: TP injection queue is full (%i), deleting it, %s'\
% (self._inject_q.qsize(), err))
self._inject_q = Queue()
def _get_new_rp_ref(self, num):
if num not in self.Proc:
self.init_ue(num)
return 0
else:
for i in range(0, 257):
if i not in self.Proc[num]['RP']:
break
if i == 256:
# no RP ref available...
self._log('ERR', 'get_new_rp_ref: no RP ref available, clearing all procedure for %s'\
% num)
self.Proc[num]['RP'].clear()
self.Proc[num]['TP'].clear()
return 0
else:
return i
def send_tp(self, tp_msg, num, tp_ref=None):
"""send the SMS TP message `tp_msg' to UE msisdn `num'
associate the TP transaction ref `tp_ref' to the RP transaction
"""
# wrap the TP msg into an RP DATA msg
ref = self._get_new_rp_ref(num)
rp_msg = NAS.RP_DATA_MT(val={
'Ref': ref,
'RPOriginatorAddress': self.RP_OA
})
rp_msg.set_tpdu(tp_msg)
self.Proc[num]['RP'][ref] = (rp_msg, tp_ref)
self._log('DBG', 'sending TP msg with RP ref %i' % ref)
self.send_rp(rp_msg, num)
def send_rp(self, rp_msg, num):
if self.TRACK_PDU:
self._pdu.append((time(), 'DL', rp_msg))
self.Server.send_smsrp(num, rp_msg)
def discard_rp(self, rp_msg, num):
"""discard an RP message `rp_msg' sent to UE with msisdn `num'
"""
if num not in self.Proc:
return
rp_procs = self.Proc[num]['RP']
ref = rp_msg[2].get_val()
if ref not in rp_procs:
return
rp_req, tp_ref = rp_procs[ref]
if tp_ref is not None:
# downlink RP-DATA failed within corenet, status report required
# TP status 97 : Temporary error, SC is not making any more transfer attempts - SME busy
self._report_status(rp_req, tp_ref, 97)
# delete the RP transaction
del rp_procs[ref]
self._log('INF',
'discard_rp: delete RP procedure (%s, %i)' % (num, ref))
#--------------------------------------------------------------------------#
# custom methods to send TP messages from the SMSd to UEs
#--------------------------------------------------------------------------#
def send_text(self, text, num):
"""sends a given text (ascii string, that will be converted to SMS 7bit)
to a given phone number
"""
tp_dcs = self.TP_DCS
self.TP_DCS = {'Group': 0, 'Charset': 0, 'Class': 0} # GSM 7bit
self.send_tpud(text, num=num)
self.TP_DCS = tp_dcs
def send_tpud(self, ud, num):
"""sends a given user-data (directly the data buffer, or a tuple with
options and the data buffer) to a given phone number
each option must be a 2-tuple (Tag, Value) were Tag is an uint8 and Value
is a buffer
"""
# TODO: implement SMS UD fragmentation into several tp_msg
try:
tp_msg = NAS.SMS_DELIVER(
val={
'TP_MMS': 1, # no more messages
'TP_OA': self.TP_OA,
'TP_PID': self.TP_PID,
'TP_DCS': self.TP_DCS
})
self._set_tp_scts(tp_msg['TP_SCTS'])
if isinstance(ud, (list, tuple)):
if len(ud) > 1:
# UD header IEs
tp_msg['TP_UDHI'].set_val(1)
tp_msg['TP_UD']['UDH']['UDH'].set_val([{
'T': udh[0],
'V': udh[1]
} for udh in ud[:-1]])
data = ud[-1]
else:
data = ud
tp_msg['TP_UD']['UD'].set_val(data)
except:
self._log('WNG', 'invalid TP UD')
else:
self._inject_tp(tp_msg, num)
class Connection(object):
in_buffer_size = 4096
out_buffer_size = 4096
cql_version = None
protocol_version = 2
keyspace = None
compression = True
compressor = None
decompressor = None
ssl_options = None
last_error = None
in_flight = 0
is_defunct = False
is_closed = False
lock = None
def __init__(self,
host='127.0.0.1',
port=9042,
credentials=None,
ssl_options=None,
sockopts=None,
compression=True,
cql_version=None,
protocol_version=2):
self.host = host
self.port = port
self.credentials = credentials
self.ssl_options = ssl_options
self.sockopts = sockopts
self.compression = compression
self.cql_version = cql_version
self.protocol_version = protocol_version
self._id_queue = Queue(MAX_STREAM_PER_CONNECTION)
for i in range(MAX_STREAM_PER_CONNECTION):
self._id_queue.put_nowait(i)
self.lock = RLock()
def close(self):
raise NotImplementedError()
def defunct(self, exc):
raise NotImplementedError()
def send_msg(self, msg, cb):
raise NotImplementedError()
def wait_for_response(self, msg, **kwargs):
raise NotImplementedError()
def wait_for_responses(self, *msgs, **kwargs):
raise NotImplementedError()
def register_watcher(self, event_type, callback):
raise NotImplementedError()
def register_watchers(self, type_callback_dict):
raise NotImplementedError()
@defunct_on_error
def process_msg(self, msg, body_len):
version, flags, stream_id, opcode = map(int8_unpack, msg[:4])
if stream_id < 0:
callback = None
else:
callback = self._callbacks.pop(stream_id, None)
self._id_queue.put_nowait(stream_id)
body = None
try:
# check that the protocol version is supported
given_version = version & PROTOCOL_VERSION_MASK
if given_version != self.protocol_version:
msg = "Server protocol version (%d) does not match the specified driver protocol version (%d). " +\
"Consider setting Cluster.protocol_version to %d."
raise ProtocolError(
msg %
(given_version, self.protocol_version, given_version))
# check that the header direction is correct
if version & HEADER_DIRECTION_MASK != HEADER_DIRECTION_TO_CLIENT:
raise ProtocolError(
"Header direction in response is incorrect; opcode %04x, stream id %r"
% (opcode, stream_id))
if body_len > 0:
body = msg[8:]
elif body_len == 0:
body = ""
else:
raise ProtocolError("Got negative body length: %r" % body_len)
response = decode_response(stream_id, flags, opcode, body,
self.decompressor)
except Exception as exc:
log.exception(
"Error decoding response from Cassandra. "
"opcode: %04x; message contents: %r", opcode, body)
if callback is not None:
callback(exc)
self.defunct(exc)
return
try:
if stream_id < 0:
self.handle_pushed(response)
elif callback is not None:
callback(response)
except Exception:
log.exception("Callback handler errored, ignoring:")
@defunct_on_error
def _send_options_message(self):
if self.cql_version is None and (not self.compression or
not locally_supported_compressions):
log.debug(
"Not sending options message for new connection(%s) to %s "
"because compression is disabled and a cql version was not "
"specified", id(self), self.host)
self._compressor = None
self.cql_version = DEFAULT_CQL_VERSION
self._send_startup_message()
else:
log.debug(
"Sending initial options message for new connection (%s) to %s",
id(self), self.host)
self.send_msg(OptionsMessage(), self._handle_options_response)
@defunct_on_error
def _handle_options_response(self, options_response):
if self.is_defunct:
return
if not isinstance(options_response, SupportedMessage):
if isinstance(options_response, ConnectionException):
raise options_response
else:
log.error("Did not get expected SupportedMessage response; " \
"instead, got: %s", options_response)
raise ConnectionException("Did not get expected SupportedMessage " \
"response; instead, got: %s" \
% (options_response,))
log.debug("Received options response on new connection (%s) from %s",
id(self), self.host)
supported_cql_versions = options_response.cql_versions
remote_supported_compressions = options_response.options['COMPRESSION']
if self.cql_version:
if self.cql_version not in supported_cql_versions:
raise ProtocolError(
"cql_version %r is not supported by remote (w/ native "
"protocol). Supported versions: %r" %
(self.cql_version, supported_cql_versions))
else:
self.cql_version = supported_cql_versions[0]
self._compressor = None
compression_type = None
if self.compression:
overlap = (set(locally_supported_compressions.keys())
& set(remote_supported_compressions))
if len(overlap) == 0:
log.debug(
"No available compression types supported on both ends."
" locally supported: %r. remotely supported: %r",
locally_supported_compressions.keys(),
remote_supported_compressions)
else:
compression_type = iter(overlap).next() # choose any
# set the decompressor here, but set the compressor only after
# a successful Ready message
self._compressor, self.decompressor = \
locally_supported_compressions[compression_type]
self._send_startup_message(compression_type)
@defunct_on_error
def _send_startup_message(self, compression=None):
opts = {}
if compression:
opts['COMPRESSION'] = compression
sm = StartupMessage(cqlversion=self.cql_version, options=opts)
self.send_msg(sm, cb=self._handle_startup_response)
@defunct_on_error
def _handle_startup_response(self,
startup_response,
did_authenticate=False):
if self.is_defunct:
return
if isinstance(startup_response, ReadyMessage):
log.debug("Got ReadyMessage on new connection (%s) from %s",
id(self), self.host)
if self._compressor:
self.compressor = self._compressor
self.connected_event.set()
elif isinstance(startup_response, AuthenticateMessage):
log.debug("Got AuthenticateMessage on new connection (%s) from %s",
id(self), self.host)
if self.credentials is None:
raise AuthenticationFailed(
'Remote end requires authentication.')
self.authenticator = startup_response.authenticator
cm = CredentialsMessage(creds=self.credentials)
callback = partial(self._handle_startup_response,
did_authenticate=True)
self.send_msg(cm, cb=callback)
elif isinstance(startup_response, ErrorMessage):
log.debug(
"Received ErrorMessage on new connection (%s) from %s: %s",
id(self), self.host, startup_response.summary_msg())
if did_authenticate:
raise AuthenticationFailed(
"Failed to authenticate to %s: %s" %
(self.host, startup_response.summary_msg()))
else:
raise ConnectionException(
"Failed to initialize new connection to %s: %s" %
(self.host, startup_response.summary_msg()))
else:
msg = "Unexpected response during Connection setup: %r"
log.error(msg, startup_response)
raise ProtocolError(msg % (startup_response, ))
def set_keyspace_blocking(self, keyspace):
if not keyspace or keyspace == self.keyspace:
return
query = QueryMessage(query='USE "%s"' % (keyspace, ),
consistency_level=ConsistencyLevel.ONE)
try:
result = self.wait_for_response(query)
except InvalidRequestException as ire:
# the keyspace probably doesn't exist
raise ire.to_exception()
except Exception as exc:
conn_exc = ConnectionException(
"Problem while setting keyspace: %r" % (exc, ), self.host)
self.defunct(conn_exc)
raise conn_exc
if isinstance(result, ResultMessage):
self.keyspace = keyspace
else:
conn_exc = ConnectionException(
"Problem while setting keyspace: %r" % (result, ), self.host)
self.defunct(conn_exc)
raise conn_exc
def set_keyspace_async(self, keyspace, callback):
"""
Use this in order to avoid deadlocking the event loop thread.
When the operation completes, `callback` will be called with
two arguments: this connection and an Exception if an error
occurred, otherwise :const:`None`.
"""
if not keyspace or keyspace == self.keyspace:
callback(self, None)
return
query = QueryMessage(query='USE "%s"' % (keyspace, ),
consistency_level=ConsistencyLevel.ONE)
def process_result(result):
if isinstance(result, ResultMessage):
self.keyspace = keyspace
callback(self, None)
elif isinstance(result, InvalidRequestException):
callback(self, result.to_exception())
else:
callback(
self,
self.defunct(
ConnectionException(
"Problem while setting keyspace: %r" % (result, ),
self.host)))
self.send_msg(query, process_result, wait_for_id=True)
def __str__(self):
status = ""
if self.is_defunct:
status = " (defunct)"
elif self.is_closed:
status = " (closed)"
return "<%s(%r) %s:%d%s>" % (self.__class__.__name__, id(self),
self.host, self.port, status)
__repr__ = __str__
class APRSUploader(object):
"""
Queued APRS Telemetry Uploader class
This performs uploads to an APRS-IS server.
Incoming telemetry packets are fed into queue, which is checked regularly.
At a regular interval, the most recent telemetry packet is extracted, and converted to an
APRS object format, and then uploaded into APRS-IS.
If an upload attempt times out, the packet is discarded.
If the queue fills up (probably indicating no network connection, and a fast packet downlink rate),
it is immediately emptied, to avoid upload of out-of-date packets.
Note that this uploader object is intended to handle telemetry from multiple sondes
"""
# We require the following fields to be present in the incoming telemetry dictionary data
REQUIRED_FIELDS = [
"frame",
"id",
"datetime",
"lat",
"lon",
"alt",
"temp",
"type",
"freq",
"freq_float",
"datetime_dt",
]
def __init__(
self,
aprs_callsign="N0CALL",
aprs_passcode="00000",
object_name_override=None,
object_comment="RadioSonde",
position_report=False,
aprsis_host="rotate.aprs2.net",
aprsis_port=14580,
aprsis_reconnect=300,
station_beacon=False,
station_beacon_rate=30,
station_beacon_position=(0.0, 0.0, 0.0),
station_beacon_comment="radiosonde_auto_rx SondeGate v<version>",
station_beacon_icon="/r",
synchronous_upload_time=30,
callsign_validity_threshold=5,
upload_queue_size=16,
upload_timeout=5,
inhibit=False,
):
""" Initialise an APRS Uploader object.
Args:
aprs_callsign (str): Callsign of the uploader, used when logging into APRS-IS.
aprs_passcode (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated,
is used to plot the listener's position on the Habitat map, both when this class is initialised, and
when a new sonde ID is observed.
object_name_override (str): Override the object name in the uploaded sentence with this value.
WARNING: This will horribly break the aprs.fi map if multiple sondes are uploaded simultaneously under the same callsign.
USE WITH CAUTION!!!
object_comment (str): A comment to go with the object. Various fields will be replaced with telmetry data.
position_report (bool): If True, upload positions as APRS position reports, otherwise, upload as an Object.
aprsis_host (str): APRS-IS Server to upload packets to.
aprsis_port (int): APRS-IS TCP port number.
aprsis_reconnect (int): Reconnect to the APRS-IS server at least every X minutes. Reconnections will occur when telemetry needs to be sent.
station_beacon (bool): Enable beaconing of station position.
station_beacon_rate (int): Time delay between beacon uploads (minutes)
station_beacon_position (tuple): (lat, lon, alt), in decimal degrees, of the station position.
station_beacon_comment (str): Comment field for the station beacon. <version> will be replaced with the current auto_rx version.
station_beacon_icon (str): The APRS icon to be used, as the two characters (symbol table, symbol index), as per http://www.aprs.org/symbols.html
synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
and also acts as decimation on the number of sentences uploaded to APRS-IS.
callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
upload_queue_size (int): Maximum number of sentences to keep in the upload queue. If the queue is filled,
it will be emptied (discarding the queue contents).
upload_timeout (int): Timeout (Seconds) when performing uploads to APRS-IS. Default: 10 seconds.
inhibit (bool): Inhibit all uploads. Mainly intended for debugging.
"""
self.aprs_callsign = aprs_callsign
self.aprs_passcode = aprs_passcode
self.object_comment = object_comment
self.position_report = position_report
self.aprsis_host = aprsis_host
self.aprsis_port = aprsis_port
self.aprsis_reconnect = aprsis_reconnect
self.upload_timeout = upload_timeout
self.upload_queue_size = upload_queue_size
self.synchronous_upload_time = synchronous_upload_time
self.callsign_validity_threshold = callsign_validity_threshold
self.inhibit = inhibit
self.station_beacon = {
"enabled": station_beacon,
"position": station_beacon_position,
"rate": station_beacon_rate,
"comment": station_beacon_comment,
"icon": station_beacon_icon,
}
if object_name_override is None:
self.object_name_override = "<id>"
else:
self.object_name_override = object_name_override
self.log_info("Using APRS Object Name Override: %s" %
self.object_name_override)
# Our two Queues - one to hold sentences to be upload, the other to temporarily hold
# input telemetry dictionaries before they are converted and processed.
self.aprs_upload_queue = Queue(upload_queue_size)
self.input_queue = Queue()
# Dictionary where we store sorted telemetry data for upload when required.
# Elements will be named after payload IDs, and will contain:
# 'count' (int): Number of times this callsign has been observed. Uploads will only occur when
# this number rises above callsign_validity_threshold.
# 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires,
# this queue will be dumped, and the most recent telemetry uploaded.
self.observed_payloads = {}
# Record of when we last uploaded a user station position to Habitat.
self.last_user_position_upload = 0
# APRS-IS Socket Object
self.aprsis_socket = None
self.aprsis_lastconnect = 0
self.aprsis_upload_lock = Lock()
# Attempt to connect to the APRS-IS server.
# If this fails, we will attempt to re-connect when a packet needs to be uploaded.
self.connect()
# Start the uploader thread.
self.upload_thread_running = True
self.upload_thread = Thread(target=self.aprs_upload_thread)
self.upload_thread.start()
# Start the input queue processing thread.
self.input_processing_running = True
self.input_thread = Thread(target=self.process_queue)
self.input_thread.start()
self.timer_thread_running = True
self.timer_thread = Thread(target=self.upload_timer)
self.timer_thread.start()
self.log_info("APRS Uploader Started.")
def connect(self):
""" Connect to an APRS-IS Server """
# create socket & connect to server
self.aprsis_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.aprsis_socket.settimeout(self.upload_timeout)
try:
self.aprsis_socket.connect((self.aprsis_host, self.aprsis_port))
# Send logon string
# _logon = 'user %s pass %s vers VK5QI-AutoRX filter b/%s \r\n' % (self.aprs_callsign, self.aprs_passcode, self.aprs_callsign)
_logon = "user %s pass %s vers VK5QI-AutoRX\r\n" % (
self.aprs_callsign,
self.aprs_passcode,
)
self.log_debug("Logging in: %s" % _logon)
self.aprsis_socket.sendall(_logon.encode("ascii"))
# Set packet filters to limit inbound bandwidth.
_filter = "#filter p/ZZ\r\n"
self.log_debug("Setting Filter: %s" % _filter)
self.aprsis_socket.sendall(_filter.encode("ascii"))
_filter = "#filter -t/po\r\n"
self.log_debug("Setting Filter: %s" % _filter)
self.aprsis_socket.sendall(_filter.encode("ascii"))
# Wait for login to complete.
time.sleep(1)
# Check response
_resp = self.aprsis_socket.recv(1024)
try:
_resp = _resp.decode("ascii").strip()
except:
print(_resp)
if _resp[0] != "#":
raise IOError("Invalid response from APRS-IS Server: %s" %
_resp)
else:
self.log_debug("Server Logon Response: %s" % str(_resp))
self.log_info("Connected to APRS-IS server %s:%d" %
(self.aprsis_host, self.aprsis_port))
self.aprsis_lastconnect = time.time()
return True
except Exception as e:
self.log_error("Connection to APRS-IS Failed - %s" % str(e))
self.aprsis_socket = None
return False
def flush_rx(self):
""" Flush the APRS-IS RX buffer """
try:
_start = time.time()
_data = self.aprsis_socket.recv(32768)
_dur = time.time() - _start
self.log_debug("Incoming data from APRS-IS: %s" % (_data.decode()))
except:
# Ignore any exceptions from attempting to read the buffer.
pass
def aprsis_upload(self, source, packet, igate=False, retries=5):
""" Upload a packet to APRS-IS
Args:
source (str): Callsign of the packet source.
packet (str): APRS packet to upload.
igate (boolean): If True, iGate the packet into APRS-IS
(i.e. use the original source call, but add SONDEGATE and our callsign to the path.)
retries (int): Number of times to retry uploading.
"""
# If we are inhibited, just return immediately.
if self.inhibit:
self.log_info("Upload Inhibited: %s" % packet)
return True
self.aprsis_upload_lock.acquire()
# If we have not connected in a long time, reset the APRS-IS connection.
if (time.time() - self.aprsis_lastconnect) > (self.aprsis_reconnect *
60):
self.disconnect()
time.sleep(1)
self.connect()
# Generate APRS packet
if igate:
# If we are emulating an IGATE, then we need to add in a path, a q-construct, and our own callsign.
# We have the TOCALL field 'APRARX' allocated by Bob WB4APR, so we can now use this to indicate
# that these packets have arrived via radiosonde_auto_rx!
_packet = "%s>APRARX,SONDEGATE,TCPIP,qAR,%s:%s\r\n" % (
source,
self.aprs_callsign,
packet,
)
else:
# Otherwise, we are probably just placing an object, usually sourced by our own callsign
_packet = "%s>APRS:%s\r\n" % (source, packet)
_attempts = 1
while _attempts < retries:
try:
# Immediately throw exception if we're not connected.
# This will trigger a reconnect.
if self.aprsis_socket is None:
raise IOError("Socket not connected.")
# Attempt to send the packet.
# This will timeout if the socket is locked up.
self.aprsis_socket.sendall(_packet.encode("ascii"))
# If OK, return.
self.log_info("Uploaded to APRS-IS: %s" % str(_packet).strip())
self.aprsis_upload_lock.release()
return True
except Exception as e:
# If something broke, forcibly shutdown the socket, then reconnect.
self.log_error("Upload Error: %s" % str(e))
self.log_info("Attempting to reconnect...")
self.disconnect()
time.sleep(1)
self.connect()
_attempts += 1
# If we end up here, something has really broken.
self.aprsis_upload_lock.release()
return False
def disconnect(self):
""" Close APRS-IS connection """
try:
self.aprsis_socket.shutdown(0)
except Exception as e:
self.log_debug("Socket shutdown failed - %s" % str(e))
try:
self.aprsis_socket.close()
except Exception as e:
self.log_debug("Socket close failed - %s" % str(e))
def beacon_station_position(self):
""" Send a station position beacon into APRS-IS """
if self.station_beacon["enabled"]:
if (self.station_beacon["position"][0]
== 0.0) and (self.station_beacon["position"][1] == 0.0):
self.log_error(
"Station position is 0,0, not uploading position beacon.")
self.last_user_position_upload = time.time()
return
# Generate the station position packet
# Note - this is now generated as an APRS position report, for radiosondy.info compatability.
_packet = generate_station_object(
self.aprs_callsign,
self.station_beacon["position"][0],
self.station_beacon["position"][1],
self.station_beacon["comment"],
self.station_beacon["icon"],
position_report=True,
)
# Send the packet as an iGated packet.
self.aprsis_upload(self.aprs_callsign, _packet, igate=True)
self.last_user_position_upload = time.time()
def update_station_position(self, lat, lon, alt):
""" Update the internal station position record. Used when determining the station position by GPSD """
self.station_beacon["position"] = (lat, lon, alt)
def aprs_upload_thread(self):
""" Handle uploading of packets to APRS """
self.log_debug("Started APRS Uploader Thread.")
while self.upload_thread_running:
if self.aprs_upload_queue.qsize() > 0:
# If the queue is completely full, jump to the most recent telemetry sentence.
if self.aprs_upload_queue.qsize() == self.upload_queue_size:
while not self.aprs_upload_queue.empty():
_telem = self.aprs_upload_queue.get()
self.log_warning(
"Uploader queue was full - possible connectivity issue."
)
else:
# Otherwise, get the first item in the queue.
_telem = self.aprs_upload_queue.get()
# Convert to a packet.
try:
(_packet, _call) = telemetry_to_aprs_position(
_telem,
object_name=self.object_name_override,
aprs_comment=self.object_comment,
position_report=self.position_report,
)
except Exception as e:
self.log_error(
"Error converting telemetry to APRS packet - %s" %
str(e))
_packet = None
# Attempt to upload it.
if _packet is not None:
# If we are uploading position reports, the source call is the generated callsign
# usually based on the sonde serial number, and we iGate the position report.
# Otherwise, we upload APRS Objects, sourced by our own callsign, but still iGated via us.
if self.position_report:
self.aprsis_upload(_call, _packet, igate=True)
else:
self.aprsis_upload(self.aprs_callsign,
_packet,
igate=True)
else:
# Wait for a short time before checking the queue again.
time.sleep(0.1)
self.log_debug("Stopped APRS Uploader Thread.")
def upload_timer(self):
""" Add packets to the aprs upload queue if it is time for us to upload. """
while self.timer_thread_running:
if int(time.time()) % self.synchronous_upload_time == 0:
# Time to upload!
for _id in self.observed_payloads.keys():
# If no data, continue...
if self.observed_payloads[_id]["data"].empty():
continue
else:
# Otherwise, dump the queue and keep the latest telemetry.
while not self.observed_payloads[_id]["data"].empty():
_telem = self.observed_payloads[_id]["data"].get()
# Attept to add it to the habitat uploader queue.
try:
self.aprs_upload_queue.put_nowait(_telem)
except Exception as e:
self.log_error(
"Error adding sentence to queue: %s" % str(e))
# Sleep a second so we don't hit the synchronous upload time again.
time.sleep(1)
# Flush APRS-IS RX buffer
self.flush_rx()
else:
# Not yet time to upload, wait for a bit.
time.sleep(0.1)
def process_queue(self):
""" Process packets from the input queue.
This thread handles packets from the input queue (provided by the decoders)
Packets are sorted by ID, and a dictionary entry is created.
"""
while self.input_processing_running:
# Process everything in the queue.
while self.input_queue.qsize() > 0:
# Grab latest telem dictionary.
_telem = self.input_queue.get_nowait()
_id = _telem["id"]
if _id not in self.observed_payloads:
# We haven't seen this ID before, so create a new dictionary entry for it.
self.observed_payloads[_id] = {"count": 1, "data": Queue()}
self.log_debug(
"New Payload %s. Not observed enough to allow upload."
% _id)
# However, we don't yet add anything to the queue for this payload...
else:
# We have seen this payload before!
# Increment the 'seen' counter.
self.observed_payloads[_id]["count"] += 1
# If we have seen this particular ID enough times, add the data to the ID's queue.
if (self.observed_payloads[_id]["count"] >=
self.callsign_validity_threshold):
# Add the telemetry to the queue
self.observed_payloads[_id]["data"].put(_telem)
else:
self.log_debug(
"Payload ID %s not observed enough to allow upload."
% _id)
if (time.time() - self.last_user_position_upload
) > self.station_beacon["rate"] * 60:
if self.aprsis_socket != None:
self.beacon_station_position()
time.sleep(0.1)
def add(self, telemetry):
""" Add a dictionary of telemetry to the input queue.
Args:
telemetry (dict): Telemetry dictionary to add to the input queue.
"""
# Discard any telemetry which is indicated to be encrypted.
if "encrypted" in telemetry:
if telemetry["encrypted"] == True:
return
# Check the telemetry dictionary contains the required fields.
for _field in self.REQUIRED_FIELDS:
if _field not in telemetry:
self.log_error("JSON object missing required field %s" %
_field)
return
# Add it to the queue if we are running.
if self.input_processing_running:
self.input_queue.put(telemetry)
else:
self.log_error("Processing not running, discarding.")
def close(self):
""" Shutdown uploader and processing threads. """
self.log_debug("Waiting for threads to close...")
self.input_processing_running = False
self.timer_thread_running = False
self.upload_thread_running = False
self.disconnect()
# Wait for all threads to close.
if self.upload_thread is not None:
self.upload_thread.join()
if self.timer_thread is not None:
self.timer_thread.join()
if self.input_thread is not None:
self.input_thread.join()
def log_debug(self, line):
""" Helper function to log a debug message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.debug("APRS-IS - %s" % line)
def log_info(self, line):
""" Helper function to log an informational message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.info("APRS-IS - %s" % line)
def log_error(self, line):
""" Helper function to log an error message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.error("APRS-IS - %s" % line)
def log_warning(self, line):
""" Helper function to log a warning message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.warning("APRS-IS - %s" % line)
class printcore():
def __init__(self, port = None, baud = None):
"""Initializes a printcore instance. Pass the port and baud rate to
connect immediately"""
self.baud = None
self.port = None
self.analyzer = GCodeAnalyzer()
self.printer = None # Serial instance connected to the printer,
# should be None when disconnected
self.clear = 0 # clear to send, enabled after responses
self.online = False # The printer has responded to the initial command
# and is active
self.printing = False # is a print currently running, true if printing
# , false if paused
self.mainqueue = None
self.priqueue = Queue(0)
self.queueindex = 0
self.lineno = 0
self.resendfrom = -1
self.paused = False
self.sentlines = {}
self.log = deque(maxlen = 10000)
self.sent = []
self.writefailures = 0
self.tempcb = None # impl (wholeline)
self.recvcb = None # impl (wholeline)
self.sendcb = None # impl (wholeline)
self.preprintsendcb = None # impl (wholeline)
self.printsendcb = None # impl (wholeline)
self.layerchangecb = None # impl (wholeline)
self.errorcb = None # impl (wholeline)
self.startcb = None # impl ()
self.endcb = None # impl ()
self.onlinecb = None # impl ()
self.loud = False # emit sent and received lines to terminal
self.greetings = ['start', 'Grbl ']
self.wait = 0 # default wait period for send(), send_now()
self.read_thread = None
self.stop_read_thread = False
self.send_thread = None
self.stop_send_thread = False
self.print_thread = None
if port is not None and baud is not None:
self.connect(port, baud)
self.xy_feedrate = None
self.z_feedrate = None
self.pronterface = None
@locked
def disconnect(self):
"""Disconnects from printer and pauses the print
"""
if self.printer:
if self.read_thread:
self.stop_read_thread = True
self.read_thread.join()
self.read_thread = None
if self.print_thread:
self.printing = False
self.print_thread.join()
self._stop_sender()
try:
self.printer.close()
except socket.error:
pass
except OSError:
pass
self.printer = None
self.online = False
self.printing = False
@locked
def connect(self, port = None, baud = None):
"""Set port and baudrate if given, then connect to printer
"""
if self.printer:
self.disconnect()
if port is not None:
self.port = port
if baud is not None:
self.baud = baud
if self.port is not None and self.baud is not None:
# Connect to socket if "port" is an IP, device if not
host_regexp = re.compile("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$")
is_serial = True
if ":" in port:
bits = port.split(":")
if len(bits) == 2:
hostname = bits[0]
try:
port = int(bits[1])
if host_regexp.match(hostname) and 1 <= port <= 65535:
is_serial = False
except:
pass
self.writefailures = 0
if not is_serial:
self.printer_tcp = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self.timeout = 0.25
self.printer_tcp.settimeout(1.0)
try:
self.printer_tcp.connect((hostname, port))
self.printer_tcp.settimeout(self.timeout)
self.printer = self.printer_tcp.makefile()
except socket.error as e:
print _("Could not connect to %s:%s:") % (hostname, port)
self.printer = None
self.printer_tcp = None
print _("Socket error %s:") % e.errno,
print e.strerror
return
else:
disable_hup(self.port)
self.printer_tcp = None
try:
self.printer = Serial(port = self.port,
baudrate = self.baud,
timeout = 0.25)
except SerialException as e:
print _("Could not connect to %s at baudrate %s:") % (self.port, self.baud)
self.printer = None
print _("Serial error: %s") % e
return
self.stop_read_thread = False
self.read_thread = Thread(target = self._listen)
self.read_thread.start()
self._start_sender()
def reset(self):
"""Reset the printer
"""
if self.printer and not self.printer_tcp:
self.printer.setDTR(1)
time.sleep(0.2)
self.printer.setDTR(0)
def _readline(self):
try:
try:
line = self.printer.readline()
if self.printer_tcp and not line:
raise OSError(-1, "Read EOF from socket")
except socket.timeout:
return ""
if len(line) > 1:
self.log.append(line)
if self.recvcb:
try: self.recvcb(line)
except: pass
if self.loud: print "RECV:", line.rstrip()
return line
except SelectError as e:
if 'Bad file descriptor' in e.args[1]:
print _("Can't read from printer (disconnected?) (SelectError {0}): {1}").format(e.errno, e.strerror)
return None
else:
print _("SelectError ({0}): {1}").format(e.errno, e.strerror)
raise
except SerialException as e:
print _("Can't read from printer (disconnected?) (SerialException): {0}").format(e)
return None
except socket.error as e:
print _("Can't read from printer (disconnected?) (Socket error {0}): {1}").format(e.errno, e.strerror)
return None
except OSError as e:
if e.errno == errno.EAGAIN: # Not a real error, no data was available
return ""
print _("Can't read from printer (disconnected?) (OS Error {0}): {1}").format(e.errno, e.strerror)
return None
def _listen_can_continue(self):
if self.printer_tcp:
return not self.stop_read_thread and self.printer
return (not self.stop_read_thread
and self.printer
and self.printer.isOpen())
def _listen_until_online(self):
while not self.online and self._listen_can_continue():
self._send("M105")
empty_lines = 0
while self._listen_can_continue():
line = self._readline()
if line is None: break # connection problem
# workaround cases where M105 was sent before printer Serial
# was online an empty line means read timeout was reached,
# meaning no data was received thus we count those empty lines,
# and once we have seen 15 in a row, we just break and send a
# new M105
# 15 was chosen based on the fact that it gives enough time for
# Gen7 bootloader to time out, and that the non received M105
# issues should be quite rare so we can wait for a long time
# before resending
if not line:
empty_lines += 1
if empty_lines == 15: break
else: empty_lines = 0
if line.startswith(tuple(self.greetings)) \
or line.startswith('ok') or "T:" in line:
if self.onlinecb:
try: self.onlinecb()
except: pass
self.online = True
return
def _listen(self):
"""This function acts on messages from the firmware
"""
self.clear = True
if not self.printing:
self._listen_until_online()
while self._listen_can_continue():
line = self._readline()
if line is None:
break
if line.startswith('DEBUG_'):
continue
if line.startswith(tuple(self.greetings)) or line.startswith('ok'):
self.clear = True
if line.startswith('ok') and "T:" in line and self.tempcb:
#callback for temp, status, whatever
try: self.tempcb(line)
except: pass
elif line.startswith('Error'):
if self.errorcb:
#callback for errors
try: self.errorcb(line)
except: pass
# Teststrings for resend parsing # Firmware exp. result
# line="rs N2 Expected checksum 67" # Teacup 2
if line.lower().startswith("resend") or line.startswith("rs"):
for haystack in ["N:", "N", ":"]:
line = line.replace(haystack, " ")
linewords = line.split()
while len(linewords) != 0:
try:
toresend = int(linewords.pop(0))
self.resendfrom = toresend
#print str(toresend)
break
except:
pass
self.clear = True
self.clear = True
def _start_sender(self):
self.stop_send_thread = False
self.send_thread = Thread(target = self._sender)
self.send_thread.start()
def _stop_sender(self):
if self.send_thread:
self.stop_send_thread = True
self.send_thread.join()
self.send_thread = None
def _sender(self):
while not self.stop_send_thread:
try:
command = self.priqueue.get(True, 0.1)
except QueueEmpty:
continue
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
self._send(command)
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
def _checksum(self, command):
return reduce(lambda x, y: x ^ y, map(ord, command))
def startprint(self, gcode, startindex = 0):
"""Start a print, gcode is an array of gcode commands.
returns True on success, False if already printing.
The print queue will be replaced with the contents of the data array,
the next line will be set to 0 and the firmware notified. Printing
will then start in a parallel thread.
"""
if self.printing or not self.online or not self.printer:
return False
self.printing = True
self.mainqueue = gcode
self.lineno = 0
self.queueindex = startindex
self.resendfrom = -1
self._send("M110", -1, True)
if not gcode.lines:
return True
self.clear = False
resuming = (startindex != 0)
self.print_thread = Thread(target = self._print,
kwargs = {"resuming": resuming})
self.print_thread.start()
return True
# run a simple script if it exists, no multithreading
def runSmallScript(self, filename):
if filename is None: return
f = None
try:
with open(filename) as f:
for i in f:
l = i.replace("\n", "")
l = l[:l.find(";")] # remove comments
self.send_now(l)
except:
pass
def pause(self):
"""Pauses the print, saving the current position.
"""
if not self.printing: return False
self.paused = True
self.printing = False
# try joining the print thread: enclose it in try/except because we
# might be calling it from the thread itself
try:
self.print_thread.join()
except:
pass
self.print_thread = None
# saves the status
self.pauseX = self.analyzer.x - self.analyzer.xOffset
self.pauseY = self.analyzer.y - self.analyzer.yOffset
self.pauseZ = self.analyzer.z - self.analyzer.zOffset
self.pauseE = self.analyzer.e - self.analyzer.eOffset
self.pauseF = self.analyzer.f
self.pauseRelative = self.analyzer.relative
def resume(self):
"""Resumes a paused print.
"""
if not self.paused: return False
if self.paused:
# restores the status
self.send_now("G90") # go to absolute coordinates
xyFeedString = ""
zFeedString = ""
if self.xy_feedrate is not None:
xyFeedString = " F" + str(self.xy_feedrate)
if self.z_feedrate is not None:
zFeedString = " F" + str(self.z_feedrate)
self.send_now("G1 X%s Y%s%s" % (self.pauseX, self.pauseY,
xyFeedString))
self.send_now("G1 Z" + str(self.pauseZ) + zFeedString)
self.send_now("G92 E" + str(self.pauseE))
# go back to relative if needed
if self.pauseRelative: self.send_now("G91")
# reset old feed rate
self.send_now("G1 F" + str(self.pauseF))
self.paused = False
self.printing = True
self.print_thread = Thread(target = self._print,
kwargs = {"resuming": True})
self.print_thread.start()
def send(self, command, wait = 0):
"""Adds a command to the checksummed main command queue if printing, or
sends the command immediately if not printing"""
if self.online:
if self.printing:
self.mainqueue.append(command)
else:
self.priqueue.put_nowait(command)
else:
print "Not connected to printer."
def send_now(self, command, wait = 0):
"""Sends a command to the printer ahead of the command queue, without a
checksum"""
if self.online:
self.priqueue.put_nowait(command)
else:
print "Not connected to printer."
def _print(self, resuming = False):
self._stop_sender()
try:
if self.startcb:
#callback for printing started
try: self.startcb(resuming)
except:
print "Print start callback failed with:"
traceback.print_exc(file = sys.stdout)
while self.printing and self.printer and self.online:
self._sendnext()
self.sentlines = {}
self.log.clear()
self.sent = []
if self.endcb:
#callback for printing done
try: self.endcb()
except:
print "Print end callback failed with:"
traceback.print_exc(file = sys.stdout)
except:
print "Print thread died due to the following error:"
traceback.print_exc(file = sys.stdout)
finally:
self.print_thread = None
self._start_sender()
#now only "pause" is implemented as host command
def processHostCommand(self, command):
command = command.lstrip()
if command.startswith(";@pause"):
if self.pronterface is not None:
self.pronterface.pause(None)
else:
self.pause()
def _sendnext(self):
if not self.printer:
return
while self.printer and self.printing and not self.clear:
time.sleep(0.001)
self.clear = False
if not (self.printing and self.printer and self.online):
self.clear = True
return
if self.resendfrom < self.lineno and self.resendfrom > -1:
self._send(self.sentlines[self.resendfrom], self.resendfrom, False)
self.resendfrom += 1
return
self.resendfrom = -1
if not self.priqueue.empty():
self._send(self.priqueue.get_nowait())
self.priqueue.task_done()
return
if self.printing and self.queueindex < len(self.mainqueue):
(layer, line) = self.mainqueue.idxs(self.queueindex)
gline = self.mainqueue.all_layers[layer][line]
if self.layerchangecb and self.queueindex > 0:
(prev_layer, prev_line) = self.mainqueue.idxs(self.queueindex - 1)
if prev_layer != layer:
try: self.layerchangecb(layer)
except: traceback.print_exc()
if self.preprintsendcb:
if self.queueindex + 1 < len(self.mainqueue):
(next_layer, next_line) = self.mainqueue.idxs(self.queueindex + 1)
next_gline = self.mainqueue.all_layers[next_layer][next_line]
else:
next_gline = None
gline = self.preprintsendcb(gline, next_gline)
if gline is None:
self.queueindex += 1
self.clear = True
return
tline = gline.raw
if tline.lstrip().startswith(";@"): # check for host command
self.processHostCommand(tline)
self.queueindex += 1
self.clear = True
return
tline = tline.split(";")[0]
if len(tline) > 0:
self._send(tline, self.lineno, True)
self.lineno += 1
if self.printsendcb:
try: self.printsendcb(gline)
except: traceback.print_exc()
else:
self.clear = True
self.queueindex += 1
else:
self.printing = False
self.clear = True
if not self.paused:
self.queueindex = 0
self.lineno = 0
self._send("M110", -1, True)
def _send(self, command, lineno = 0, calcchecksum = False):
if calcchecksum:
prefix = "N" + str(lineno) + " " + command
command = prefix + "*" + str(self._checksum(prefix))
if "M110" not in command:
self.sentlines[lineno] = command
if self.printer:
self.sent.append(command)
# run the command through the analyzer
try: self.analyzer.Analyze(command)
except:
print "Warning: could not analyze command %s:" % command
traceback.print_exc(file = sys.stdout)
if self.loud:
print "SENT:", command
if self.sendcb:
try: self.sendcb(command)
except: pass
try:
self.printer.write(str(command + "\n"))
if self.printer_tcp: self.printer.flush()
self.writefailures = 0
except socket.error as e:
print _("Can't write to printer (disconnected?) (Socket error {0}): {1}").format(e.errno, e.strerror)
self.writefailures += 1
except SerialException as e:
print _("Can't write to printer (disconnected?) (SerialException): {0}").format(e)
self.writefailures += 1
except RuntimeError as e:
print _("Socket connection broken, disconnected. ({0}): {1}").format(e.errno, e.strerror)
self.writefailures += 1
class ZUsageStatisticsService(IZService, IZDataStoreListener, IZMediaStorageServiceListener, IZAccountStoreListener, IZRunnable):
def __init__(self):
self.serializer = ZUsagePacketSerializer()
self.logger = None
self.running = False
self.version = ZVersion()
self.profileGuid = getApplicationModel().getUserProfile().getGuid()
self.usageDir = None
# end __init__()
def start(self, applicationModel):
self.queue = Queue(0)
userProfile = applicationModel.getUserProfile()
self.usageDir = userProfile.getDirectory(u"usage") #$NON-NLS-1$
engine = applicationModel.getEngine()
self.logger = engine.getService(IZBlogAppServiceIDs.LOGGER_SERVICE_ID)
self.logger.debug(u"Anonymous Usage Statistics Service started.") #$NON-NLS-1$
accountStore = engine.getService(IZBlogAppServiceIDs.ACCOUNT_STORE_SERVICE_ID)
dataStore = engine.getService(IZBlogAppServiceIDs.DATA_STORE_SERVICE_ID)
mediaStoreService = engine.getService(IZBlogAppServiceIDs.MEDIA_STORAGE_SERVICE_ID)
accountStore.addListener(self)
dataStore.addListener(self)
mediaStoreService.addListener(self)
self.done = False
self.running = True
thread = ZThread(self, u"ZUsageStatisticsService", True) #$NON-NLS-1$
thread.start()
# end start()
def stop(self):
self.logger = None
packet = self._createPacket(IZUsagePacketTypes.EXIT_SERVICE)
self.queue.put_nowait(packet)
while self.running:
pass
# end stop()
def run(self):
done = False
while not done:
packet = self.queue.get()
if packet.getType() == IZUsagePacketTypes.EXIT_SERVICE:
done = True
else:
self._savePacket(packet)
self.running = False
# end run()
def _savePacket(self, packet):
packetDom = self.serializer.serialize(packet)
fname = os.path.join(self.usageDir, packet.getId() + u".xml") #$NON-NLS-1$
packetDom.save(fname, True)
# end _savePacket()
#
# def onDocumentAdded(self, document):
# self.logger.debug(u"Document added.")
# # end onDocumentAdded()
#
# def onDocumentChanged(self, document, metaDataOnly):
# self.logger.debug(u"Document changed.")
# # end onDocumentChange()
#
# def onDocumentDeleted(self, document):
# self.logger.debug(u"Document deleted.")
# # end onDocumentDeleted()
def onMediaStorageAdded(self, mediaStore):
packet = self._createPacket(IZUsagePacketTypes.NEW_MEDIA_STORE)
packet.addAttribute(u"mediastorage.site-id", mediaStore.getMediaSiteId()) #$NON-NLS-1$
packet.addAttribute(u"mediastorage.id-hash", hash(mediaStore.getId())) #$NON-NLS-1$
self.queue.put_nowait(packet)
# end onMediaStorageAdded()
def onMediaStorageRemoved(self, mediaStore):
packet = self._createPacket(IZUsagePacketTypes.DELETE_MEDIA_STORE)
packet.addAttribute(u"mediastorage.site-id", mediaStore.getMediaSiteId()) #$NON-NLS-1$
packet.addAttribute(u"mediastorage.id-hash", hash(mediaStore.getId())) #$NON-NLS-1$
self.queue.put_nowait(packet)
# end onMediaStorageRemoved() #$NON-NLS-1$
def onAccountAdded(self, account):
packet = self._createPacket(IZUsagePacketTypes.NEW_ACCOUNT)
if isinstance(account, IZBlogAccount):
apiInfo = account.getAPIInfo()
packet.addAttribute(u"account.type", apiInfo.getType()) #$NON-NLS-1$
packet.addAttribute(u"account.id-hash", unicode(hash(account.getId()))) #$NON-NLS-1$
self.queue.put_nowait(packet)
# end onAccountAdded()
def onAccountChanged(self, account): #@UnusedVariable
pass
# end onAccountChange()
def onAccountDeleted(self, account):
packet = self._createPacket(IZUsagePacketTypes.NEW_ACCOUNT)
if isinstance(account, IZBlogAccount):
apiInfo = account.getAPIInfo()
packet.addAttribute(u"account.type", apiInfo.getType()) #$NON-NLS-1$
packet.addAttribute(u"account.id-hash", unicode(hash(account.getId()))) #$NON-NLS-1$
self.queue.put_nowait(packet)
# end onAccountDeleted()
def _createPacket(self, type):
packet = ZUsagePacket(type)
packet.addAttribute(u"global.app-version", self.version.getFullVersionString()) #$NON-NLS-1$
packet.addAttribute(u"global.profile-guid", self.profileGuid) #$NON-NLS-1$
return packet
class ExecThread(threading.Thread):
"""Thread that does the execution. It can accept options with an execution, and queues execs if necessary."""
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self.queue = Queue()
def run(self):
from flexget.ui.webui import manager
while True:
kwargs = self.queue.get() or {}
opts = kwargs.pop('options', None)
parsed_options = kwargs.pop('parsed_options', None)
output = kwargs.pop('output', None)
if opts:
# make copy of original options and apply options from opts
old_opts = copy(manager.options)
self._apply_options(manager.options, opts)
if parsed_options:
old_opts = manager.options
manager.options = parsed_options
if output:
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = output
sys.stderr = output
# TODO: Use a filter to capture only the logging for this execution
streamhandler = logging.StreamHandler(output)
streamhandler.setFormatter(FlexGetFormatter())
logging.getLogger().addHandler(streamhandler)
try:
manager.execute(**kwargs)
finally:
# Inform queue we are done processing this item.
self.queue.task_done()
# Restore manager's previous options and stdout
if opts:
manager.options = old_opts
if output:
# Write EOF to the output, so that a listener knows when the output is over
output.write('EOF')
sys.stdout = old_stdout
sys.stderr = old_stderr
logging.getLogger().removeHandler(streamhandler)
def _apply_options(self, parser, options):
"""Applies dict :options: to ArgParse parser results"""
for name, value in options.iteritems():
if hasattr(parser, name):
log.debug('setting options %s to %s' % (name, value))
setattr(parser, name, value)
else:
log.error('Option %s does not exist, ignoring it' % name)
def execute(self, **kwargs):
"""
Adds an execution to the queue.
keyword arguments:
options: Dict containing option, value pairs for this execution
parsed_options: Parsed OptionParser to be used for this execution
output: a BufferQueue object that will be filled with output from the execution.
all other keyword arguments will be passed to manager.execute
kwargs options and parsed_options are mutually exclusive
"""
if 'options' in kwargs and not isinstance(kwargs['options'], dict):
raise TypeError('options should be a dict, got %s' %
type(kwargs['options']))
if 'tasks' in kwargs and not hasattr(kwargs['tasks'], '__iter__'):
raise TypeError('tasks should be iterable, got %s' %
type(kwargs['tasks']))
if 'options' in kwargs and 'parsed_options' in kwargs:
raise ValueError(
'options and parsed_options are mutually exclusive')
if kwargs.get('output') and self.queue.unfinished_tasks:
kwargs['output'].write(
'There is already an execution running. ' +
'This execution will start when the previous completes.')
self.queue.put_nowait(kwargs)
class DragonflyWrapper(object):
def __init__(self):
engine = Sapi5InProcEngine()
engine.connect()
engine.speak('Speak recognition active!')
logger.info('Speak recognition active!')
self._results = Queue()
self._grammar = None
def process_recognition_failure(self):
logger.info('Grammar:process_recognition_failure')
def set_grammar(self, spec, choices_values):
spec_truncated = (spec[:75] + '...') if len(spec) > 75 else spec
logger.info('Set Grammar: %s %s', spec_truncated, choices_values)
# TODO: cache the rule
assert self._results.empty()
if self._grammar:
# remove the old rule
self._grammar.unload()
self._grammar = Grammar("G")
# attach failure callback
self._grammar.process_recognition_failure = self.process_recognition_failure
rule = self._make_rule(spec, choices_values, self._result_callback)
self._grammar.add_rule(rule)
self._grammar.load()
winsound.PlaySound(data_path + "/grammar_loaded.wav",
winsound.SND_ASYNC)
logger.info("Grammar loaded")
def spin_once(self):
pythoncom.PumpWaitingMessages()
@property
def results(self):
return self._results
@staticmethod
def _make_rule(spec, choices_values, callback):
# build the dragonfly request
extras = []
for name, choices in choices_values.iteritems():
extras.append(Choice(name, dict((c, c) for c in choices)))
class CustomRule(CompoundRule):
def _process_recognition(self, node, extras):
logger.info('_process_recognition callback: %s', str(node))
callback(node, extras)
def _process_begin(self):
logger.debug('Rule:__process_begin')
rule = CustomRule(spec=spec, extras=extras)
return rule
def _result_callback(self, node, extras):
self._results.put_nowait(Result(node=node, extras=extras))
class MetricQueue(object):
""" Object to which various metrics are written, for distribution to metrics collection
system(s) such as Prometheus.
"""
def __init__(self, prom):
# Define the various exported metrics.
self.resp_time = prom.create_histogram(
'response_time',
'HTTP response time in seconds',
labelnames=['endpoint'],
buckets=API_RESPONSE_TIME_BUCKETS)
self.resp_code = prom.create_counter('response_code',
'HTTP response code',
labelnames=['endpoint', 'code'])
self.non_200 = prom.create_counter('response_non200',
'Non-200 HTTP response codes',
labelnames=['endpoint'])
self.error_500 = prom.create_counter('response_500',
'5XX HTTP response codes',
labelnames=['endpoint'])
self.multipart_upload_start = prom.create_counter(
'multipart_upload_start', 'Multipart upload started')
self.multipart_upload_end = prom.create_counter(
'multipart_upload_end',
'Multipart upload ends.',
labelnames=['type'])
self.build_capacity_shortage = prom.create_gauge(
'build_capacity_shortage', 'Build capacity shortage.')
self.builder_time_to_start = prom.create_histogram(
'builder_tts',
'Time from triggering to starting a builder.',
labelnames=['builder_type'],
buckets=BUILDER_START_TIME_BUCKETS)
self.builder_time_to_build = prom.create_histogram(
'builder_ttb',
'Time from triggering to actually starting a build',
labelnames=['builder_type'],
buckets=BUILDER_START_TIME_BUCKETS)
self.build_time = prom.create_histogram('build_time',
'Time spent building',
labelnames=['builder_type'])
self.builder_fallback = prom.create_counter(
'builder_fallback', 'Builder fell back to secondary executor')
self.build_start_success = prom.create_counter(
'build_start_success',
'Executor succeeded in starting a build',
labelnames=['builder_type'])
self.build_start_failure = prom.create_counter(
'build_start_failure',
'Executor failed to start a build',
labelnames=['builder_type'])
self.percent_building = prom.create_gauge('build_percent_building',
'Percent building.')
self.build_counter = prom.create_counter('builds',
'Number of builds',
labelnames=['name'])
self.ephemeral_build_workers = prom.create_counter(
'ephemeral_build_workers',
'Number of started ephemeral build workers')
self.ephemeral_build_worker_failure = prom.create_counter(
'ephemeral_build_worker_failure',
'Number of failed-to-start ephemeral build workers')
self.work_queue_running = prom.create_gauge('work_queue_running',
'Running items in a queue',
labelnames=['queue_name'])
self.work_queue_available = prom.create_gauge(
'work_queue_available',
'Available items in a queue',
labelnames=['queue_name'])
self.work_queue_available_not_running = prom.create_gauge(
'work_queue_available_not_running',
'Available items that are not yet running',
labelnames=['queue_name'])
self.repository_pull = prom.create_counter(
'repository_pull',
'Repository Pull Count',
labelnames=['namespace', 'repo_name', 'protocol', 'status'])
self.repository_push = prom.create_counter(
'repository_push',
'Repository Push Count',
labelnames=['namespace', 'repo_name', 'protocol', 'status'])
self.repository_build_queued = prom.create_counter(
'repository_build_queued',
'Repository Build Queued Count',
labelnames=['namespace', 'repo_name'])
self.repository_build_completed = prom.create_counter(
'repository_build_completed',
'Repository Build Complete Count',
labelnames=['namespace', 'repo_name', 'status', 'executor'])
self.chunk_size = prom.create_histogram('chunk_size',
'Registry blob chunk size',
labelnames=['storage_region'])
self.chunk_upload_time = prom.create_histogram(
'chunk_upload_time',
'Registry blob chunk upload time',
labelnames=['storage_region'])
self.authentication_count = prom.create_counter(
'authentication_count',
'Authentication count',
labelnames=['kind', 'status'])
self.repository_count = prom.create_gauge('repository_count',
'Number of repositories')
self.user_count = prom.create_gauge('user_count', 'Number of users')
self.org_count = prom.create_gauge('org_count',
'Number of Organizations')
self.robot_count = prom.create_gauge('robot_count',
'Number of robot accounts')
self.instance_key_renewal_success = prom.create_counter(
'instance_key_renewal_success',
'Instance Key Renewal Success Count',
labelnames=['key_id'])
self.instance_key_renewal_failure = prom.create_counter(
'instance_key_renewal_failure',
'Instance Key Renewal Failure Count',
labelnames=['key_id'])
self.invalid_instance_key_count = prom.create_counter(
'invalid_registry_instance_key_count',
'Invalid registry instance key count',
labelnames=['key_id'])
self.verb_action_passes = prom.create_counter(
'verb_action_passes',
'Verb Pass Count',
labelnames=['kind', 'pass_count'])
self.push_byte_count = prom.create_counter(
'registry_push_byte_count',
'Number of bytes pushed to the registry')
self.pull_byte_count = prom.create_counter(
'estimated_registry_pull_byte_count',
'Number of (estimated) bytes pulled from the registry',
labelnames=['protocol_version'])
# Deprecated: Define an in-memory queue for reporting metrics to CloudWatch or another
# provider.
self._queue = None
def enable_deprecated(self, maxsize=10000):
self._queue = Queue(maxsize)
def put_deprecated(self, name, value, **kwargs):
if self._queue is None:
logger.debug('No metric queue %s %s %s', name, value, kwargs)
return
try:
kwargs.setdefault('timestamp', datetime.datetime.now())
kwargs.setdefault('dimensions', {})
self._queue.put_nowait((name, value, kwargs))
except Full:
logger.error('Metric queue full')
def get_deprecated(self):
return self._queue.get()
def get_nowait_deprecated(self):
return self._queue.get_nowait()
class paka(object):
# /**
# * 让链接的参数直接传递给connection
# *
# */
def __init__(self, *args, **argv):
self.parameters = paka_param(*args, **argv)
self.routing_key = ''
self._typ = ''
self.queue = ''
self.exchange = ''
self.connection = None
self.channel = None
self.running = False
self._queue = Queue()
def blocking_error_out(self, *args, **argv):
if self.connection:
self.connection.socket.close()
self.connection._on_connection_closed(None, True)
self.connection = None
func = object.__getattribute__(self, 'connect')
routing_key = object.__getattribute__(self, 'routing_key')
_typ = object.__getattribute__(self, '_typ')
exchange = object.__getattribute__(self, 'exchange')
func(exchange, routing_key, _typ)
def basic_publish(self,
exchange,
routing_key,
body,
properties=None,
mandatory=False,
immediate=False):
if not self.running:
self._queue.put_nowait({
'exchange': exchange,
'routing_key': routing_key,
'body': body,
'properties': properties,
'mandatory': mandatory,
'immediate': immediate
})
try:
return self.connection.basic_publish(exchange, routing_key, body,
properties, mandatory,
immediate)
except:
self._queue.put({
'exchange': exchange,
'routing_key': routing_key,
'body': body,
'properties': properties,
'mandatory': mandatory,
'immediate': immediate
})
def connect(self,
exchange='',
routing_key='',
_typ='fanout',
queue='OtherQueueTest'):
self.routing_key = routing_key
self._typ = _typ
self.queue = queue
self.exchange = exchange
try:
self.connection = pika.BlockingConnection(
self.parameters.get_param())
self.connection._handle_disconnect = self.blocking_error_out
self.channel = self.connection.channel()
self.channel.exchange_declare(exchange=exchange, type=_typ)
self.channel.queue_bind(exchange=exchange, queue=queue)
self.running = True
while not self._queue.empty():
info = self._queue.get_nowait()
if info:
self.basic_publish(info['exchange'], info['routing_key'],
info['body'], info['properties'],
info['mandatory'], info['immediate'])
except AMQPChannelError:
self.running = False
self.blocking_error_out()
except error, e:
self.running = False
# 链接被中断，重新连一次
self.blocking_error_out()
except:
class IBusHandler(object):
def __init__(self):
self.serial_port = serial.Serial()
self.serial_port.baudrate = 9600
self.serial_port.parity = serial.PARITY_EVEN
self.serial_port.stopbits = serial.STOPBITS_ONE
self.serial_port.timeout = 0.001
self.rts_state = False
self.read_buffer = []
self.read_lock = Lock()
self.read_error_counter = 0
self.read_error_container = []
self.cancel_read_thread = False
self.read_thread = Thread(target=self._reading)
self.read_thread.daemon = True
self.packet_buffer = Queue()
self.cts_counter = 0.0
self.cts_thread = Thread(target=self._cts_watcher)
self.cts_thread.daemon = True
self.write_buffer = PriorityQueue()
self.write_counter = 0
self.cancel_write_thread = False
self.write_thread = Thread(target=self._writing)
self.write_thread.daemon = True
def __enter__(self):
self.connect()
return self
def __exit__(self, *_args):
self.disconnect()
@property
def ntsc(self):
return self.rts_state
@ntsc.setter
def ntsc(self, value):
if self.rts_state == value:
return
self.serial_port.setRTS(value)
self.rts_state = value
@staticmethod
def _calculate_checksum(packet):
result = 0
for value in packet:
result ^= value
return result
def _cut_read_buffer(self, offset=1):
with self.read_lock:
self.read_buffer = self.read_buffer[offset:]
def _wait_free_bus(self, waiting=17, timeout=1000):
#
# FIXME Log message doesn't match ``if`` condition.
#
if waiting >= timeout:
log('Error: Waiting Time (%sms) is bigger than Timeout Time (%sms)'
% (waiting, timeout))
return False
for _ in xrange(timeout):
if self.cts_counter >= waiting:
return True
time.sleep(0.001)
return False
def _reading(self):
while not self.cancel_read_thread:
data = self.serial_port.read(50)
if data:
with self.read_lock:
self.read_buffer.extend(ord(x) for x in data)
self._read_bus_packet()
log('Read/Write Thread finished')
def _writing(self):
while not self.cancel_read_thread:
try:
prio, write_counter, data = self.write_buffer.get(timeout=1)
try:
while self.cts_counter < 7.0:
time.sleep(0.001)
self.serial_port.write(data)
self.cts_counter = 0.0
self.serial_port.flush()
self.cts_counter = 0.0
log('\033[1;33;40mWRITE:\033[0m %s' %
' '.join('%02X' % i for i in data))
except serial.SerialException:
self.write_buffer.put((prio, write_counter, data))
except Empty:
pass
def _cts_watcher(self):
while not self.cancel_read_thread:
if self.serial_port.getCTS():
self.cts_counter += 0.1
time.sleep(0.0001)
else:
self.cts_counter = 0.0
#
# TODO Maybe sleeping a little or move the `sleep()`
# in the ``if`` branch after the ``if``/``else``?
#
log('CTS Thread finished')
def set_port(self, device_path):
self.serial_port.port = device_path
def connect(self):
self.serial_port.open()
self.serial_port.setRTS(0)
self.cts_thread.start()
if not self._wait_free_bus(120, 3000):
log('Error: Can not locate free Bus')
raise RuntimeError('can not locate free bus')
self.serial_port.flushInput()
self.read_thread.start()
self.write_thread.start()
def disconnect(self):
self.cancel_write_thread = True
self.cancel_read_thread = True
self.ntsc = False
time.sleep(0.6)
self.serial_port.close()
log('disconnected')
def read_bus_packet(self):
try:
return self.packet_buffer.get(timeout=1)
except Empty:
return None
def _read_bus_packet(self):
if self.cancel_read_thread:
return None
try:
data_length = self.read_buffer[1]
except IndexError:
return None
if not 3 <= data_length <= 37:
self.read_error_container.append(self.read_buffer[0])
self._cut_read_buffer()
return None
buffer_len = len(self.read_buffer)
if buffer_len < 5 or buffer_len < data_length + 2:
return None
message = self.read_buffer[:data_length + 2]
if self._calculate_checksum(message) == 0:
if self.read_error_container:
error_hex_string = ' '.join('%02X' % i
for i in self.read_error_container)
log('READ-ERR: %s' % error_hex_string)
self.read_error_counter += len(self.read_error_container)
self.read_error_container = []
self._cut_read_buffer(data_length + 2)
self.packet_buffer.put_nowait({
'src': message[0],
'len': data_length,
'dst': message[2],
'data': message[3:data_length + 1],
'xor': message[-1]
})
else:
self.read_error_container.append(self.read_buffer[0])
self._cut_read_buffer()
return None
def write_bus_packet(self,
src,
dst,
data,
highprio=False,
veryhighprio=False,
repeat=1):
#
# FIXME The ``except`` needs explicit exceptions because it is
# unclear under what circumstances the ``except`` is a proper
# handling of the exception.
#
try:
packet = [src, len(data) + 2, dst]
packet.extend(data)
except:
packet = [int(src, 16), len(data) + 2, int(dst, 16)]
packet.extend([int(s, 16) for s in data])
packet.append(self._calculate_checksum(packet))
for _ in xrange(repeat):
self.write_buffer.put_nowait(
(0 if highprio or veryhighprio else 1,
0 if veryhighprio else self.write_counter, bytearray(packet)))
self.write_counter += 1
def write_hex_message(self, hexstring):
hexstring_tmp = hexstring.upper().split(' ')
src = int(hexstring_tmp[0], 16)
dst = int(hexstring_tmp[2], 16)
data = [int(s, 16) for s in hexstring_tmp[3:-1]]
self.write_bus_packet(src, dst, data)
class WriteWarp10(object):
def __init__(self, url, token, flush_interval, flush_retry_interval,
buffer_size, default_labels, rewrite_rules, rewrite_limit):
self.url = url
self.token = token
self.flush_interval = flush_interval
self.flush_retry_interval = flush_retry_interval
self.buffer_size = buffer_size
self.default_labels = default_labels
self.rewrite_rules = rewrite_rules
self.rewrite_limit = rewrite_limit
self.queue = Queue(buffer_size)
self.flush_timer = None
@staticmethod
def config(cfg):
# Handle legacy config (not multiple-endpoint capable)
if not any([n.key == 'Endpoint' for n in cfg.children]):
# Create fake intermediary Endpoint node
cfg.children = (collectd.Config('Endpoint', cfg, ('default', ),
cfg.children), )
endpoints = []
for node in cfg.children:
if node.key == 'Endpoint':
endpoint = WriteWarp10.config_endpoint(node)
if endpoint:
if any(e['name'] == endpoint['name'] for e in endpoints):
collectd.warning('write_warp10 plugin: Duplicate '
'endpoint: %s' % endpoint['name'])
else:
endpoints.append(endpoint)
else:
collectd.warning('write_warp10 plugin: Unknown config key: '
'%s' % node.key)
if endpoints:
for e in endpoints:
ww10 = WriteWarp10(e['url'], e['token'], e['flush_interval'],
e['flush_retry_interval'], e['buffer_size'],
e['default_labels'], e['rewrite_rules'],
e['rewrite_limit'])
collectd.info('write_warp10 plugin: register init write and '
'shutdown functions')
collectd.register_init(ww10.init,
name='write_warp10/%s' % e['name'])
collectd.register_write(ww10.write,
name='write_warp10/%s' % e['name'])
collectd.register_shutdown(ww10.shutdown,
name='write_warp10/%s' % e['name'])
else:
collectd.warning('write_warp10 plugin: No valid endpoints found')
@staticmethod
def config_endpoint(cfg):
endpoint = {
'name': None,
'url': None,
'token': None,
'flush_interval': 30.0,
'flush_retry_interval': 10.0,
'buffer_size': 65536,
'default_labels': {},
'rewrite_rules': [],
'rewrite_limit': 10
}
if len(cfg.values) == 1:
endpoint['name'] = cfg.values[0]
for node in cfg.children:
if node.key == 'URL':
endpoint['url'] = node.values[0]
elif node.key == 'Token':
endpoint['token'] = node.values[0]
elif node.key == 'FlushInterval':
endpoint['flush_interval'] = float(node.values[0])
elif node.key == 'FlushRetryInterval':
endpoint['flush_retry_interval'] = float(node.values[0])
elif node.key == 'BufferSize':
endpoint['buffer_size'] = int(node.values[0])
elif node.key == 'DefaultLabel':
endpoint['default_labels'][node.values[0]] = node.values[1]
elif node.key == 'RewriteLimit':
endpoint['rewrite_limit'] = int(node.values[0])
elif node.key == 'RewriteRule':
if len(node.values) not in [2, 3]:
collectd.warning('write_warp10 plugin: Invalid '
'RewriteRule declaration: '
'%s' % node.values)
continue
rule = re.compile(r'%s' % node.values[0])
rewrite = r'%s' % node.values[1]
flags = []
if len(node.values) == 3:
flags = [
r'%s' % f.strip() for f in node.values[2].split(',')
if f.strip()
]
endpoint['rewrite_rules'].append([rule, rewrite, flags])
else:
collectd.warning('write_warp10 plugin: Unknown config key for '
'Endpoint: %s' % node.key)
if not endpoint['name'] or not endpoint['url'] \
or not endpoint['token']:
collectd.warning('write_warp10 plugin: Missing name, URL or Token '
'config for Endpoint')
endpoint = None
return endpoint
def init(self):
self.flush_timer = Timer(self.flush_interval, self._flush_timer)
self.flush_timer.start()
def write(self, vl, data=None):
datasets = collectd.get_dataset(vl.type)
for ds, value in zip(datasets, vl.values):
if math.isnan(value):
continue
ds_name, ds_type, ds_min, ds_max = ds
classname, new_labels = self._format(vl.plugin, vl.plugin_instance,
vl.type, vl.type_instance,
ds_name, ds_type)
if classname is None:
# Ignore classname that are unset (it's a feature from rewrite
# rule to destroy a point)
continue
labels = self.default_labels.copy()
labels.update(vl.meta)
labels.update(new_labels)
# Remove empty values
labels = {
k: str(v).strip()
for k, v in labels.items() if v is not None and str(v).strip()
}
msg = '%d// %s{%s} %f' % (
int(1000000 * vl.time), # Microseconds
classname,
urllib.urlencode(labels).replace('&', ', '),
value)
try:
self.queue.put_nowait(msg)
except Full:
collectd.warning('write_warp10 plugin: Buffer is full (%s '
'elements) for endpoint "%s". The WARP '
'endpoint may encounter issues. Otherwise, '
'consider increasing BufferSize or reducing '
'FlushInterval' %
(self.queue.qsize(), self.url))
def _format(self, *arr):
classname = urllib.quote('.'.join(
[x.strip() for x in arr if x.strip()]))
labels = {}
for _ in xrange(self.rewrite_limit):
last = False
next_round = False
for rule, rewrite, flags in self.rewrite_rules:
last = False
next_round = False
matches = re.match(rule, classname)
if matches:
# Replacement
classname = re.sub(rule, rewrite, classname)
# Apply flags
for flag in flags:
if flag == 'F':
return None, None
elif flag == 'L':
last = True
elif flag == 'N':
next_round = True
elif flag.startswith('T:'):
lbl_name, lbl_value = flag[2:].split('=', 1)
for ma in re.findall(r'(\\[0-9]+)', lbl_name):
v = matches.group(int(ma[1:]))
lbl_name = lbl_name.replace(ma, v)
for ma in re.findall(r'(\\[0-9]+)', lbl_value):
v = matches.group(int(ma[1:]))
lbl_value = lbl_value.replace(ma, v)
labels[lbl_name] = lbl_value
if last or next_round:
break
else:
last = True # Implicit last if we reach end of rules
if last and next_round:
raise Exception('write_warp10 plugin: Incompatible rewrite '
'flags in the same rule: L and N')
elif last:
break
elif next_round:
pass
else:
raise Exception('write_warp10 plugin: Rewrite limit exceeded')
return classname, labels
def shutdown(self):
collectd.info("write_warp10 plugin: Shutdown: Start")
self.flush_timer.cancel()
collectd.info("write_warp10 plugin: Shutdown: Timer cancelled")
self.flush_timer.join()
collectd.info("write_warp10 plugin: Shutdown: Timer thread joined")
try:
self._flush()
except Exception as e:
stack_str = repr(traceback.format_exception(*sys.exc_info()))
collectd.error('write_warp10 plugin: Failed to post data before '
'shutdown: %s' % stack_str)
def _flush_timer(self):
try:
self._flush()
next_interval = self.flush_interval
except:
next_interval = self.flush_retry_interval
self.flush_timer = Timer(next_interval, self._flush_timer)
self.flush_timer.daemon = True
self.flush_timer.start()
def _flush(self):
messages = []
try:
while True:
messages.append(self.queue.get_nowait())
except Empty:
pass
if len(messages) > 0:
for msg in messages:
collectd.debug('write_warp10 plugin: Posting: %s' % msg)
try:
# Header X-CityzenData-Token is deprecated in favor of
# X-Warp10-Token, keeping compatibility
headers = {
'X-Warp10-Token': self.token,
'X-CityzenData-Token': self.token
}
body = "\n".join(messages)
req = urllib2.Request(self.url, body, headers)
resp = urllib2.urlopen(req, timeout=80)
if resp.getcode() != 200:
raise Exception('%d %s' % (resp.getcode(), resp.read()))
except Exception as e:
stack_str = repr(traceback.format_exception(*sys.exc_info()))
collectd.error('write_warp10 plugin: Failed to post data: '
'%s' % stack_str)
try:
for msg in messages:
self.queue.put_nowait(msg)
except Full:
collectd.warning('write_warp10 plugin: Buffer is full (%s '
'elements) for endpoint "%s". The WARP '
'endpoint may encounter issues. '
'Otherwise, consider increasing '
'BufferSize or reducing FlushInterval' %
(self.queue.qsize(), self.url))
raise
class Gviz(wx.Panel):
# Mark canvas as dirty when setting showall
_showall = 0
def _get_showall(self):
return self._showall
def _set_showall(self, showall):
if showall != self._showall:
self.dirty = 1
self._showall = showall
showall = property(_get_showall, _set_showall)
def __init__(self,
parent,
size=(200, 200),
build_dimensions=[200, 200, 100, 0, 0, 0],
grid=(10, 50),
extrusion_width=0.5,
bgcolor="#000000",
realparent=None):
wx.Panel.__init__(self, parent, -1)
self.widget = self
size = [max(1.0, x) for x in size]
ratio = size[0] / size[1]
self.SetMinSize((150, 150 / ratio))
self.parent = realparent if realparent else parent
self.size = size
self.build_dimensions = build_dimensions
self.grid = grid
self.lastpos = [0, 0, 0, 0, 0, 0, 0]
self.hilightpos = self.lastpos[:]
self.Bind(wx.EVT_PAINT, self.paint)
self.Bind(wx.EVT_SIZE, self.resize)
self.lines = {}
self.pens = {}
self.arcs = {}
self.arcpens = {}
self.layers = []
self.layerindex = 0
self.filament_width = extrusion_width # set it to 0 to disable scaling lines with zoom
self.update_basescale()
self.scale = self.basescale
penwidth = max(
1.0, self.filament_width * ((self.scale[0] + self.scale[1]) / 2.0))
self.translate = [0.0, 0.0]
self.mainpen = wx.Pen(wx.Colour(0, 0, 0), penwidth)
self.arcpen = wx.Pen(wx.Colour(255, 0, 0), penwidth)
self.travelpen = wx.Pen(wx.Colour(10, 80, 80), penwidth)
self.hlpen = wx.Pen(wx.Colour(200, 50, 50), penwidth)
self.fades = [
wx.Pen(
wx.Colour(250 - 0.6**i * 100, 250 - 0.6**i * 100,
200 - 0.4**i * 50), penwidth) for i in xrange(6)
]
self.penslist = [self.mainpen, self.travelpen, self.hlpen] + self.fades
self.showall = 0
self.hilight = deque()
self.hilightarcs = deque()
self.hilightqueue = Queue(0)
self.hilightarcsqueue = Queue(0)
self.dirty = 1
self.bgcolor = wx.Colour()
self.bgcolor.SetFromName(bgcolor)
self.blitmap = wx.EmptyBitmap(self.GetClientSize()[0],
self.GetClientSize()[1], -1)
self.paint_overlay = None
def inject(self):
#import pdb; pdb.set_trace()
print "Inject code here..."
print "Layer " + str(self.layerindex + 1) + " - Z = " + str(
self.layers[self.layerindex]) + " mm"
def clearhilights(self):
self.hilight.clear()
self.hilightarcs.clear()
while not self.hilightqueue.empty():
self.hilightqueue.get_nowait()
while not self.hilightarcsqueue.empty():
self.hilightarcsqueue.get_nowait()
def clear(self):
self.lastpos = [0, 0, 0, 0, 0, 0, 0]
self.lines = {}
self.pens = {}
self.arcs = {}
self.arcpens = {}
self.layers = []
self.clearhilights()
self.layerindex = 0
self.showall = 0
self.dirty = 1
wx.CallAfter(self.Refresh)
def layerup(self):
if self.layerindex + 1 < len(self.layers):
self.layerindex += 1
self.parent.SetStatusText(
_("Layer %d - Going Up - Z = %.03f mm") %
(self.layerindex + 1, self.layers[self.layerindex]), 0)
self.dirty = 1
self.parent.setlayercb(self.layerindex)
wx.CallAfter(self.Refresh)
def layerdown(self):
if self.layerindex > 0:
self.layerindex -= 1
self.parent.SetStatusText(
_("Layer %d - Going Down - Z = %.03f mm") %
(self.layerindex + 1, self.layers[self.layerindex]), 0)
self.dirty = 1
self.parent.setlayercb(self.layerindex)
wx.CallAfter(self.Refresh)
def setlayer(self, layer):
if layer in self.layers:
self.layerindex = self.layers.index(layer)
self.dirty = 1
self.showall = 0
wx.CallAfter(self.Refresh)
def update_basescale(self):
self.basescale = 2 * [
min(
float(self.size[0] - 1) / self.build_dimensions[0],
float(self.size[1] - 1) / self.build_dimensions[1])
]
def resize(self, event):
old_basescale = self.basescale
self.size = self.GetClientSizeTuple()
self.update_basescale()
zoomratio = float(self.basescale[0]) / old_basescale[0]
wx.CallLater(200, self.zoom, 0, 0, zoomratio)
def zoom(self, x, y, factor):
if x == -1 and y == -1:
side = min(self.size)
x = y = side / 2
self.scale = [s * factor for s in self.scale]
self.translate = [
x - (x - self.translate[0]) * factor,
y - (y - self.translate[1]) * factor
]
penwidth = max(
1.0, self.filament_width * ((self.scale[0] + self.scale[1]) / 2.0))
for pen in self.penslist:
pen.SetWidth(penwidth)
self.dirty = 1
wx.CallAfter(self.Refresh)
def _line_scaler(self, x):
return (
self.scale[0] * x[0],
self.scale[1] * x[1],
self.scale[0] * x[2],
self.scale[1] * x[3],
)
def _arc_scaler(self, x):
return (
self.scale[0] * x[0],
self.scale[1] * x[1],
self.scale[0] * x[2],
self.scale[1] * x[3],
self.scale[0] * x[4],
self.scale[1] * x[5],
)
def _drawlines(self, dc, lines, pens):
scaled_lines = map(self._line_scaler, lines)
dc.DrawLineList(scaled_lines, pens)
def _drawarcs(self, dc, arcs, pens):
scaled_arcs = map(self._arc_scaler, arcs)
dc.SetBrush(wx.TRANSPARENT_BRUSH)
for i in range(len(scaled_arcs)):
dc.SetPen(pens[i] if type(pens) == list else pens)
dc.DrawArc(*scaled_arcs[i])
def repaint_everything(self):
width = self.scale[0] * self.build_dimensions[0]
height = self.scale[1] * self.build_dimensions[1]
self.blitmap = wx.EmptyBitmap(width + 1, height + 1, -1)
dc = wx.MemoryDC()
dc.SelectObject(self.blitmap)
dc.SetBackground(wx.Brush((250, 250, 200)))
dc.Clear()
dc.SetPen(wx.Pen(wx.Colour(180, 180, 150)))
for grid_unit in self.grid:
if grid_unit > 0:
for x in xrange(int(self.build_dimensions[0] / grid_unit) + 1):
draw_x = self.scale[0] * x * grid_unit
dc.DrawLine(draw_x, 0, draw_x, height)
for y in xrange(int(self.build_dimensions[1] / grid_unit) + 1):
draw_y = self.scale[1] * (self.build_dimensions[1] -
y * grid_unit)
dc.DrawLine(0, draw_y, width, draw_y)
dc.SetPen(wx.Pen(wx.Colour(0, 0, 0)))
if not self.showall:
# Draw layer gauge
dc.SetBrush(wx.Brush((43, 144, 255)))
dc.DrawRectangle(width - 15, 0, 15, height)
dc.SetBrush(wx.Brush((0, 255, 0)))
if self.layers:
dc.DrawRectangle(
width - 14,
(1.0 - (1.0 * (self.layerindex + 1)) / len(self.layers)) *
height, 13, height - 1)
if self.showall:
for i in self.layers:
self._drawlines(dc, self.lines[i], self.pens[i])
self._drawarcs(dc, self.arcs[i], self.arcpens[i])
return
if self.layerindex < len(
self.layers) and self.layers[self.layerindex] in self.lines:
for layer_i in range(max(0, self.layerindex - 6), self.layerindex):
self._drawlines(dc, self.lines[self.layers[layer_i]],
self.fades[self.layerindex - layer_i - 1])
self._drawarcs(dc, self.arcs[self.layers[layer_i]],
self.fades[self.layerindex - layer_i - 1])
self._drawlines(dc, self.lines[self.layers[self.layerindex]],
self.pens[self.layers[self.layerindex]])
self._drawarcs(dc, self.arcs[self.layers[self.layerindex]],
self.arcpens[self.layers[self.layerindex]])
self._drawlines(dc, self.hilight, self.hlpen)
self._drawarcs(dc, self.hilightarcs, self.hlpen)
self.paint_hilights(dc)
dc.SelectObject(wx.NullBitmap)
def paint_hilights(self, dc=None):
if self.hilightqueue.empty() and self.hilightarcsqueue.empty():
return
hl = []
if not dc:
dc = wx.MemoryDC()
dc.SelectObject(self.blitmap)
while not self.hilightqueue.empty():
hl.append(self.hilightqueue.get_nowait())
self._drawlines(dc, hl, self.hlpen)
hlarcs = []
while not self.hilightarcsqueue.empty():
hlarcs.append(self.hilightarcsqueue.get_nowait())
self._drawarcs(dc, hlarcs, self.hlpen)
def paint(self, event):
if self.dirty:
self.dirty = 0
self.repaint_everything()
self.paint_hilights()
dc = wx.PaintDC(self)
dc.SetBackground(wx.Brush(self.bgcolor))
dc.Clear()
dc.DrawBitmap(self.blitmap, self.translate[0], self.translate[1])
if self.paint_overlay:
self.paint_overlay(dc)
def addfile(self, gcode):
self.clear()
self.add_parsed_gcodes(gcode)
max_layers = len(self.layers)
if hasattr(self.parent, "layerslider"):
self.parent.layerslider.SetRange(0, max_layers - 1)
self.parent.layerslider.SetValue(0)
# FIXME : there's code duplication going on there, we should factor it (but
# the reason addgcode is not factored as a add_parsed_gcodes([gline]) is
# because when loading a file there's no hilight, so it simply lets us not
# do the if hilight: all the time for nothing when loading a lot of lines
def add_parsed_gcodes(self, gcode):
def _y(y):
return self.build_dimensions[1] - (y - self.build_dimensions[4])
def _x(x):
return x - self.build_dimensions[3]
for layer_idx, layer in enumerate(gcode.all_layers):
has_move = False
for gline in layer:
if gline.is_move:
has_move = True
break
if not has_move:
continue
self.lines[layer.z] = []
self.pens[layer.z] = []
self.arcs[layer.z] = []
self.arcpens[layer.z] = []
self.layers.append(layer.z)
for gline in layer:
if not gline.is_move:
continue
target = self.lastpos[:]
target[0] = gline.current_x
target[1] = gline.current_y
target[2] = gline.current_z
target[5] = 0.0
target[6] = 0.0
if gline.e is not None:
if gline.relative_e:
target[3] += gline.e
else:
target[3] = gline.e
if gline.f is not None: target[4] = gline.f
if gline.i is not None: target[5] = gline.i
if gline.j is not None: target[6] = gline.j
start_pos = self.lastpos[:]
if gline.command in ["G0", "G1"]:
self.lines[layer.z].append(
(_x(start_pos[0]), _y(start_pos[1]), _x(target[0]),
_y(target[1])))
self.pens[layer.z].append(self.mainpen if target[3] != self
.lastpos[3] else self.travelpen)
elif gline.command in ["G2", "G3"]:
# startpos, endpos, arc center
arc = [
_x(start_pos[0]),
_y(start_pos[1]),
_x(target[0]),
_y(target[1]),
_x(start_pos[0] + target[5]),
_y(start_pos[1] + target[6])
]
if gline.command == "G2": # clockwise, reverse endpoints
arc[0], arc[1], arc[2], arc[3] = arc[2], arc[3], arc[
0], arc[1]
self.arcs[layer.z].append(arc)
self.arcpens[layer.z].append(self.arcpen)
self.lastpos = target
self.dirty = 1
self.Refresh()
def addgcode(self, gcode="M105", hilight=0):
gcode = gcode.split("*")[0]
gcode = gcode.split(";")[0]
gcode = gcode.lower().strip()
if not gcode:
return
gline = gcoder.Line(gcode)
split_raw = gcoder.split(gline)
gcoder.parse_coordinates(gline, split_raw, imperial=False)
def _y(y):
return self.build_dimensions[1] - (y - self.build_dimensions[4])
def _x(x):
return x - self.build_dimensions[3]
if gline.command not in ["G0", "G1", "G2", "G3"]:
return
start_pos = self.hilightpos[:] if hilight else self.lastpos[:]
target = start_pos[:]
target[5] = 0.0
target[6] = 0.0
if gline.x is not None: target[0] = gline.x
if gline.y is not None: target[1] = gline.y
if gline.z is not None: target[2] = gline.z
if gline.e is not None: target[3] = gline.e
if gline.f is not None: target[4] = gline.f
if gline.i is not None: target[5] = gline.i
if gline.j is not None: target[6] = gline.j
z = target[2]
if not hilight and z not in self.layers:
self.lines[z] = []
self.pens[z] = []
self.arcs[z] = []
self.arcpens[z] = []
self.layers.append(z)
if gline.command in ["G0", "G1"]:
line = [
_x(start_pos[0]),
_y(start_pos[1]),
_x(target[0]),
_y(target[1])
]
if not hilight:
self.lines[z].append((_x(start_pos[0]), _y(start_pos[1]),
_x(target[0]), _y(target[1])))
self.pens[z].append(self.mainpen if target[3] != self.
lastpos[3] else self.travelpen)
else:
self.hilight.append(line)
self.hilightqueue.put_nowait(line)
elif gline.command in ["G2", "G3"]:
# startpos, endpos, arc center
arc = [
_x(start_pos[0]),
_y(start_pos[1]),
_x(target[0]),
_y(target[1]),
_x(start_pos[0] + target[5]),
_y(start_pos[1] + target[6])
]
if gline.command == "G2": # clockwise, reverse endpoints
arc[0], arc[1], arc[2], arc[3] = arc[2], arc[3], arc[0], arc[1]
if not hilight:
self.arcs[z].append(arc)
self.arcpens[z].append(self.arcpen)
else:
self.hilightarcs.append(arc)
self.hilightarcsqueue.put_nowait(arc)
if not hilight:
self.lastpos = target
self.dirty = 1
else:
self.hilightpos = target
self.Refresh()
class LogCatAnalyzerThread():
""" Logger analyzer that will check input messages to check if
they validate some criteria
"""
def __init__(self, logger):
# Analyzer thread stop condition
self._stop_event = threading.Event()
# Messages to trigger
self.__messages_to_trigger = {}
# Lock object
self.__lock_message_triggered = threading.RLock()
# Internal buffer
self.__queue = Queue()
# Working thread
self.__analyzer_thread = None
# Logger to be used to output messages
self._logger = logger
# Delay to wait before processing new item in the queue
self.analyzer_loop_delay = 0.1
def stop(self):
self._stop_event.set()
if self.__analyzer_thread is not None:
try:
self.__analyzer_thread.join(5)
except (KeyboardInterrupt, SystemExit):
raise
except BaseException:
pass
finally:
del self.__analyzer_thread
self.__analyzer_thread = None
def start(self):
self._stop_event.clear()
self.__analyzer_thread = threading.Thread(target=self.__run)
self.__analyzer_thread.name = "LogCatAnalyzerThread"
self.__analyzer_thread.daemon = True
self.__analyzer_thread.start()
def push(self, line):
self.__queue.put_nowait(line)
def __run(self):
while not self._stop_event.is_set():
while not self.__queue.empty():
try:
line = self.__queue.get_nowait()
self.__analyze_line(line)
except Empty:
pass
self._stop_event.wait(self.analyzer_loop_delay)
def __analyze_line(self, line):
if line:
line = line.rstrip('\r\n')
# Check all messages to be triggered
self.__lock_message_triggered.acquire()
for trig_message in self.__messages_to_trigger:
if trig_message.startswith("regex:"):
reg_ex = trig_message.split("regex:")[1]
try:
if re.search(reg_ex, line) is not None:
# Message received, store log line
self.__messages_to_trigger[trig_message].append(
line)
except re.error as ex:
if self._logger is not None:
self._logger.error(
"Cannot compute regular expression \"%s\": %s"
% (reg_ex, ex))
elif line.find(trig_message) != -1:
# Message received, store log line
self.__messages_to_trigger[trig_message].append(line)
self.__lock_message_triggered.release()
def add_trigger_messages(self, messages):
for message in messages:
self.add_trigger_message(message)
def add_trigger_message(self, message):
""" Trigger a message
:type message: string
:param message: message to be triggered
"""
self.__lock_message_triggered.acquire()
self.__messages_to_trigger[message] = list()
self.__lock_message_triggered.release()
def remove_trigger_message(self, message):
""" Remove a triggered message
:type message: string
:param message: message to be removed
"""
if message in self.__messages_to_trigger:
self.__lock_message_triggered.acquire()
del self.__messages_to_trigger[message]
self.__lock_message_triggered.release()
def is_message_received(self, message, timeout):
""" Check if a message is received
:type message: string
:param message: message that we look for
:type timeout: int
:param timeout: time limit where we expect to receive the message
:return: Array of message received, empty array if nothing
:rtype: list
"""
remove_trigger_message = False
if message not in self.__messages_to_trigger:
self.add_trigger_message(message)
remove_trigger_message = True
messages_received = None
begin_time = time.time()
end_time = begin_time + float(timeout)
while (not messages_received) and (time.time() < end_time):
messages_received = self.get_message_triggered_status(message)
time.sleep(0.2)
if messages_received:
# Clone the list to return as remove trigger message
# is going to delete it
messages_received = list(messages_received)
if remove_trigger_message:
self.remove_trigger_message(message)
return messages_received
def get_message_triggered_status(self, message):
""" Get the status of a message triggered
:type message: string
:param message: message triggered
:return: Array of message received, empty array if nothing
:rtype: list
"""
if message in self.__messages_to_trigger:
return self.__messages_to_trigger[message]
else:
return None
def reset_trigger_message(self, message):
""" Reset triggered message
:type message: string
:param message: message to be reseted
"""
if message in self.__messages_to_trigger:
self.__lock_message_triggered.acquire()
self.__lock_message_triggered.release()
def create_step60(maindir, mbconnect=None, maxsongs=100, nfilesbuffer=0):
"""
Makes sure we have the similar artists to the top 100 most familiar
artists, and then go on with more similar artists.
INPUT
maindir - root directory of the Million Song dataset
mbconnect - open pg connection to Musicbrainz
maxsongs - max number of song per search (max=100)
nfilesbuffer - number of files we leave unfilled in the dataset
RETURN
the number of songs actually created
"""
# will contain artists TID that are done or already in the queue
artists_done = set()
# get all artists ids
artist_queue = Queue()
artists = get_most_familiar_artists(nresults=100)
n_most_familiars = len(artists)
npr.shuffle(artists)
for a in artists:
artists_done.add(a.id)
artist_queue.put_nowait(a)
# for each of them create all songs
cnt_created = 0
cnt_artists = 0
while not artist_queue.empty():
artist = artist_queue.get_nowait()
cnt_artists += 1
# CLOSED CREATION?
if CREATION_CLOSED:
break
if cnt_artists % 10 == 0:
nh5 = count_h5_files(maindir)
print 'found', nh5, 'h5 song files in', maindir
sys.stdout.flush()
if nh5 > TOTALNFILES - nfilesbuffer:
return cnt_created
# verbose
print 'doing artist', cnt_artists, '(pid=' + str(os.getpid()) + ')'
sys.stdout.flush()
# encode that artist unless it was done in step10
#if cnt_artists > n_most_familiars:
# we had to relaunch this function, lets not redo all the same artists over and over
if cnt_artists > 1000:
cnt_created += create_track_files_from_artist(maindir,
artist,
mbconnect=mbconnect,
maxsongs=maxsongs)
# get similar artists, add to queue
similars = get_similar_artists(artist)
if len(similars) == 0: continue
npr.shuffle(similars)
similars = similars[:
10] # we keep 10 at random, the radius of artists grows faster
# the thread dont redo the same artists over and over
# too bad for the artists we miss (if any...)
for a in similars:
if a.id in artists_done:
continue
artists_done.add(a.id)
artist_queue.put_nowait(a)
return cnt_created
class SplunkHandler(logging.Handler):
"""
A logging handler to send events to a Splunk Enterprise instance
running the Splunk HTTP Event Collector.
"""
instances = [] # For keeping track of running class instances
def __init__(self,
host,
port,
token,
index,
hostname=None,
source=None,
sourcetype='text',
verify=True,
timeout=60,
flush_interval=15.0,
queue_size=5000):
SplunkHandler.instances.append(self)
logging.Handler.__init__(self)
self.host = host
self.port = port
self.token = token
self.index = index
self.source = source
self.sourcetype = sourcetype
self.verify = verify
self.timeout = timeout
self.flush_interval = flush_interval
self.log_payload = ""
self.SIGTERM = False # 'True' if application requested exit
self.timer = None
self.testing = False # Used for slightly altering logic during unit testing
# It is possible to get 'behind' and never catch up, so we limit the queue size
self.queue = Queue(maxsize=queue_size)
if hostname is None:
self.hostname = socket.gethostname()
else:
self.hostname = hostname
# prevent infinite recursion by silencing requests and urllib3 loggers
logging.getLogger('requests').propagate = False
# and do the same for ourselves
logging.getLogger(__name__).propagate = False
# disable all warnings from urllib3 package
if not self.verify:
requests.packages.urllib3.disable_warnings()
# Start a worker thread responsible for sending logs
self.timer = Timer(self.flush_interval, self._splunk_worker)
self.timer.daemon = True # Auto-kill thread if main process exits
self.timer.start()
def emit(self, record):
record = self.format_record(record)
try:
# Put log message into queue; worker thread will pick up
self.queue.put_nowait(record)
except Full:
print("Log queue full; log data will be dropped.")
def format_record(self, record):
if self.source is None:
source = record.pathname
else:
source = self.source
current_time = time.time()
if self.testing:
current_time = None
params = {
'time': current_time,
'host': self.hostname,
'index': self.index,
'source': source,
'sourcetype': self.sourcetype,
'event': self.format(record),
}
return json.dumps(params, sort_keys=True)
def _splunk_worker(self):
queue_empty = True
# Pull everything off the queue.
while not self.queue.empty():
try:
item = self.queue.get(block=False)
self.log_payload = self.log_payload + item
self.queue.task_done()
except Empty:
pass
# If the payload is getting very long, stop reading and send immediately.
if not self.SIGTERM and len(self.log_payload) >= 524288: # 50MB
queue_empty = False
break
if self.log_payload:
url = 'https://%s:%s/services/collector' % (self.host, self.port)
try:
r = requests.post(
url,
data=self.log_payload,
headers={'Authorization': "Splunk %s" % self.token},
verify=self.verify,
timeout=self.timeout,
)
r.raise_for_status() # Throws exception for 4xx/5xx status
except Exception as e:
try:
print(traceback.format_exc())
print("Exception in Splunk logging handler: %s" % str(e))
except:
pass
self.log_payload = ""
# Restart the timer
timer_interval = self.flush_interval
if not self.SIGTERM:
if not queue_empty:
timer_interval = 1.0 # Start up again right away if queue was not cleared
self.timer = Timer(timer_interval, self._splunk_worker)
self.timer.daemon = True # Auto-kill thread if main process exits
self.timer.start()
def shutdown(self):
self.SIGTERM = True
self.timer.cancel(
) # Cancels the scheduled Timer, allows exit immediatley
# Send the remaining items that might be sitting in queue.
self._splunk_worker()
# Called when application exit imminent (main thread ended / got kill signal)
@atexit.register
def catch_exit():
for instance in SplunkHandler.instances:
try:
instance.shutdown()
except:
pass
class APRSUploader(object):
'''
Queued APRS Telemetry Uploader class
This performs uploads to the Habitat servers, and also handles generation of flight documents.
Incoming telemetry packets are fed into queue, which is checked regularly.
If a new callsign is sighted, a payload document is created in the Habitat DB.
The telemetry data is then converted into a UKHAS-compatible format, before being added to queue to be
uploaded as network speed permits.
If an upload attempt times out, the packet is discarded.
If the queue fills up (probably indicating no network connection, and a fast packet downlink rate),
it is immediately emptied, to avoid upload of out-of-date packets.
Note that this uploader object is intended to handle telemetry from multiple sondes
'''
# We require the following fields to be present in the incoming telemetry dictionary data
REQUIRED_FIELDS = [
'frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq',
'freq_float', 'datetime_dt'
]
def __init__(
self,
aprs_callsign='N0CALL',
aprs_passcode="00000",
object_name_override=None,
object_comment="RadioSonde",
position_report=False,
aprsis_host='rotate.aprs2.net',
aprsis_port=14580,
station_beacon=False,
station_beacon_rate=30,
station_beacon_position=(0.0, 0.0, 0.0),
station_beacon_comment="radiosonde_auto_rx SondeGate v<version>",
station_beacon_icon="/r",
synchronous_upload_time=30,
callsign_validity_threshold=5,
upload_queue_size=16,
upload_timeout=10,
inhibit=False):
""" Initialise an APRS Uploader object.
Args:
aprs_callsign (str): Callsign of the uploader, used when logging into APRS-IS.
aprs_passcode (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated,
is used to plot the listener's position on the Habitat map, both when this class is initialised, and
when a new sonde ID is observed.
object_name_override (str): Override the object name in the uploaded sentence with this value.
WARNING: This will horribly break the aprs.fi map if multiple sondes are uploaded simultaneously under the same callsign.
USE WITH CAUTION!!!
object_comment (str): A comment to go with the object. Various fields will be replaced with telmetry data.
position_report (bool): If True, upload positions as APRS position reports, otherwise, upload as an Object.
aprsis_host (str): APRS-IS Server to upload packets to.
aprsis_port (int): APRS-IS TCP port number.
station_beacon (bool): Enable beaconing of station position.
station_beacon_rate (int): Time delay between beacon uploads (minutes)
station_beacon_position (tuple): (lat, lon, alt), in decimal degrees, of the station position.
station_beacon_comment (str): Comment field for the station beacon. <version> will be replaced with the current auto_rx version.
station_beacon_icon (str): The APRS icon to be used, as the two characters (symbol table, symbol index), as per http://www.aprs.org/symbols.html
synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
and also acts as decimation on the number of sentences uploaded to APRS-IS.
callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
upload_queue_size (int): Maximum number of sentences to keep in the upload queue. If the queue is filled,
it will be emptied (discarding the queue contents).
upload_timeout (int): Timeout (Seconds) when performing uploads to APRS-IS. Default: 10 seconds.
inhibit (bool): Inhibit all uploads. Mainly intended for debugging.
"""
self.aprs_callsign = aprs_callsign
self.aprs_passcode = aprs_passcode
self.object_comment = object_comment
self.position_report = position_report
self.aprsis_host = aprsis_host
self.aprsis_port = aprsis_port
self.upload_timeout = upload_timeout
self.upload_queue_size = upload_queue_size
self.synchronous_upload_time = synchronous_upload_time
self.callsign_validity_threshold = callsign_validity_threshold
self.inhibit = inhibit
self.station_beacon = {
'enabled': station_beacon,
'position': station_beacon_position,
'rate': station_beacon_rate,
'comment': station_beacon_comment,
'icon': station_beacon_icon
}
if object_name_override is None:
self.object_name_override = "<id>"
else:
self.object_name_override = object_name_override
# Our two Queues - one to hold sentences to be upload, the other to temporarily hold
# input telemetry dictionaries before they are converted and processed.
self.aprs_upload_queue = Queue(upload_queue_size)
self.input_queue = Queue()
# Dictionary where we store sorted telemetry data for upload when required.
# Elements will be named after payload IDs, and will contain:
# 'count' (int): Number of times this callsign has been observed. Uploads will only occur when
# this number rises above callsign_validity_threshold.
# 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires,
# this queue will be dumped, and the most recent telemetry uploaded.
self.observed_payloads = {}
# Record of when we last uploaded a user station position to Habitat.
self.last_user_position_upload = 0
# Start the uploader thread.
self.upload_thread_running = True
self.upload_thread = Thread(target=self.aprs_upload_thread)
self.upload_thread.start()
# Start the input queue processing thread.
self.input_processing_running = True
self.input_thread = Thread(target=self.process_queue)
self.input_thread.start()
self.timer_thread_running = True
self.timer_thread = Thread(target=self.upload_timer)
self.timer_thread.start()
self.log_info("APRS Uploader Started.")
def aprsis_upload(self, source, packet, igate=False):
""" Upload a packet to APRS-IS
Args:
source (str): Callsign of the packet source.
packet (str): APRS packet to upload.
igate (boolean): If True, iGate the packet into APRS-IS
(i.e. use the original source call, but add SONDEGATE and our callsign to the path.)
"""
# If we are inhibited, just return immediately.
if self.inhibit:
self.log_info("Upload Inhibited: %s" % packet)
return True
# Generate APRS packet
if igate:
# If we are emulating an IGATE, then we need to add in a path, a q-construct, and our own callsign.
# We have the TOCALL field 'APRARX' allocated by Bob WB4APR, so we can now use this to indicate
# that these packets have arrived via radiosonde_auto_rx!
_packet = '%s>APRARX,SONDEGATE,TCPIP,qAR,%s:%s\n' % (
source, self.aprs_callsign, packet)
else:
# Otherwise, we are probably just placing an object, usually sourced by our own callsign
_packet = '%s>APRS:%s\n' % (source, packet)
# create socket & connect to server
_s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
_s.settimeout(self.upload_timeout)
try:
_s.connect((self.aprsis_host, self.aprsis_port))
# Send logon string
_logon = 'user %s pass %s vers VK5QI-AutoRX \n' % (
self.aprs_callsign, self.aprs_passcode)
_s.send(_logon.encode('ascii'))
# send packet
_s.send(_packet.encode('ascii'))
# close socket
_s.shutdown(0)
_s.close()
self.log_info("Uploaded to APRS-IS: %s" % _packet)
return True
except Exception as e:
self.log_error("Upload to APRS-IS Failed - %s" % str(e))
return False
def beacon_station_position(self):
''' Send a station position beacon into APRS-IS '''
if self.station_beacon['enabled']:
if (self.station_beacon['position'][0]
== 0.0) and (self.station_beacon['position'][1] == 0.0):
self.log_error(
"Station position is 0,0, not uploading position beacon.")
self.last_user_position_upload = time.time()
return
# Generate the station position packet
# Note - this is now generated as an APRS position report, for radiosondy.info compatability.
_packet = generate_station_object(
self.aprs_callsign,
self.station_beacon['position'][0],
self.station_beacon['position'][1],
self.station_beacon['comment'],
self.station_beacon['icon'],
position_report=True)
# Send the packet as an iGated packet.
self.aprsis_upload(self.aprs_callsign, _packet, igate=True)
self.last_user_position_upload = time.time()
def update_station_position(self, lat, lon, alt):
""" Update the internal station position record. Used when determining the station position by GPSD """
self.station_beacon['position'] = (lat, lon, alt)
def aprs_upload_thread(self):
''' Handle uploading of packets to Habitat '''
self.log_debug("Started APRS Uploader Thread.")
while self.upload_thread_running:
if self.aprs_upload_queue.qsize() > 0:
# If the queue is completely full, jump to the most recent telemetry sentence.
if self.aprs_upload_queue.qsize() == self.upload_queue_size:
while not self.aprs_upload_queue.empty():
_telem = self.aprs_upload_queue.get()
self.log_warning(
"Uploader queue was full - possible connectivity issue."
)
else:
# Otherwise, get the first item in the queue.
_telem = self.aprs_upload_queue.get()
# Convert to a packet.
try:
(_packet, _call) = telemetry_to_aprs_position(
_telem,
object_name=self.object_name_override,
aprs_comment=self.object_comment,
position_report=self.position_report)
except Exception as e:
self.log_error(
"Error converting telemetry to APRS packet - %s" %
str(e))
_packet = None
# Attempt to upload it.
if _packet is not None:
# If we are uploading position reports, the source call is the generated callsign
# usually based on the sonde serial number, and we iGate the position report.
# Otherwise, we upload APRS Objects, sourced by our own callsign, but still iGated via us.
if self.position_report:
self.aprsis_upload(_call, _packet, igate=True)
else:
self.aprsis_upload(self.aprs_callsign,
_packet,
igate=True)
else:
# Wait for a short time before checking the queue again.
time.sleep(0.1)
self.log_debug("Stopped APRS Uploader Thread.")
def upload_timer(self):
""" Add packets to the habitat upload queue if it is time for us to upload. """
while self.timer_thread_running:
if int(time.time()) % self.synchronous_upload_time == 0:
# Time to upload!
for _id in self.observed_payloads.keys():
# If no data, continue...
if self.observed_payloads[_id]['data'].empty():
continue
else:
# Otherwise, dump the queue and keep the latest telemetry.
while not self.observed_payloads[_id]['data'].empty():
_telem = self.observed_payloads[_id]['data'].get()
# Attept to add it to the habitat uploader queue.
try:
self.aprs_upload_queue.put_nowait(_telem)
except Exception as e:
self.log_error(
"Error adding sentence to queue: %s" % str(e))
# Sleep a second so we don't hit the synchronous upload time again.
time.sleep(1)
else:
# Not yet time to upload, wait for a bit.
time.sleep(0.1)
def process_queue(self):
""" Process packets from the input queue.
This thread handles packets from the input queue (provided by the decoders)
Packets are sorted by ID, and a dictionary entry is created.
"""
while self.input_processing_running:
# Process everything in the queue.
while self.input_queue.qsize() > 0:
# Grab latest telem dictionary.
_telem = self.input_queue.get_nowait()
_id = _telem['id']
if _id not in self.observed_payloads:
# We haven't seen this ID before, so create a new dictionary entry for it.
self.observed_payloads[_id] = {'count': 1, 'data': Queue()}
self.log_debug(
"New Payload %s. Not observed enough to allow upload."
% _id)
# However, we don't yet add anything to the queue for this payload...
else:
# We have seen this payload before!
# Increment the 'seen' counter.
self.observed_payloads[_id]['count'] += 1
# If we have seen this particular ID enough times, add the data to the ID's queue.
if self.observed_payloads[_id][
'count'] >= self.callsign_validity_threshold:
# Add the telemetry to the queue
self.observed_payloads[_id]['data'].put(_telem)
else:
self.log_debug(
"Payload ID %s not observed enough to allow upload."
% _id)
if (time.time() - self.last_user_position_upload
) > self.station_beacon['rate'] * 60:
self.beacon_station_position()
time.sleep(0.1)
def add(self, telemetry):
""" Add a dictionary of telemetry to the input queue.
Args:
telemetry (dict): Telemetry dictionary to add to the input queue.
"""
# Discard any telemetry which is indicated to be encrypted.
if 'encrypted' in telemetry:
if telemetry['encrypted'] == True:
return
# Check the telemetry dictionary contains the required fields.
for _field in self.REQUIRED_FIELDS:
if _field not in telemetry:
self.log_error("JSON object missing required field %s" %
_field)
return
# Add it to the queue if we are running.
if self.input_processing_running:
self.input_queue.put(telemetry)
else:
self.log_error("Processing not running, discarding.")
def close(self):
''' Shutdown uploader and processing threads. '''
self.log_debug("Waiting for threads to close...")
self.input_processing_running = False
self.timer_thread_running = False
self.upload_thread_running = False
# Wait for all threads to close.
if self.upload_thread is not None:
self.upload_thread.join()
if self.timer_thread is not None:
self.timer_thread.join()
if self.input_thread is not None:
self.input_thread.join()
def log_debug(self, line):
""" Helper function to log a debug message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.debug("APRS-IS - %s" % line)
def log_info(self, line):
""" Helper function to log an informational message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.info("APRS-IS - %s" % line)
def log_error(self, line):
""" Helper function to log an error message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.error("APRS-IS - %s" % line)
def log_warning(self, line):
""" Helper function to log a warning message with a descriptive heading.
Args:
line (str): Message to be logged.
"""
logging.warning("APRS-IS - %s" % line)
class BaseDriver(object):
browser = None
def __init__(self, config=None):
self.driver = None
self.config = config or {}
self.alive = False
self.ready_condition = threading.Condition()
self.task_queue = Queue()
# turn of logging
selenium_logger = logging.getLogger(
'selenium.webdriver.remote.remote_connection')
selenium_logger.setLevel(logging.WARNING)
def _initialize_driver(self, driver, config):
# config.setdefault('window_size', {'width': 1200, 'height': 800})
# driver.set_window_size(config['window_size']['width'], config['window_size']['height'])
if not config.setdefault('cookies', False):
driver.delete_all_cookies()
self.initialize_driver(driver, config)
def create_driver(self, config):
pass
def initialize_driver(self, driver, config):
pass
def wait_until_ready(self):
with self.ready_condition:
self.ready_condition.wait()
def start(self, wait=False):
t = threading.Thread(target=self._start)
t.daemon = True
t.start()
if wait:
self.wait_until_ready()
def _start(self):
self.driver = self.create_driver(self.config)
self._initialize_driver(self.driver, self.config)
log.info("Browser initialized")
with self.ready_condition:
self.ready_condition.notifyAll()
self.alive = True
while self.alive:
task = self.task_queue.get()
self.get(task)
def get(self, url):
try:
# self.driver.set_page_load_timeout(50)
self.driver.get(url)
except TimeoutException:
pass
def queue_url(self, url):
self.task_queue.put_nowait(url)
def close(self):
self.driver.close()
def stop(self):
self.driver.stop_client()
def pid(self):
import psutil
gecko_pid = self.driver.service.process.pid
return psutil.Process(gecko_pid).children()[0].pid
def quit(self):
self.alive = False
self.driver.quit()
class TipWidget(QWidget):
TIMER_INTERVAL = 10
TIMER_COUNTER_STEP = 30
MAX_OPACITY_TIME = 100
TEXT_TYPE_INFO = 1
TEXT_TYPE_WARNING = 2
TEXT_TYPE_ERROR = 3
INFO_ICON_PATH = ":/tip/icons/tip/info.png"
WARNING_ICON_PATH = ":/tip/icons/tip/warning.png"
ERROR_ICON_PATH = ":/tip/icons/tip/error.png"
SHORT_TIME = 2000
LONG_TIME = 4000
MAX_QUEUE_SIZE = 20
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.setWindowFlags(Qt.FramelessWindowHint | Qt.Dialog)
self.ui = Ui_TipWidget()
self.ui.setupUi(self)
self.timer = QTimer()
self.timer.timeout.connect(self.__onTimeout)
self.queue = Queue(self.MAX_QUEUE_SIZE)
self.timerCounter = 0
self.maxOpacityCounter = 0
self.setWindowOpacity(0)
def __makeText(self, text, type=TEXT_TYPE_INFO, t=SHORT_TIME):
if self.timer.isActive():
if not self.queue.full():
self.queue.put_nowait((text, type, t))
return
iconPath = self.INFO_ICON_PATH
if type == self.TEXT_TYPE_WARNING:
iconPath = self.WARNING_ICON_PATH
elif type == self.TEXT_TYPE_ERROR:
iconPath = self.ERROR_ICON_PATH
self.ui.image_label.setPixmap(QPixmap(iconPath))
self.ui.text_label.setText(text)
self.timer.start(self.TIMER_INTERVAL)
self.show()
def makeInfoText(self, text="", t=SHORT_TIME):
self.__makeText(text, self.TEXT_TYPE_INFO, t)
def makeWarningText(self, text="", t=SHORT_TIME):
self.__makeText(text, self.TEXT_TYPE_WARNING, t)
def makeErrorText(self, text="", t=LONG_TIME):
self.__makeText(text, self.TEXT_TYPE_ERROR, t)
def __onTimeout(self):
self.timerCounter += self.TIMER_COUNTER_STEP
if self.timerCounter >= self.SHORT_TIME:
self.__onFinished()
return
if self.timerCounter < self.SHORT_TIME / 2:
self.setWindowOpacity(1.0 / (self.SHORT_TIME / 2) *
self.timerCounter)
else:
self.maxOpacityCounter += 1
if self.maxOpacityCounter < self.MAX_OPACITY_TIME:
self.timerCounter -= self.TIMER_COUNTER_STEP
return
self.setWindowOpacity(1.0 / (self.SHORT_TIME / 2) *
(self.SHORT_TIME - self.timerCounter))
def __onFinished(self):
self.close()
self.timer.stop()
self.maxOpacityCounter = 0
self.timerCounter = 0
if not self.queue.empty():
args = self.queue.get()
self.__makeText(args[0], args[1], args[2])
def ibfWorker(publisherAddr,
sinkAddr,
cells,
k,
m,
blockSz,
secret,
public,
w,
pbSize,
hashProdOne=True):
workerName = mp.current_process().name
context = zmq.Context()
taskQ = Queue()
endQ = Queue()
taskLock = threading.Lock()
sharedTimer = ExpTimer()
threadsNumber = 1 #ALWAYS 1! This is here not to slow down the subscriber
subSocket = context.socket(zmq.SUB)
subSocket.setsockopt(zmq.SUBSCRIBE, b'work')
subSocket.setsockopt(zmq.SUBSCRIBE, b'end')
subSocket.connect(publisherAddr)
hashFunc = [Hash1, Hash2, Hash3, Hash4, Hash5, Hash6]
cells = set(cells)
print "PreprocWorker", workerName, "initiated"
results = {
"worker": workerName,
"cells": {},
"w": {},
"timers": None,
"timerNames": []
}
for i in xrange(threadsNumber):
taskThread = threading.Thread(
target=ibfTask,
args=(taskQ, endQ, results, workerName, k, m, hashFunc, blockSz,
secret, public, cells, i, taskLock, sharedTimer, pbSize))
taskThread.daemon = True
taskThread.start()
for i in range(m):
results["cells"][i] = Cell(0, blockSz)
while True:
try:
workItem = subSocket.recv_multipart()
if workItem[0] == 'end':
for i in xrange(threadsNumber):
taskQ.put_nowait("end")
finishedTaskThreads = set()
while len(finishedTaskThreads) != threadsNumber:
tName = endQ.get(True)
endQ.task_done()
finishedTaskThreads.add(tName)
results["timers"] = sharedTimer
for i in range(m):
if results["cells"][i].count == 0:
del results["cells"][i]
rpc = RpcPdrClient(context)
inMsg = rpc.rpcAddress(sinkAddr, results)
if inMsg == "ACK":
break
else:
taskQ.put_nowait(workItem[1])
except zmq.ZMQError as e:
print "ZMQ Exception", str(e)
except (RuntimeError, TypeError, NameError) as e:
print "Error", str(e)
class SerialWriterThread():
"""
Logger based on serial utility
"""
def __init__(self, logger):
# Output file
self._output_file_path = None
# Reader thread stop condition
self._stop_event = threading.Event()
# Internal buffer
self._queue = Queue()
self.__writer_thread = None
# Logger
self._logger = logger
def stop(self):
"""
Stop the writer thread
"""
self._stop_event.set()
if self.__writer_thread is not None:
try:
self.__writer_thread.join(5)
except (KeyboardInterrupt, SystemExit):
raise
except BaseException:
pass
finally:
del self.__writer_thread
self.__writer_thread = None
return
def start(self):
"""
Start the write thread
"""
self._stop_event.clear()
self.__writer_thread = threading.Thread(target=self._run)
self.__writer_thread.name = "SerialWriterThread"
self.__writer_thread.daemon = True
self.__writer_thread.start()
def push(self, line):
"""
Push data in the internal queue
:type line: string
:param line: data to be written
"""
self._queue.put_nowait(line)
def set_output_path(self, output_path):
"""
Set stdout file path
:type output_path: string
:param output_path: path of the log file to be created
"""
self._output_file_path = output_path
def _run(self):
"""
Runner thread method
"""
# Create the output file if output file was specified
if self._output_file_path is None:
self._logger.info("No file specified for serial logger output")
self._output_file_path = strftime(
"_%Y-%m-%d_%Hh%M.%S") + "_serial.log"
self._logger.info("%s will be used." % self._output_file_path)
while not self._stop_event.is_set():
try:
with open(self._output_file_path, "ab") as output_stream:
while not self._queue.empty():
try:
line = self._queue.get_nowait()
if len(line) > 0:
output_stream.write(line)
output_stream.flush()
except Exception as e: # pylint: disable=W0703
self._logger.error(str(e))
except (IOError, OSError, TypeError) as e:
self._logger.error(str(e))
break
class JsonRpcServer(object):
log = logging.getLogger("jsonrpc.JsonRpcServer")
def __init__(self, bridge, threadFactory=None):
self._bridge = bridge
self._workQueue = Queue()
self._threadFactory = threadFactory
def queueRequest(self, req):
self._workQueue.put_nowait(req)
def _serveRequest(self, ctx, req):
mangledMethod = req.method.replace(".", "_")
logLevel = logging.DEBUG
if mangledMethod in ('Host_getVMList', 'Host_getAllVmStats',
'Host_getStats', 'StorageDomain_getStats',
'VM_getStats', 'Host_fenceNode'):
logLevel = logging.TRACE
self.log.log(logLevel, "Calling '%s' in bridge with %s", req.method,
req.params)
try:
method = getattr(self._bridge, mangledMethod)
except AttributeError:
if req.isNotification():
return
ctx.requestDone(
JsonRpcResponse(None, JsonRpcMethodNotFoundError(), req.id))
return
try:
params = req.params
server_address = ctx.client.get_local_address()
self._bridge.register_server_address(server_address)
if isinstance(req.params, list):
res = method(*params)
else:
res = method(**params)
self._bridge.unregister_server_address()
except JsonRpcError as e:
ctx.requestDone(JsonRpcResponse(None, e, req.id))
except Exception as e:
self.log.exception("Internal server error")
ctx.requestDone(
JsonRpcResponse(None, JsonRpcInternalError(str(e)), req.id))
else:
res = True if res is None else res
self.log.log(logLevel, "Return '%s' in bridge with %s", req.method,
res)
ctx.requestDone(JsonRpcResponse(res, None, req.id))
@traceback(on=log.name)
def serve_requests(self):
while True:
self.log.debug("Waiting for request")
obj = self._workQueue.get()
if obj is None:
break
client, msg = obj
self._parseMessage(client, msg)
def _parseMessage(self, client, msg):
ctx = _JsonRpcServeRequestContext(client)
try:
rawRequests = json.loads(msg)
except:
ctx.addResponse(JsonRpcResponse(None, JsonRpcParseError(), None))
ctx.sendReply()
return
if isinstance(rawRequests, list):
# Empty batch request
if len(rawRequests) == 0:
ctx.addResponse(
JsonRpcResponse(None, JsonRpcInvalidRequestError(), None))
ctx.sendReply()
return
else:
# From this point on we know it's always a list
rawRequests = [rawRequests]
# JSON Parsed handling each request
requests = []
for rawRequest in rawRequests:
try:
req = JsonRpcRequest.fromRawObject(rawRequest)
requests.append(req)
except JsonRpcError as err:
ctx.addResponse(JsonRpcResponse(None, err, None))
except:
ctx.addResponse(
JsonRpcResponse(None, JsonRpcInternalError(), None))
ctx.setRequests(requests)
# No request was built successfully or is only notifications
if ctx.counter == 0:
ctx.sendReply()
for request in requests:
self._runRequest(ctx, request)
def _runRequest(self, ctx, request):
if self._threadFactory is None:
self._serveRequest(ctx, request)
else:
self._threadFactory(partial(self._serveRequest, ctx, request))
def stop(self):
self.log.info("Stopping JsonRPC Server")
self._workQueue.put_nowait(None)
