class Loader(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.setDaemon(True)
self.queue = Queue(5)
self.killed = False
self.lock = threading.Lock()
self.locks = {}
def load(self, filename, callback):
self.locks[filename] = threading.Condition()
self.queue.put((filename, callback, self.locks[filename]))
def full(self):
return self.queue.full()
def get_lock(self, filename):
self.lock.acquire()
try:
return self.locks[filename]
finally:
self.lock.release()
def run(self):
while not self.killed:
filename, callback, lock = self.queue.get()
lock.acquire()
def discovered(d, is_media):
callback(d, is_media)
lock.acquire()
lock.notifyAll()
lock.release()
d = discoverer.Discoverer(filename)
d.connect('discovered', discovered)
d.discover()
lock.wait()
lock.release()
self.lock.acquire()
del self.locks[filename]
self.lock.release()
def wait(self, filename):
'''Wait for metadata to finish loading'''
if self.queue.full():
return
self.lock.acquire()
if filename not in self.locks:
self.lock.release()
return
condition = self.locks[filename]
self.lock.release()
condition.acquire()
condition.wait()
condition.release()
def kill(self):
self.killed = True
def test_queue():
q = Queue()
print dir(q)
q.put('a')
q.put('b')
print q.queue
print q.get()
print q.queue
print q.empty()
print q.full()
lock = threading.Lock()
class ThreadPool: def __init__(self, num_threads): self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks) def add_task(self, func, *args, **kargs): self.tasks.put((func, args, kargs)) print self.tasks.full() def wait_completion(self): self.tasks.join()
class VideoStream(object):
def __init__(self, url, queueSize=4, mode='buffer'):
from Queue import Queue
self.stopped = False
self.frameBuffer = Queue(maxsize=queueSize)
self.mode = mode
self.stream = cv2.VideoCapture(url)
if cv2_version == '3':
self.stream.set(cv2.CAP_PROP_BUFFERSIZE, 3)
def start(self):
""" start a thread to read frames from the file video stream. """
from threading import Thread
t = Thread(target=self.update, args=())
t.daemon = True
t.start()
return self
def update(self):
""" capture frame from stream and add it to queue in a loop until eos """
while self.stream.isOpened():
# if the thread stop indicator variable is set, stop capturing
if self.stopped:
return
# otherwise, ensure the queue has room in it and add frame to it
if not self.frameBuffer.full():
(grabbed, frame) = self.stream.read()
if not grabbed:
self.stop()
return
self.frameBuffer.put(frame)
# for stream mode, stash the last frame in the queue if queue is full
if self.mode == 'stream' and self.frameBuffer.full():
self.frameBuffer.get()
self.stop()
return
def read(self):
""" returns next frame in the queue. """
return self.frameBuffer.get()
def more(self):
""" checks if there are still frames in the queue. """
return self.frameBuffer.qsize() > 0
def stop(self):
""" Stops the videostream thread """
self.stopped = True
def testQueue(self):
queue = Queue(5)
self.assertEqual(queue.empty(), True)
with self.assertRaises(Exception):
queue.dequeue()
queue.enqueue(12)
queue.enqueue(10)
queue.enqueue(17)
self.assertEqual(queue.arr, [12, 10, 17, None, None])
queue.enqueue(7)
queue.enqueue(21)
self.assertEqual(queue.full(), True)
queue.dequeue()
queue.dequeue()
self.assertEqual(queue.arr, [None, None, 17, 7, 21])
queue.dequeue()
self.assertEqual(queue.dequeue(), 7)
queue.dequeue()
self.assertEqual(queue.empty(), True)
class AntiFlapping(object):
"""
AntiFlapping class to process event in a timely maneer
"""
def __init__(self, window):
self.window = window
self.tasks = Queue(maxsize=1)
self._window_ended = True
self._thread = Thread(name="AntiFlapping", target=self._run)
self._thread.start()
def newEvent(self, func, kwargs={}):
"""
newEvent Triggered.
"""
if not self.tasks.full() and self._window_ended:
self.tasks.put({'func': func, 'args':kwargs})
def _run(self):
"""
internal runloop that will fire tasks in order.
"""
while True:
task = self.tasks.get()
self._window_ended = False
sleep(self.window)
self._window_ended = True
if task['args']:
task['func'](**task['args'])
else:
task['func']()
class AntiFlapping(object):
"""
AntiFlapping class to process event in a timely maneer
"""
def __init__(self, window):
self.window = int(window)
self.tasks = Queue(maxsize=1)
self._window_ended = True
self._thread = Thread(name="AntiFlapping", target=self._run)
self._thread.start()
def newEvent(self, func, kwargs={}):
"""
newEvent Triggered.
"""
if not self.tasks.full() and self._window_ended:
self.tasks.put({'func': func, 'args': kwargs})
def _run(self):
"""
internal runloop that will fire tasks in order.
"""
while True:
task = self.tasks.get()
self._window_ended = False
sleep(self.window)
self._window_ended = True
if task['args']:
task['func'](**task['args'])
else:
task['func']()
class ConnectionKeeper(cherrypy.process.plugins.SimplePlugin):
def __init__(self, bus, factory, description):
self.bus = bus
self.description = description
self.factory = factory
self.connection_queue = Queue(10)
cherrypy.process.plugins.SimplePlugin.__init__(self, bus)
def start(self):
self.bus.log("Populating connection queue for %s" % self.description)
while not self.connection_queue.full():
self.connection_queue.put(self.factory())
def stop(self):
self.bus.log("Cleaning up connections for %s" % self.description)
while not self.connection_queue.empty():
self.connection_queue.get(False)
def connect(self):
return self.connection_queue.get()
def disconnect(self, cxn):
self.connection_queue.put(cxn)
class CmdVelCH:
def __init__(self):
rospy.init_node('cmd_vel_command_handler')
# Publisher to Differential Drive robot controller
self.cmd_vel_pub = rospy.Publisher('cmd_vel', Twist, queue_size=10)
ac_rate = rospy.get_param("~action_cycle_rate")
self.rate = rospy.Rate(ac_rate)
# Subscriber to Velocity Command coming from Environment
rospy.Subscriber('env_cmd_vel',
Twist,
self.callback_env_cmd_vel,
queue_size=1)
self.msg = Twist()
# Queue with maximum size 1
self.queue = Queue(maxsize=1)
def callback_env_cmd_vel(self, data):
try:
# Add to the Queue the next command to execute
self.queue.put(data)
except:
pass
def cmd_vel_publisher(self):
while not rospy.is_shutdown():
# If a command from the environment is waiting to be executed,
# publish the command, otherwise publish zero velocity message
if self.queue.full():
self.cmd_vel_pub.publish(self.queue.get())
else:
self.cmd_vel_pub.publish(Twist())
self.rate.sleep()
class FileVideoStream:
def __init__(self, path, queueSize=128):
# initialize the file video stream along with the boolean
# used to indicate if the thread should be stopped or not
# self.videoname,_ = os.path.splitext(path)
self.stream = cv2.VideoCapture(path)
self.stopped = False
self.image_name = None
self.sequence = self.stream.get(cv2.CAP_PROP_POS_FRAMES)
# initialize the queue used to store frames read from
# the video file
self.Q = Queue(maxsize=queueSize)
def start(self):
# start a thread to read frames from the file video stream
t = Thread(target=self.update, args=())
t.daemon = True
t.start()
return self
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
return
# otherwise, ensure the queue has room in it
if not self.Q.full():
# read the next frame from the file
(grabbed, frame) = self.stream.read()
current_index = self.stream.get(cv2.CAP_PROP_POS_FRAMES)
self.sequence = current_index
self.temporal_location = self.stream.get(cv2.CAP_PROP_POS_MSEC)
self.image_name = "img-%d.jpg" % current_index
# if the `grabbed` boolean is `False`, then we have
# reached the end of the video file
if not grabbed:
self.stop()
return
# add the frame to the queue
self.Q.put(frame)
def read(self):
# return next frame in the queue
return self.Q.get()
def more(self):
# return True if there are still frames in the queue
return self.Q.qsize() > 0
def get_image_name(self):
return self.image_name
def stop(self):
# indicate that the thread should be stopped
self.stopped = True
class HashQueue(object):
def __init__(self, maxsize=100):
self.maxsize = maxsize
self.queue = Queue(maxsize)
self.data = {}
def put(self, key, value):
if self.queue.full():
item = self.queue.get()
self.data.pop(item, None)
self.queue.put(key)
self.data[key] = value
def get(self, key):
return self.data.get(key, '')
def status(self, interval=30):
counter = 0
for message, timestamp in self.data.values():
if timestamp > dt.now() - timedelta(seconds=interval):
counter += 1
return counter
class BaseSinker(threading.Thread):
def __init__(self, filename, timer=10):
threading.Thread.__init__(self)
self.queue = Queue()
self.filename = filename
self.timer = timer
self.establish_date = arrow.utcnow().to('Asia/Shanghai').strftime(
"%Y-%m-%d")
if not os.path.exists("Flows"):
os.mkdir("Flows")
logger.warning('Directory(Flows) not exist, re-build.')
if os.path.exists("Flows/{}.out".format(filename)):
timestamp = time.localtime(
os.stat("Flows/{}.out".format(filename)).st_ctime)
create_date = "{}-{}-{}".format(timestamp[0], timestamp[1],
timestamp[2])
if time.strptime(self.establish_date, '%Y-%m-%d') != \
time.strptime(create_date, '%Y-%m-%d'):
shutil.move("Flows/{0}.out".format(filename),
"Flows/{0}_{1}.out".format(filename, create_date))
logger.info('New flow file({0}.out) created'.format(filename))
def run(self):
logger.info('Message sinker started.')
while True:
current_date = arrow.utcnow().to('Asia/Shanghai').strftime(
"%Y-%m-%d")
if current_date != self.establish_date:
shutil.move(
"Flows/{0}.out".format(self.filename),
"Flows/{0}_{1}.out".format(self.filename,
self.establish_date))
self.establish_date = current_date
self.sink()
time.sleep(self.timer)
def sink(self):
pass
def qsize(self):
return self.queue.qsize()
def empty(self):
return self.queue.empty()
def full(self):
return self.queue.full()
def put(self, item, block=True, timeout=None):
self.queue.put(item, block, timeout)
def put_nowait(self, item):
return self.queue.put(item, False)
def get(self, block=True, timeout=None):
return self.queue.get(block, timeout)
def get_nowait(self):
return self.queue.get(False)
def create_queue(raw_range):
range_queue = Queue(50)
for foo in raw_range:
if range_queue.full():
print("Warning. queue is full(max size is 50)")
break
range_queue.put(foo, 1)
return range_queue
def start_servers(self):
# Prepare to start the servers
server_list = self._config['servers']
server_count = len(server_list)
self.notice("Starting " + str(server_count) + " servers\n")
wait = Queue(maxsize=server_count)
# Add servers
for server_config in server_list:
handler = server_config['handler']
server_module = import_module("servers", handler)
if server_module is None:
self.error("Invalid handler " + handler)
sys.exit(2)
server = None
if server_config['mode'] == "http":
print("Starting http on port " + str(server_config['port']))
server = server_module(server_config['domain'],
int(server_config['port']),
server_config['timeout'], wait,
self.__loggers)
elif server_config['mode'] == "https":
if "cert_path" not in server_config:
self.error("cert_path not set for https " + handler)
sys.exit(3)
print("Starting https on port " + str(server_config['port']))
server = server_module(server_config['domain'],
int(server_config['port']),
server_config['timeout'], wait,
self.__loggers,
server_config['cert_path'])
server.start()
self._servers.append(server)
# Wait until they indicate they have bound
while not wait.full():
pass
wait = None
# Drop privileges
self.drop_privileges()
# Wait until told to exit
try:
while True:
time.sleep(2)
except KeyboardInterrupt:
print("\nShutting down...")
# for server in self._servers:
# server.stop()
# server.join()
sys.exit(0)
class Pool(object):
'''一个数据库连接池'''
def __init__(self,
maxActive=5,
maxWait=None,
init_size=0,
db_type="kafka",
**config):
self.__freeConns = Queue(maxActive)
self.maxWait = maxWait
self.db_type = db_type
self.config = config
if init_size > maxActive:
init_size = maxActive
for i in range(init_size):
self.free(self._create_conn())
def __del__(self):
print("__del__ Pool..")
self.release()
def release(self):
'''释放资源，关闭池中的所有连接'''
print("release Pool..")
while self.__freeConns and not self.__freeConns.empty():
con = self.get()
con.release()
self.__freeConns = None
def _create_conn(self):
'''创建连接 '''
if self.db_type in dbcs:
return dbcs[self.db_type](**self.config)
def get(self, timeout=None):
'''获取一个连接
@param timeout:超时时间
'''
if timeout is None:
timeout = self.maxWait
conn = None
if self.__freeConns.empty(): # 如果容器是空的，直接创建一个连接
conn = self._create_conn()
else:
conn = self.__freeConns.get(timeout=timeout)
conn.pool = self
return conn
def free(self, conn):
'''将一个连接放回池中
@param conn: 连接对象
'''
conn.pool = None
if (self.__freeConns.full()): # 如果当前连接池已满，直接关闭连接
conn.release()
return
self.__freeConns.put_nowait(conn)
class MessageChannel(object):
"""
Abstract base-class of all message channel types.
Provides functionality and attribute initialization
and management common to all channel types. Considered
"abstract" because implementation of notify() raises
exception and must be overridden, and because there
is no API method defined for getting messages out of the
channel.
"""
implements(IMessageChannel)
def __init__(self, name, host=None):
self.name = name
self.host = host
self.added = Counter()
self.removed = Counter()
self.messages = Queue()
super(MessageChannel, self).__init__()
def getname(self):
return self.name
def geturl(self):
return "/".join([self.host.geturl(), self.getname()])
def send(self, message):
self.messages.put(message)
self.added.increment()
self.notify()
def receive(self, blocking=True, timeout=None):
try:
message = self.messages.get(blocking, timeout)
except Empty:
message = None
else:
self.removed.increment()
return message
def full(self):
return self.messages.full()
def empty(self):
return self.messages.empty()
def attach(self, listener):
raise TypeError("notify() must be overridden")
def detach(self, listener):
raise TypeError("notify() must be overridden")
def subscribe(self, channel):
raise TypeError("notify() must be overridden")
def unsubscribe(self, channel):
raise TypeError("notify() must be overridden")
def notify(self):
raise TypeError("notify() must be overridden")
def __str__(self):
descriptors = ["%s('%s')" % (type(self).__name__, self.getname())]
descriptors.append('+%d' % self.added.get())
descriptors.append('-%d' % self.removed.get())
return " ".join(descriptors)
def __repr__(self):
return '<%s at %#x>' % (self, id(self))
class ThreadStream:
def __init__(self, path=0, queueSize=128):
self.stream = cv2.VideoCapture(path)
self.stopped = True
self.Q = Queue(maxsize=queueSize)
def start(self):
self.stopped = False
t = Thread(target=self.update, args=())
t.daemon = True
t.start()
return self
def update(self):
while True:
if (self.stopped):
return
if not self.Q.full():
success, frame = self.stream.read()
if (not success):
self.stop()
return
scale_percent = 50
#calculate the 50 percent of original dimensions
width = int(frame.shape[1] * scale_percent / 100)
height = int(frame.shape[0] * scale_percent / 100)
# dsize
dsize = (width, height)
# resize image
frame = cv2.resize(frame, dsize)
self.Q.put(frame)
else:
print("queue full")
def read(self):
return self.Q.get(block=True, timeout=10)
def more(self):
if (self.Q.qsize() <= 0):
print("Queue Empty")
return not self.stopped
else:
return True
def stop(self):
self.stopped = True
self.stream.release()
self.Q.queue.clear()
class VideoStream:
def __init__(self,
video_source,
video_width,
video_height,
video_fps,
queue_size=1):
self.video_fps = video_fps
vc = cv2.VideoCapture(video_source)
if hasattr(cv2, 'cv'):
vc.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, video_width)
vc.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, video_height)
vc.set(cv2.cv.CV_CAP_PROP_FPS, video_fps)
else:
vc.set(cv2.CAP_PROP_FRAME_WIDTH, video_width)
vc.set(cv2.CAP_PROP_FRAME_HEIGHT, video_height)
vc.set(cv2.CAP_PROP_FPS, video_fps)
self.stream = vc
self.stopped = False
self.queue = Queue(maxsize=queue_size)
self.thread = Thread(target=self.update, args=())
self.thread.daemon = True
self.thread.start()
def update(self):
while True:
if self.stopped:
break
(flg, frame) = self.stream.read()
if not flg:
Exception("Video capture is wrong")
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
if self.queue.full():
time.sleep(1 / float(self.video_fps))
else:
if not self.queue.empty():
self.queue.get()
self.queue.put(frame)
else:
self.queue.put(frame)
self.stream.release()
def read(self):
return self.queue.get()
def release(self):
self.stopped = True
self.thread.join()
class _RingBuffer():
def __init__(self, size):
self._buffer = Queue(maxsize=size)
def put(self, item):
if self._buffer.full():
#If the buffer is full, discard the oldest element to make room
self._buffer.get()
self._buffer.put(item)
def get(self, block=True):
return self._buffer.get(block=block)
def __start_scan_thread(self, target):
"""
Creates a new scan thread, if max_threads allows to.
Every scan thread is assigned a queue that contains the targets to
scan. For every element of the queue, the scan callback function is
called.
input:
target a new target to scan. basically the result of cUtil.mkIP2
"""
self.__threads = filter(lambda t: t[0].is_alive(), self.__threads)
func = self.__start_scan_thread.__func__
def inc_counter():
"""
in order to not constantly add a target to the first queue,
we need a counter, that points to the next queue to add the
target. as consequence, every queue should have round about the
same qsize
"""
try:
if func.counter < len(self.__threads)-1:
func.counter += 1
else:
func.counter = 0
except AttributeError:
func.counter = 0
if len(self.__threads) < self.max_threads:
# create a queue to be filled with targets
q = Queue(maxsize=1000)
q.put(target)
# create a new thread to process targets
t = Thread(target=scan_thread,
args=(q, self.__cb_scan))
t.start()
self.__threads.append((t, q))
else:
inc_counter()
assigned = False # successfully assigned target to thread
while not assigned:
t, q = self.__threads[func.counter]
if not q.full():
q.put(target)
assigned = True
break
else:
inc_counter()
class FIFOCache:
def __init__(self, size=100):
self.size = size
self.cache = Queue(size)
def put(self, obj):
if self.cache.full():
self.cache.get_nowait()
self.cache.put_nowait(obj)
def get(self):
if self.cache.empty():
return None
else:
return self.cache.get_nowait()
def isFull(self):
return self.cache.full()
def isEmpty(self):
return self.cache.empty()
def __start_scan_thread(self, target):
"""
Creates a new scan thread, if max_threads allows to.
Every scan thread is assigned a queue that contains the targets to
scan. For every element of the queue, the scan callback function is
called.
input:
target a new target to scan. basically the result of cUtil.mkIP2
"""
self.__threads = filter(lambda t: t[0].is_alive(), self.__threads)
func = self.__start_scan_thread.__func__
def inc_counter():
"""
in order to not constantly add a target to the first queue,
we need a counter, that points to the next queue to add the
target. as consequence, every queue should have round about the
same qsize
"""
try:
if func.counter < len(self.__threads) - 1:
func.counter += 1
else:
func.counter = 0
except AttributeError:
func.counter = 0
if len(self.__threads) < self.max_threads:
# create a queue to be filled with targets
q = Queue(maxsize=1000)
q.put(target)
# create a new thread to process targets
t = Thread(target=scan_thread, args=(q, self.__cb_scan))
t.start()
self.__threads.append((t, q))
else:
inc_counter()
assigned = False # successfully assigned target to thread
while not assigned:
t, q = self.__threads[func.counter]
if not q.full():
q.put(target)
assigned = True
break
else:
inc_counter()
class _BatchWriter(object):
#: Truncate overly big items to that many bytes for the error message.
ERRMSG_DATA_TRUNCATION_LEN = 1024
def __init__(self, url, start, auth, size, interval, qsize,
maxitemsize, content_encoding, uploader, callback=None):
self.url = url
self.offset = start
self._nextid = count(start)
self.auth = auth
self.size = size
self.interval = interval
self.maxitemsize = maxitemsize
self.content_encoding = content_encoding
self.checkpoint = time.time()
self.itemsq = Queue(size * 2 if qsize is None else qsize)
self.closed = False
self.flushme = False
self.uploader = uploader
self.callback = callback
def write(self, item):
assert not self.closed, 'attempting writes to a closed writer'
data = jsonencode(item)
if len(data) > self.maxitemsize:
truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..."
raise ValueTooLarge(
'Value exceeds max encoded size of {} bytes: {!r}'
.format(self.maxitemsize, truncated_data))
self.itemsq.put(data)
if self.itemsq.full():
self.uploader.interrupt()
return self._nextid.next()
def flush(self):
self.flushme = True
self._waitforq()
self.flushme = False
def close(self, block=True):
self.closed = True
if block:
self._waitforq()
def _waitforq(self):
self.uploader.interrupt()
self.itemsq.join()
def __str__(self):
return self.url
class FileVideoStream:
def __init__(self, path, queueSize=128):
# initialize the file video stream along with the boolean
# used to indicate if the thread should be stopped or not
self.stream = cv2.VideoCapture(path)
self.stopped = False
# initialize the queue used to store frames read from
# the video file
self.Q = Queue(maxsize=queueSize)
def start(self):
# start a thread to read frames from the file video stream
t = Thread(target=self.update, args=())
t.daemon = True
t.start()
return self
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
return
# otherwise, ensure the queue has room in it
if not self.Q.full():
# read the next frame from the file
(grabbed, frame) = self.stream.read()
# if the `grabbed` boolean is `False`, then we have
# reached the end of the video file
if not grabbed:
self.stop()
return
# add the frame to the queue
self.Q.put(frame)
def read(self):
# return next frame in the queue
return self.Q.get()
def more(self):
# return True if there are still frames in the queue
return self.Q.qsize() > 0
def stop(self):
# indicate that the thread should be stopped
self.stopped = True
class ProcessHelper:
def __init__(self, sftp_configs, load_configs, zvelo_configs, eventhub_configs, adls_configs, metadata,
max_process = 4, processed_file_log = None):
self.sftp_configs = sftp_configs
self.load_configs = load_configs
if zvelo_configs is not None:
zvelo_path = zvelo_configs["path"].decode("ascii").encode()
self.zvelo_helper = ZveloHelper(zvelo_path)
else:
self.zvelo_helper = None
self.eventhub_configs = eventhub_configs
self.adls_configs = adls_configs
self.metadata = metadata
self.process_queue = Queue(max_process)
self.file_queue = Queue()
self.processed_file_log = processed_file_log
def add_file(self, file_path):
self.file_queue.put(file_path)
def join_all(self):
for process in iter(self.process_queue.get, None):
if process is not None:
process.join()
def start_process(self):
if self.processed_file_log is not None:
log_writer = open(self.processed_file_log, "a+")
else:
log_writer = None
while (not self.process_queue.empty()) \
and (not self.process_queue.queue[0].is_alive()):
process = self.process_queue.get()
if log_writer is not None:
log_writer.write(process.file_path + '\n')
if log_writer is not None:
log_writer.close()
while (not self.process_queue.full()) \
and (not self.file_queue.empty()):
file_path = self.file_queue.get()
process = SendDataProcess(self.sftp_configs, self.load_configs, self.zvelo_helper,
self.eventhub_configs, self.adls_configs, self.metadata, file_path)
process.start()
self.process_queue.put(process)
class VideoWriter(object):
'''
Threaded video encoding. Images are put in a queue and encoded in
a background process.
'''
def __init__(self, fname, fourcc, w, h, fps, color=True):
self.fname = fname
if cv2.__version__[0] == "3":
self.fourcc = cv2.VideoWriter_fourcc(*fourcc)
else:
self.fourcc = cv2.VideoWriter_fourcc(*fourcc)
self.dimension = (w, h)
self.fps = fps
self.color = color
self.vw = cv2.VideoWriter(fname, self.fourcc, self.fps, self.dimension, self.color)
self.Q = Queue(maxsize=128)
self.stopped = Event()
self.t = Thread(target=self.writer, args=())
#self.t.daemon = True
self.frame_number = 0
def running(self):
return not self.stopped.is_set()
def start(self):
self.t.start()
def stop(self):
self.stopped.set()
self.t.join()
def write(self, frame):
if self.t.is_alive():
while self.Q.full(): pass
self.Q.put(frame)
else:
raise IOError("The writer is not ready")
def writer(self):
while self.running():
if not self.Q.empty():
frame = self.Q.get()
self.frame_number += 1
self.vw.write(frame)
while not self.Q.empty():
frame = self.Q.get()
self.frame_number += 1
self.vw.write(frame)
self.vw.release()
class VideoStream(object):
"""
Desc: A faster video streaming class which uses Python's threading module
for creating and maintaining threads.
"""
def __init__(self, path, queue_size=128):
self.stream = cv2.VideoCapture(path)
self.exit = False
self.queue = Queue(maxsize=queue_size)
def start(self):
thread = Thread(name="Video Reader",
target=self.__read_frames,
args=())
thread.daemon = True
thread.start()
time.sleep(
1.0
) # So that the thread can get enough time for building up initial queue.
return
def __read_frames(self):
"""
Desc: This function is dedicated to reading frames and pushing them into the Queue.
It will run in a separate daemon thread.
"""
while True:
if self.exit:
self.stream.release()
return
if not self.queue.full():
success, frame = self.stream.read()
if not success:
self.stream.release()
self.stop()
return
self.queue.put(frame)
def read(self):
return self.queue.get()
def is_empty(self):
return self.queue.qsize() == 0
def stop(self):
self.exit = True
def run(doc_file, vocab_file, K=10, alpha=0.01, beta=0.01, num_iter=20):
vocab = dc.vocab_from_file(vocab_file)
# Word-topic + doc-topic should be less than 30 MB
V = len(vocab)
# Try to get the number of documents processed at a time to stay in the L3 cache (play around with this and see what works)
batch_size = 5 * 10**8 / V
#print "Running batch size of %i" % batch_size
#print "K = %i, V = %i" % (K, V)
queue = Queue(maxsize=batch_size)
global_nd = np.zeros(batch_size, dtype=np.intc)
global_wt = np.zeros((V, K), dtype=np.intc)
global_zt = np.zeros(K, dtype=np.intc)
t = time()
with open(doc_file) as f:
for j, line in enumerate(f):
if not queue.full():
queue.put(line.strip())
else:
#print "Queue full, starting to process a batch"
docs = []
while not queue.empty():
doc = queue.get()
docs.append(doc)
doc_word = dc.convert_to_np(docs, vocab)
docs = []
dt, wt, zt, WS, DS, ZS = dc.sample_ready(doc_word, K)
del doc_word
# Update the local wt and zt with the latest global versions
wt += global_wt
zt += global_zt
randoms = np.array(
[np.random.random_sample() for i in range(K)])
#print "Starting another batch"
for i in range(num_iter):
fp.sample(WS, DS, ZS, wt, dt, zt, randoms, alpha, beta)
nd = np.sum(dt, axis=1).astype(np.intc)
ll = rg._loglikelihood(wt, dt, zt, nd, alpha, beta)
#print ll
# Update the globals
#global_nd = nd
global_wt = wt
global_zt = zt
print "Processed %i so far" % j
print "Final log likelihood: %d" % ll
print time() - t
def start_servers(self):
# Start watching for port misses
watch = HoneyPokeWatcher(self._config['ssh_port'],
self._config['ignore_watch'],
self.check_server)
watch.start()
# Prepare to start the servers
server_count = len(self._config['ports'])
print("Starting " + str(server_count) + " servers\n")
wait = Queue(maxsize=server_count)
checked_certs = False
# Add servers
for port in self._config['ports']:
use_ssl = False
if 'ssl' in port and port['ssl'] is True:
use_ssl = True
if not checked_certs:
if not os.path.exists(
"honeypoke_key.pem") or not os.path.exists(
"honeypoke_cert.pem"):
print("SSL enabled, but PEM files not found!")
sys.exit(0)
checked_certs = True
self.add_server(int(port['port']), port['protocol'], use_ssl,
self.__loggers, wait)
# Wait until they indicate they have bound
while not wait.full():
pass
wait = None
# Drop privileges
self.drop_privileges()
# Wait until told to exit
try:
while True:
time.sleep(2)
except KeyboardInterrupt:
print("\nShutting down...")
# for server in self._servers:
# server.stop()
# server.join()
sys.exit(0)
class _BatchWriter(object):
def __init__(self, url, start, auth, size, interval, qsize,
maxitemsize, content_encoding, uploader, callback=None):
self.url = url
self.offset = start
self._nextid = count(start)
self.auth = auth
self.size = size
self.interval = interval
self.maxitemsize = maxitemsize
self.content_encoding = content_encoding
self.checkpoint = time.time()
self.itemsq = Queue(size * 2 if qsize is None else qsize)
self.closed = False
self.flushme = False
self.uploader = uploader
self.callback = callback
def write(self, item):
assert not self.closed, 'attempting writes to a closed writer'
data = jsonencode(item)
if len(data) > self.maxitemsize:
raise ValueTooLarge('value exceeds max encoded size of {}'\
.format(self.maxitemsize))
self.itemsq.put(data)
if self.itemsq.full():
self.uploader.interrupt()
return self._nextid.next()
def flush(self):
self.flushme = True
self._waitforq()
self.flushme = False
def close(self, block=True):
self.closed = True
if block:
self._waitforq()
def _waitforq(self):
self.uploader.interrupt()
self.itemsq.join()
def __str__(self):
return self.url
class JointTrajectoryCH:
def __init__(self):
rospy.init_node('joint_trajectory_command_handler')
self.real_robot = rospy.get_param("~real_robot")
ac_rate = rospy.get_param("~action_cycle_rate")
self.rate = rospy.Rate(ac_rate)
# Publisher to JointTrajectory robot controller
if self.real_robot:
# self.jt_pub = rospy.Publisher('/scaled_pos_traj_controller/command', JointTrajectory, queue_size=10)
self.jt_pub = rospy.Publisher('/pos_traj_controller/command', JointTrajectory, queue_size=10)
else:
self.jt_pub = rospy.Publisher('/eff_joint_traj_controller/command', JointTrajectory, queue_size=10)
# Subscriber to JointTrajectory Command coming from Environment
rospy.Subscriber('env_arm_command', JointTrajectory, self.callback_env_joint_trajectory, queue_size=1)
self.msg = JointTrajectory()
# Queue with maximum size 1
self.queue = Queue(maxsize=1)
# Flag used to publish empty JointTrajectory message only once when interrupting execution
self.stop_flag = False
def callback_env_joint_trajectory(self,data):
try:
# Add to the Queue the next command to execute
self.queue.put(data)
except:
pass
def joint_trajectory_publisher(self):
while not rospy.is_shutdown():
# If a command from the environment is waiting to be executed,
# publish the command, otherwise preempt trajectory
if self.queue.full():
self.jt_pub.publish(self.queue.get())
self.stop_flag = False
else:
# If the empty JointTrajectory message has no been published publish it and
# set the stop_flag to True, else pass
if not self.stop_flag:
self.jt_pub.publish(JointTrajectory())
self.stop_flag = True
else:
pass
self.rate.sleep()
def main():
q = Queue(maxsize = 5)
largest_array = []
for i in str(num):
if q.full():
q.get()
q.put(i)
if len(largest_array) < 5:
largest_array.append(i)
q.put(i)
if is_bigger([n for n in q.queue], largest_array):
largest_array = [n for n in q.queue]
print product(largest_array)
class MessageQueue(Thread):
def __init__(self, filename, timer=10):
Thread.__init__(self)
self.queue = Queue()
self.filename = filename
self.timer = timer
self.establish_date = arrow.utcnow().to('Asia/Shanghai').strftime(
"%Y-%m-%d")
def run(self):
while True:
current_date = arrow.utcnow().to('Asia/Shanghai').strftime(
"%Y-%m-%d")
if current_date != self.establish_date:
shutil.move(
"Flows/{0}.out".format(self.filename),
"Flows/{0}.{1}.out".format(self.filename,
self.establish_date))
self.establish_date = current_date
self.sink()
time.sleep(self.timer)
def sink(self):
pass
def qsize(self):
return self.queue.qsize()
def empty(self):
return self.queue.empty()
def full(self):
return self.queue.full()
def put(self, item, block=True, timeout=None):
self.queue.put(item, block, timeout)
def put_nowait(self, item):
return self.queue.put(item, False)
def get(self, block=True, timeout=None):
return self.queue.get(block, timeout)
def get_nowait(self):
return self.queue.get(False)
class ThreadPool(object):
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads):
self.tasks = Queue(num_threads)
for _ in range(num_threads):
Worker(self.tasks)
def add_task(self, func, *args, **kargs):
"""Add a task to the queue"""
self.tasks.put((func, args, kargs))
def wait_completion(self):
"""Wait for completion of all the tasks in the queue"""
self.tasks.join()
def full(self):
"""Return status of the Queue"""
return self.tasks.full()
class ThreadPool(object):
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads):
self.tasks = Queue(num_threads)
for _ in range(num_threads):
Worker(self.tasks)
def add_task(self, func, *args, **kargs):
"""Add a task to the queue"""
self.tasks.put((func, args, kargs))
def wait_completion(self):
"""Wait for completion of all the tasks in the queue"""
self.tasks.join()
def full(self):
"""Return status of the Queue"""
return self.tasks.full()
class ConsumerProducer(object):
def __init__(self):
self.q = Queue(maxsize=10)
self.counter = 0
def random_gem(self, dd):
return self.q.put(dd)
def playbook(self):
with ThreadPoolExecutor(max_workers=5) as executor:
for x in [i for i in range(100)]:
executor.submit(self.random_gem(x))
if self.q.full():
executor.submit(self.empty_queue())
def empty_queue(self):
if not self.q.empty():
self.q.get(block=False)
class FileVideoStream:
def __init__(self, path, queueSize=128):
self.stream = cv2.VideoCapture(path)
self.Q = Queue(maxsize=queueSize)
self.dequeuer_sema = Semaphore(0)
self.enqueuer_cv = Condition()
self.stopped = False
def start(self):
t = Thread(target=self.update, args=())
t.daemon = True
t.start()
return self
def update(self):
success, frame = self.stream.read()
num_consecutive_fails = 0
while num_consecutive_fails < 3:
if success:
self.enqueuer_cv.acquire()
if self.Q.full():
self.enqueuer_cv.wait()
self.enqueuer_cv.release()
num_consecutive_fails = 0
self.Q.put(frame)
self.dequeuer_sema.release()
else:
num_consecutive_fails += 1
success, frame = self.stream.read()
self.stopped = True
self.dequeuer_sema.release()
def read(self):
self.enqueuer_cv.acquire()
frame = self.Q.get()
self.enqueuer_cv.notify()
self.enqueuer_cv.release()
return frame
def more(self):
self.dequeuer_sema.acquire()
return not self.stopped or self.Q.qsize() > 0
def parse_sentences(lines): ''' Parse a list of sentences; returns a list of parses, each in lispish notation. ''' assert type(lines) in (list, tuple) if not lines: return [] # Create jobs result_queue = Queue(len(lines)) for idx,line in enumerate(lines): job_queue.put((idx,line,result_queue)) log(5, 'Equeued', (idx,line,result_queue)) while not result_queue.full(): time.sleep(.05) # collect results result = dict(result_queue.get() for i in range(result_queue.qsize())) return [result.get(idx,'') for idx in range(len(lines))]
class CommandQueue(object):
def __init__(self, timeTolerance=None):
self.__commandQueue = Queue()
self.__timeTolerance = timeTolerance
self.__finalCommand = None
self.__finalCommandTimeStamp = None
def add_command(self, timeStamp, command, ignorable=True):
if self.__commandQueue.full():
self.__commandQueue.get()
self.__commandQueue.task_done()
if command == self.__finalCommand:
return
self.__commandQueue.put((timeStamp, command, ignorable))
self.__finalCommand = command
self.__finalCommandTimeStamp = timeStamp
def getFinalCommandInfo(self):
return self.__finalCommand, self.__finalCommandTimeStamp
def get_command(self, currentTime):
command = None
if currentTime is None:
currentTime = time.time()
while 1 == 1:
if self.__commandQueue.empty():
self.__finalCommand = None
self.__finalCommandTimeStamp = None
break
command = self.__commandQueue.get()
if self.__timeTolerance is None or currentTime - command[
0] <= self.__timeTolerance or not command[2]:
break
command = None
return command
def task_done(self):
self.__commandQueue.task_done()
def setTimeTolerance(self, value):
if value > 0:
self.__timeTolerance = value
def _iter_keys(self, keys):
if not keys:
return
q = Queue(self.batch_size)
s = self.meta_session.clone()
for k in keys:
if not q.full():
q.put((k, s.read_latest(k)))
else:
data = self._fetch_response_data(q.get())
q.put((k, s.read_latest(k)))
if data:
yield data
while q.qsize():
data = self._fetch_response_data(q.get())
if data:
yield data
def parse_sentences(lines):
'''
Parse a list of sentences; returns a list of parses, each in lispish notation.
'''
assert type(lines) in (list, tuple)
if not lines:
return []
# Create jobs
result_queue = Queue(len(lines))
for idx, line in enumerate(lines):
job_queue.put((idx, line, result_queue))
log(5, 'Equeued', (idx, line, result_queue))
while not result_queue.full():
time.sleep(.05)
# collect results
result = dict(result_queue.get() for i in range(result_queue.qsize()))
return [result.get(idx, '') for idx in range(len(lines))]
class ThreadPool:
def __init__(self, numThreads=10):
self.jobs = Queue(numThreads)
self.event = threading.Event()
for _ in range(numThreads):
Worker(self.event, self.jobs)
self.logger = logging.getLogger("PoolManager.ThreadPool")
def addJob(self, func, *args, **kargs):
self.jobs.put((func, args, kargs))
self.logger.info("Added a new job")
self.logger.info("Threads left in pool: " + str(self.jobs.qsize()))
self.logger.info("If the Queue full? : " + str(self.jobs.full()))
def waitAll(self):
self.jobs.join()
def killAll(self):
self.event.set()
class ThreadPool: def __init__( self, numThreads = 10 ): self.jobs = Queue( numThreads ) self.event = threading.Event() for _ in range( numThreads ): Worker( self.event, self.jobs ) self.logger = logging.getLogger( "PoolManager.ThreadPool" ) def addJob( self, func, *args, **kargs ): self.jobs.put( ( func, args, kargs ) ) self.logger.info( "Added a new job" ) self.logger.info( "Threads left in pool: " + str( self.jobs.qsize() ) ) self.logger.info( "If the Queue full? : " + str( self.jobs.full() ) ) def waitAll( self ): self.jobs.join() def killAll( self ): self.event.set();
class ThreadPool():
"""创建一个线程池"""
def __init__(self, thread_num, timeout):
self.work_queue = Queue()
self.result_queue = Queue()
self.thread_pool = []
self.thread_num = thread_num
self.timeout = timeout
self.start_thread_pool()
def start_thread_pool(self):
for i in range(self.thread_num):
worker = Worker(self.work_queue, self.result_queue, self.timeout)
self.thread_pool.append(worker)
def add_task(self, func, *args, **kwargs):
if not self.work_queue.full():
self.work_queue.put(func, args, kwargs)
def get_result(self):
if not self.result_queue.empty():
return self.result_queue.get()
else:
return None
def task_done(self):
return self.work_queue.task_done()
def task_join(self):
self.work_queue.join()
def stop_thread_pool(self):
for i in range(self.thread_num):
self.thread_pool[i].stop()
del self.thread_pool[:]
class Terminal(object):
"""Terminal class."""
def __init__(self, name="", size=0):
self.name = name
if size > 0:
self.log = Queue(size)
else:
self.log = Queue(LOG_MAX)
def write_line(self, line=""):
"""Writes line to the terminal log"""
print line
if self.log.full():
self.log.get_nowait()
self.log.put(line)
else:
self.log.put(line)
return None
def parse_str(self, in_str=""):
"""Converts raw string input for use in exec_cmd()"""
self.write_line(in_str)
return None
class SFClient:
def __init__(self, host, port, qsize=10):
self._in_queue = Queue(qsize)
self._s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._s.connect((host, port))
data = self._s.recv(2)
if data != 'U ':
print "Wrong handshake"
self._s.send("U ")
print "Connected"
thread.start_new_thread(self.run, ())
def run(self):
while True:
length = ord(self._s.recv(1))
data = self._s.recv(length)
data = [ord(c) for c in data][1:]
#print "Recv %d bytes" % (length), ActiveMessage(data)
if self._in_queue.full():
print "Warning: Buffer overflow"
self._in_queue.get()
p = RawPacket()
p.data = data
self._in_queue.put(p, block=False)
def read(self, timeout=0):
return self._in_queue.get()
def write(self, payload):
print "SFClient: write:", payload
if type(payload) != type([]):
# Assume this will be derived from Packet
payload = payload.payload()
payload = [0] + payload
self._s.send(chr(len(payload)))
self._s.send(''.join([chr(c) for c in payload]))
return True
class AsynchExecutor(object):
'''
Executes tasks submitted to a FIFO queue using a daemon thread.
Calls to submit() block until there is space in the queue.
The daemon thread will block if the queue is empty.
'''
LOG = logging.getLogger(__name__)
def __init__(self, queue_size=10):
self.LOG.debug('Initializing AsynchExecutor')
self.running = False
self.task_queue = Queue(queue_size) # FIFO queue
def consume_task_queue(executor):
while executor.is_running():
task = executor.task_queue.get()
self.LOG.debug('Running task %s' % task)
if task == 'SHUTDOWN':
break
try:
retval, retval2, retval3 = task.work_func(*task.params)
self.LOG.debug('AsynchTask finished with retval %s' % retval)
task.on_success(retval, retval2, retval3)
except Exception as e:
task.on_error(e)
self.work_thread = Thread(target=consume_task_queue, args=(self,))
self.work_thread.daemon = True
def start(self):
''' Starts the daemon thread '''
self.running = True
self.work_thread.start()
self.LOG.debug('Started AsynchExecutor worker thread')
def shutdown(self):
''' Stops the worker thread and joins with it '''
self.LOG.debug('Stopping AsynchExecutor worker thread')
self.running = False
self.task_queue.put('SHUTDOWN')
self.work_thread.join()
self.LOG.debug('AsynchExecutor worker thread stopped')
def is_running(self):
return self.running
def submit(self, work_func, **kwargs):
'''
Uses the functions passed as parameters to create and
AsynchTask and places it in the task queue.
This operation blocks if the task queue is full.
'''
def no_action(param):
pass
if 'on_success' not in kwargs:
on_success = no_action
else:
on_success = kwargs['on_success']
if 'on_error' not in kwargs:
on_error = no_action
else:
on_error = kwargs['on_error']
if 'params' not in kwargs:
params = ()
else:
params = kwargs['params']
task = AsynchTask(work_func, on_success, on_error, params)
self.LOG.debug('Submitting new AsynchTask to FIFO queue...')
self.task_queue.put(task)
self.LOG.debug('AsynchTask accepted into FIFO queue')
def is_full(self):
'''
If the task queue is full, calling submit() will block.
Use this to check if submitting will block.
'''
return self.task_queue.full()
class DataQueue(object):
"""
Convenience class for using the DAF's notifications feature. This is a
collection that, once connected to EDEX by calling start(), fills with
data as notifications come in.
Example for getting obs data:
from DataQueue import DataQueue, GEOMETRY
request = DataAccessLayer.newDataRequest('obs')
request.setParameters('temperature')
request.setLocationNames('KOMA')
q = DataQueue(GEOMETRY, request)
q.start()
for item in q:
print(item.getNumber('temperature'))
"""
def __init__(self, dtype, request, maxsize=_DEFAULT_MAXSIZE):
"""
Create a new DataQueue.
Args:
dtype: Either GRID or GEOMETRY; must match the type of data
requested.
request: IDataRequest describing the data you want. It must at
least have datatype set. All data produced will satisfy the
constraints you specify.
maxsize: Maximum number of data objects the queue can hold at
one time. If the limit is reached, any data coming in after
that will not appear until one or more items are removed using
DataQueue.get().
"""
assert maxsize > 0
assert dtype in (GEOMETRY, GRID)
self._maxsize = maxsize
self._queue = Queue(maxsize=maxsize)
self._thread = None
if dtype is GEOMETRY:
self._notifier = DNL.getGeometryDataUpdates(request)
elif dtype is GRID:
self._notifier = DNL.getGridDataUpdates(request)
def start(self):
"""Start listening for notifications and requesting data."""
if self._thread is not None:
# Already started
return
kwargs = {'callback': self._data_received}
self._thread = Thread(target=self._notifier.subscribe, kwargs=kwargs)
self._thread.daemon = True
self._thread.start()
timer = 0
while not self._notifier.subscribed:
time.sleep(0.1)
timer += 1
if timer >= 100: # ten seconds
raise RuntimeError('timed out when attempting to subscribe')
def _data_received(self, data):
for d in data:
if not isinstance(d, list):
d = [d]
for item in d:
self._queue.put(item)
def get(self, block=True, timeout=None):
"""
Get and return the next available data object. By default, if there is
no data yet available, this method will not return until data becomes
available.
Args:
block: Specifies behavior when the queue is empty. If True, wait
until an item is available before returning (the default). If
False, return None immediately if the queue is empty.
timeout: If block is True, wait this many seconds, and return None
if data is not received in that time.
Returns:
IData
"""
if self.closed:
raise Closed
try:
return self._queue.get(block, timeout)
except Empty:
return None
def get_all(self):
"""
Get all data waiting for processing, in a single list. Always returns
immediately. Returns an empty list if no data has arrived yet.
Returns:
List of IData
"""
data = []
for _ in range(self._maxsize):
next_item = self.get(False)
if next_item is None:
break
data.append(next_item)
return data
def close(self):
"""Close the queue. May not be re-opened after closing."""
if not self.closed:
self._notifier.close()
self._thread.join()
def qsize(self):
"""Return number of items in the queue."""
return self._queue.qsize()
def empty(self):
"""Return True if the queue is empty."""
return self._queue.empty()
def full(self):
"""Return True if the queue is full."""
return self._queue.full()
@property
def closed(self):
"""True if the queue has been closed."""
return not self._notifier.subscribed
@property
def maxsize(self):
"""
Maximum number of data objects the queue can hold at one time.
If this limit is reached, any data coming in after that will not appear
until one or more items are removed using get().
"""
return self._maxsize
def __iter__(self):
if self._thread is not None:
while not self.closed:
yield self.get()
def __enter__(self):
self.start()
return self
def __exit__(self, *unused):
self.close()
class FifoReadout(object):
def __init__(self, dut):
self.dut = dut
self.callback = None
self.errback = None
self.readout_thread = None
self.worker_thread = None
self.watchdog_thread = None
self.fill_buffer = False
self.readout_interval = 0.05
self._moving_average_time_period = 10.0
self._data_deque = deque()
self._data_buffer = deque()
self._words_per_read = deque(maxlen=int(self._moving_average_time_period / self.readout_interval))
self._result = Queue(maxsize=1)
self._calculate = Event()
self.stop_readout = Event()
self.force_stop = Event()
self.timestamp = None
self.update_timestamp()
self._is_running = False
self.reset_rx()
self.reset_sram_fifo()
@property
def is_running(self):
return self._is_running
@property
def is_alive(self):
if self.worker_thread:
return self.worker_thread.is_alive()
else:
False
@property
def data(self):
if self.fill_buffer:
return self._data_buffer
else:
logging.warning('Data requested but software data buffer not active')
def data_words_per_second(self):
if self._result.full():
self._result.get()
self._calculate.set()
try:
result = self._result.get(timeout=2 * self.readout_interval)
except Empty:
self._calculate.clear()
return None
return result / float(self._moving_average_time_period)
def start(self, callback=None, errback=None, reset_rx=False, reset_sram_fifo=False, clear_buffer=False, fill_buffer=False, no_data_timeout=None):
if self._is_running:
raise RuntimeError('Readout already running: use stop() before start()')
self._is_running = True
logging.info('Starting FIFO readout...')
self.callback = callback
self.errback = errback
self.fill_buffer = fill_buffer
if reset_rx:
self.reset_rx()
if reset_sram_fifo:
self.reset_sram_fifo()
else:
fifo_size = self.dut['SRAM']['FIFO_SIZE']
if fifo_size != 0:
logging.warning('SRAM FIFO not empty when starting FIFO readout: size = %i', fifo_size)
self._words_per_read.clear()
if clear_buffer:
self._data_deque.clear()
self._data_buffer.clear()
self.stop_readout.clear()
self.force_stop.clear()
if self.errback:
self.watchdog_thread = Thread(target=self.watchdog, name='WatchdogThread')
self.watchdog_thread.daemon = True
self.watchdog_thread.start()
if self.callback:
self.worker_thread = Thread(target=self.worker, name='WorkerThread')
self.worker_thread.daemon = True
self.worker_thread.start()
self.readout_thread = Thread(target=self.readout, name='ReadoutThread', kwargs={'no_data_timeout': no_data_timeout})
self.readout_thread.daemon = True
self.readout_thread.start()
def stop(self, timeout=10.0):
if not self._is_running:
raise RuntimeError('Readout not running: use start() before stop()')
self._is_running = False
self.stop_readout.set()
try:
self.readout_thread.join(timeout=timeout)
if self.readout_thread.is_alive():
if timeout:
raise StopTimeout('FIFO stop timeout after %0.1f second(s)' % timeout)
else:
logging.warning('FIFO stop timeout')
except StopTimeout as e:
self.force_stop.set()
if self.errback:
self.errback(sys.exc_info())
else:
logging.error(e)
if self.readout_thread.is_alive():
self.readout_thread.join()
if self.errback:
self.watchdog_thread.join()
if self.callback:
self.worker_thread.join()
self.callback = None
self.errback = None
logging.info('Stopped FIFO readout')
def print_readout_status(self):
sync_status = self.get_rx_sync_status()
discard_count = self.get_rx_fifo_discard_count()
error_count = self.get_rx_8b10b_error_count()
logging.info('Data queue size: %d', len(self._data_deque))
logging.info('SRAM FIFO size: %d', self.dut['SRAM']['FIFO_SIZE'])
logging.info('Channel: %s', " | ".join([channel.name.rjust(3) for channel in self.dut.get_modules('fei4_rx')]))
logging.info('RX sync: %s', " | ".join(["YES".rjust(3) if status is True else "NO".rjust(3) for status in sync_status]))
logging.info('RX FIFO discard counter: %s', " | ".join([repr(count).rjust(3) for count in discard_count]))
logging.info('RX FIFO 8b10b error counter: %s', " | ".join([repr(count).rjust(3) for count in error_count]))
if not any(self.get_rx_sync_status()) or any(discard_count) or any(error_count):
logging.warning('RX errors detected')
def readout(self, no_data_timeout=None):
'''Readout thread continuously reading SRAM.
Readout thread, which uses read_data() and appends data to self._data_deque (collection.deque).
'''
logging.debug('Starting %s', self.readout_thread.name)
curr_time = get_float_time()
time_wait = 0.0
while not self.force_stop.wait(time_wait if time_wait >= 0.0 else 0.0):
try:
time_read = time()
if no_data_timeout and curr_time + no_data_timeout < get_float_time():
raise NoDataTimeout('Received no data for %0.1f second(s)' % no_data_timeout)
data = self.read_data()
except Exception:
no_data_timeout = None # raise exception only once
if self.errback:
self.errback(sys.exc_info())
else:
raise
if self.stop_readout.is_set():
break
else:
data_words = data.shape[0]
if data_words > 0:
last_time, curr_time = self.update_timestamp()
status = 0
if self.callback:
self._data_deque.append((data, last_time, curr_time, status))
if self.fill_buffer:
self._data_buffer.append((data, last_time, curr_time, status))
self._words_per_read.append(data_words)
elif self.stop_readout.is_set():
break
else:
self._words_per_read.append(0)
finally:
time_wait = self.readout_interval - (time() - time_read)
if self._calculate.is_set():
self._calculate.clear()
self._result.put(sum(self._words_per_read))
if self.callback:
self._data_deque.append(None) # last item, will stop worker
logging.debug('Stopped %s', self.readout_thread.name)
def worker(self):
'''Worker thread continuously calling callback function when data is available.
'''
logging.debug('Starting %s', self.worker_thread.name)
while True:
try:
data = self._data_deque.popleft()
except IndexError:
self.stop_readout.wait(self.readout_interval) # sleep a little bit, reducing CPU usage
else:
if data is None: # if None then exit
break
else:
try:
self.callback(data)
except Exception:
self.errback(sys.exc_info())
logging.debug('Stopped %s', self.worker_thread.name)
def watchdog(self):
logging.debug('Starting %s', self.watchdog_thread.name)
while True:
try:
if not any(self.get_rx_sync_status()):
raise RxSyncError('No RX sync')
if any(self.get_rx_8b10b_error_count()):
raise EightbTenbError('RX 8b10b error(s) detected')
if any(self.get_rx_fifo_discard_count()):
raise FifoError('RX FIFO discard error(s) detected')
except Exception:
self.errback(sys.exc_info())
if self.stop_readout.wait(self.readout_interval * 10):
break
logging.debug('Stopped %s', self.watchdog_thread.name)
def read_data(self):
'''Read SRAM and return data array
Can be used without threading.
Returns
-------
data : list
A list of SRAM data words.
'''
return self.dut['SRAM'].get_data()
def update_timestamp(self):
curr_time = get_float_time()
last_time = self.timestamp
self.timestamp = curr_time
return last_time, curr_time
def read_status(self):
raise NotImplementedError()
def reset_sram_fifo(self):
fifo_size = self.dut['SRAM']['FIFO_SIZE']
logging.info('Resetting SRAM FIFO: size = %i', fifo_size)
self.update_timestamp()
self.dut['SRAM']['RESET']
sleep(0.2) # sleep here for a while
fifo_size = self.dut['SRAM']['FIFO_SIZE']
if fifo_size != 0:
logging.warning('SRAM FIFO not empty after reset: size = %i', fifo_size)
def reset_rx(self, channels=None):
logging.info('Resetting RX')
if channels:
filter(lambda channel: self.dut[channel].RX_RESET, channels)
else:
filter(lambda channel: channel.RX_RESET, self.dut.get_modules('fei4_rx'))
sleep(0.1) # sleep here for a while
def get_rx_sync_status(self, channels=None):
if channels:
return map(lambda channel: True if self.dut[channel].READY else False, channels)
else:
return map(lambda channel: True if channel.READY else False, self.dut.get_modules('fei4_rx'))
def get_rx_8b10b_error_count(self, channels=None):
if channels:
return map(lambda channel: self.dut[channel].DECODER_ERROR_COUNTER, channels)
else:
return map(lambda channel: channel.DECODER_ERROR_COUNTER, self.dut.get_modules('fei4_rx'))
def get_rx_fifo_discard_count(self, channels=None):
if channels:
return map(lambda channel: self.dut[channel].LOST_DATA_COUNTER, channels)
else:
return map(lambda channel: channel.LOST_DATA_COUNTER, self.dut.get_modules('fei4_rx'))
class Agent:
def __init__(self, personality=None):
self.personality = None
self.neurotics = 0.0
self.MAX_NEUROTICS = 3.0
self._in_q = Queue()
self._out_q = Queue(1)
self.FRAMES_PER_SECOND = 60.0
self.SECS_PER_UPDATE = 1.0/self.FRAMES_PER_SECOND
self.EVENT_DURATION = 1.0 # seconds
self.BASE_VELOCITY = 1.0/250 # self.TIME_TO_TRAVEL*self.SECS_PER_UPDATE
self.DISTANCE_TOLERANCE = 1.0/10000.0
self.set_personality(personality)
def start(self):
data = threading.local()
thread = threading.Thread(name='Agent Runloop', target=self._run, args=(data,))
thread.daemon = True
thread.start()
def stop(self):
self._in_q.put_nowait(Invocation(self._stop, (), {}))
def put(self, values=None):
if values:
if isinstance(values, np.ndarray):
v = values
elif isinstance(values, OCC):
v = values.pad.state
else:
raise ValueError('Valores de evento inválidos')
self._in_q.put_nowait(Invocation(self._put, (v,), {}))
def get(self):
mood = self._out_q.get()
if mood is not None and len(mood) == 3:
return PAD(pleasure=mood[0], arousal=mood[1], dominance=mood[2])
def set_personality(self, values):
if values:
if isinstance(values, np.ndarray):
ocean = OCEAN(personality=values)
elif isinstance(values, OCEAN):
ocean = values
else:
raise ValueError('Valores de personalidade inválidos')
else:
ocean = OCEAN()
self._in_q.put_nowait(Invocation(self._set_personality, (ocean,), {}))
def _run(self, data):
data.running = True
data.events = []
data.state = np.zeros(3)
previous = time()
lag = 0.0
print 'running...'
while data.running:
current = time()
elapsed = current - previous
previous = current
lag += elapsed
self._process_input(data)
while lag >= self.SECS_PER_UPDATE:
self._update(data)
lag -= self.SECS_PER_UPDATE
self._process_output(data)
print 'stopped!'
def _process_input(self, data):
while not self._in_q.empty():
job = self._in_q.get()
job.fn(data, *job.args, **job.kwargs)
def _process_output(self, data):
if self._out_q.full():
with self._out_q.mutex:
self._out_q.queue.clear()
self._out_q.put_nowait(data.state)
def _update(self, data):
if len(data.events) > 0:
# calculate events weighted average
vectors = []
weights = []
replacement = []
for i in range(len(data.events)):
if data.events[i].get_influence() > 0:
vectors.append(data.events[i].values)
weights.append(data.events[i].get_influence())
replacement.append(data.events[i])
data.events = list(replacement)
if len(vectors) > 0 and len(weights) > 0:
avg_event = np.average(vectors, axis=0, weights=weights)
# move mood towards average event
data.state = self._move_to(data.state, avg_event)
else:
# move mood towards personality
data.state = self._move_to(data.state, data.personality)
def _move_to(self, _from, _to):
if np.allclose(_from, _to, self.DISTANCE_TOLERANCE):
return np.array(_to)
direction = _to - _from
direction /= np.linalg.norm(direction)
_from += direction * self.BASE_VELOCITY * self.neurotics
return _from
def _stop(self, data):
print 'stopping...'
data.running = False
def _put(self, data, value):
print 'putting data...'
data.events.append(Event(value, self.EVENT_DURATION))
def _set_personality(self, data, value):
print 'setting personality...'
data.personality = np.array(value.pad.state)
data.state = np.array(value.pad.state)
self.personality = value
self.neurotics = Agent.map_value(value.neuroticism, -1, 1, 1, self.MAX_NEUROTICS)
@staticmethod
def map_value(value=0.0, in_min=0.0, in_max=1.0, out_min=0.0, out_max=1.0):
return (float(value) - float(in_min)) * (float(out_max) - float(out_min)) / (
float(in_max) - float(in_min)) + float(out_min)
class Controller(Thread):
""" Thread object that handles LLDB events and commands. """
CTRL_VOICE = 238 # doesn't matter what this is
TARG_NEW = 1
TARG_DEL = 1 << 1
PROC_NEW = 1 << 2
PROC_DEL = 1 << 3
BP_CHANGED = 1 << 4
BAD_STATE = 1 << 5 # multiple targets
def __init__(self, vimx):
""" Creates the LLDB SBDebugger object and more! """
import logging
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
self._sink = open('/dev/null')
self._dbg = lldb.SBDebugger.Create()
self._dbg.SetOutputFileHandle(self._sink, False)
self._ipreter = self._dbg.GetCommandInterpreter()
self._rcx = lldb.SBListener("the_ear") # receiver
self._trx = lldb.SBBroadcaster("the_mouth") # transmitter for user events
self._trx.AddListener(self._rcx, self.CTRL_VOICE)
self._target = None
self._process = None
self._num_bps = 0
self.in_queue = Queue(maxsize=2)
self.out_queue = Queue(maxsize=1)
self.vimx = vimx
self.busy_stack = 0 # when > 0, buffers are not updated
self.buffers = VimBuffers(vimx)
self.session = Session(self, vimx)
super(Controller, self).__init__() # start the thread
def is_busy(self):
return self.busy_stack > 0
def busy_more(self):
self.busy_stack += 1
def busy_less(self):
self.busy_stack -= 1
if self.busy_stack < 0:
self.logger.critical("busy_stack < 0")
self.busy_stack = 0
def safe_call(self, method, args=[], sync=False, timeout=None):
""" (thread-safe) Call `method` with `args`. If `sync` is True, wait for
`method` to complete and return its value. If timeout is set and non-
negative, and the `method` did not complete within `timeout` seconds,
an EventLoopError is raised!
"""
if self._dbg is None:
self.logger.critical("Debugger not found!")
raise EventLoopError("Dead debugger!")
if self.out_queue.full(): # garbage
self.out_queue.get() # clean
try:
self.in_queue.put((method, args, sync), block=False)
interrupt = lldb.SBEvent(self.CTRL_VOICE, "the_sound")
self._trx.BroadcastEvent(interrupt)
if sync:
return self.out_queue.get(block=True, timeout=timeout)
except Empty:
raise EventLoopError("Timed out!")
except Full:
self.logger.critical("Event loop thread is probably dead!")
raise EventLoopError("Dead event loop!")
def safe_execute(self, tokens):
""" (thread-safe) Executes an lldb command defined by a list of tokens.
If a token contains white-spaces, they are escaped using backslash.
"""
cmd = ' '.join([ t.replace(' ', '\\ ') for t in tokens ])
self.safe_call(self.exec_command, [cmd])
def safe_exit(self):
""" Exit from the event-loop, and wait for the thread to join.
Should be called from outside this thread.
"""
self.safe_call(None)
self.join()
def complete_command(self, arg, line, pos):
""" Returns a list of viable completions for line, and cursor at pos. """
pos = int(pos)
result = lldb.SBStringList()
if arg == line and line != '':
# provide all possible completions when completing 't', 'b', 'di' etc.
num = self._ipreter.HandleCompletion('', 0, 1, -1, result)
cands = ['']
for x in result:
if x == line:
cands.insert(1, x)
elif x.startswith(line):
cands.append(x)
else:
num = self._ipreter.HandleCompletion(line, pos, 1, -1, result)
cands = [x for x in result]
if len(cands) > 1:
if cands[0] == '' and arg != '':
if not cands[1].startswith(arg) or not cands[-1].startswith(arg):
return []
return cands[1:]
else:
return []
def update_buffers(self, buf=None):
""" Update lldb buffers and signs placed in source files.
@param buf
If None, all buffers and signs would be updated.
Otherwise, update only the specified buffer.
"""
if self.is_busy():
return
commander = self.get_command_result
if buf is None:
self.buffers.update(self._target, commander)
else:
self.buffers.update_buffer(buf, self._target, commander)
def get_state_changes(self):
""" Get a value denoting how target, process, and/or breakpoint have changed.
If a new process found, add our listener to its broadcaster.
"""
changes = 0
if self._dbg.GetNumTargets() > 1:
return self.BAD_STATE
if self._target is None or not self._target.IsValid():
target = self._dbg.GetSelectedTarget()
if target.IsValid():
changes = self.TARG_NEW
self._target = target
elif self._target is not None:
changes = self.TARG_DEL
self._target = None
if self._target is None:
if self._process is not None:
changes |= self.PROC_DEL
self._process = None
if self._num_bps > 0:
changes |= self.BP_CHANGED
self._num_bps = 0
return changes
if self._process is None or not self._process.IsValid():
process = self._target.process
if process.IsValid():
changes |= self.PROC_NEW
self._process = process
process.broadcaster.AddListener(self._rcx,
lldb.SBProcess.eBroadcastBitStateChanged | \
lldb.SBProcess.eBroadcastBitSTDOUT | \
lldb.SBProcess.eBroadcastBitSTDERR)
self._proc_cur_line_len = 0
self._proc_lines_count = 0
self._proc_sigstop_count = 0
elif self._process is not None:
changes |= self.PROC_DEL
self._process = None
num_bps = self._target.GetNumBreakpoints()
if self._num_bps != num_bps:
# TODO what if one was added and another deleted?
changes |= self.BP_CHANGED
self._num_bps = num_bps
# TODO Watchpoints
return changes
def change_buffer_cmd(self, buf, cmd):
""" Change a buffer command and update it. """
self.buffers._content_map[buf] = cmd
self.update_buffers(buf=buf)
if self._target is not None:
self.vimx.command('drop [lldb]%s' % buf)
def do_btswitch(self):
""" Switch backtrace command to show all threads. """
cmd = self.buffers._content_map['backtrace']
if cmd != 'bt all':
cmd = 'bt all'
else:
cmd = 'bt'
self.change_buffer_cmd('backtrace', cmd)
def bp_set_line(self, spath, line):
from os.path import abspath
fpath = abspath(spath).encode('ascii', 'ignore')
bp = self._target.BreakpointCreateByLocation(fpath, line)
self.buffers.logs_append(u'\u2192(lldb-bp) %s:%d\n' % (spath, line))
self.session.bp_map_auto(bp, (spath, line))
self.update_buffers(buf='breakpoints')
def do_breakswitch(self, bufnr, line):
""" Switch breakpoint at the specified line in the buffer. """
key = (bufnr, line)
if self.buffers.bp_list.has_key(key):
bps = self.buffers.bp_list[key]
args = "delete %s" % " ".join([str(b.GetID()) for b in bps])
self.exec_command("breakpoint " + args)
else:
self.bp_set_line(self.vimx.get_buffer_name(bufnr), line)
def do_breakdelete(self, bp_id):
""" Delete a breakpoint by id """
if bp_id:
self.exec_command("breakpoint delete {}".format(bp_id))
def put_stdin(self, instr):
""" Call PutSTDIN() of process with instr. """
if self._process is not None:
self._process.PutSTDIN(instr)
else:
self.vimx.log('No active process!')
def get_command_result(self, command, add2hist=False):
""" Runs command in the interpreter and returns (success, output)
Not to be called directly for commands which changes debugger state;
use exec_command instead.
"""
result = lldb.SBCommandReturnObject()
self._ipreter.HandleCommand(command.encode('ascii', 'ignore'), result, add2hist)
return (result.Succeeded(), result.GetOutput() if result.Succeeded() else result.GetError())
def exec_command(self, command):
""" Runs command in the interpreter, calls update_buffers, and display the
result in the logs buffer. Returns True if succeeded.
"""
self.buffers.logs_append(u'\u2192(lldb) %s\n' % command)
(success, output) = self.get_command_result(command, True)
if not success:
self.buffers.logs_append(output, u'\u2717')
elif len(output) > 0:
self.buffers.logs_append(output, u'\u2713')
state_changes = self.get_state_changes()
if state_changes & self.TARG_NEW != 0:
self.session.new_target(self._target)
elif state_changes & self.BP_CHANGED != 0 and self._target is not None:
self.session.bp_changed(command, self._target.breakpoint_iter())
self.update_buffers()
return success
def run(self):
""" This thread's event loop. """
import traceback
to_count = 0
while True:
event = lldb.SBEvent()
if self._rcx.WaitForEvent(30, event): # 30 second timeout
def event_matches(broadcaster, skip=True):
if event.BroadcasterMatchesRef(broadcaster):
if skip:
while self._rcx.GetNextEventForBroadcaster(broadcaster, event):
pass
return True
return False
if event_matches(self._trx, skip=False):
try:
method, args, sync = self.in_queue.get(block=False)
if method is None:
break
self.logger.info('Calling %s with %s' % (method.func_name, repr(args)))
ret = method(*args)
if sync:
self.out_queue.put(ret, block=False)
except Exception:
self.logger.critical(traceback.format_exc())
elif event_matches(self._process.broadcaster):
# Dump stdout and stderr to logs buffer
while True:
out = ''
stdout = self._process.GetSTDOUT(256)
if stdout is not None:
out += stdout
stderr = self._process.GetSTDERR(256)
if stderr is not None:
out += stderr
if len(out) == 0:
break
n_lines = self.buffers.logs_append(out)
if n_lines == 0:
self._proc_cur_line_len += len(out)
else:
self._proc_cur_line_len = 0
self._proc_lines_count += n_lines
if self._proc_cur_line_len > 8192 or self._proc_lines_count > 2048:
# detect and stop/kill insane process
if self._process.state == lldb.eStateStopped:
pass
elif self._proc_sigstop_count > 7:
self._process.Kill()
self.buffers.logs_append(u'\u2717SIGSTOP limit exceeded! Sent SIGKILL!\n')
else:
self._process.SendAsyncInterrupt()
self._proc_sigstop_count += 1
self.buffers.logs_append(u'\u2717Output limits exceeded! Sent SIGSTOP!\n')
break
# end of dump while
self.update_buffers()
else: # Timed out
to_count += 1
if to_count > 172800: # in case WaitForEvent() does not wait!
self.logger.critical('Broke the loop barrier!')
break
# end of event-loop while
self._dbg.Terminate()
self._dbg = None
self._sink.close()
self.logger.info('Terminated!')
class TaskManager(object):
"""
Task manager class based on thread module which
executes assigned tasks concurently. It uses a
pool of thread workers, queue of tasks and pid
set to monitor jobs execution.
.. doctest::
Use case:
mgr = TaskManager()
jobs = []
jobs.append(mgr.spaw(func, args))
mgr.joinall(jobs)
"""
def __init__(self, nworkers=10, name='TaskManager', qtype='Queue', debug=0):
self.name = name
self.debug = debug
self._pids = set()
self._uids = UidSet()
if qtype == 'PriorityQueue':
self._tasks = PriorityQueue()
else:
self._tasks = Queue()
self._workers = [Worker(name, self._tasks, self._pids, self._uids) \
for _ in xrange(0, nworkers)]
def status(self):
"Return status of task manager queue"
info = {'qsize':self._tasks.qsize(), 'full':self._tasks.full(),
'unfinished':self._tasks.unfinished_tasks,
'nworkers':len(self._workers)}
return {self.name:info}
def nworkers(self):
"""Return number of workers associated with this manager"""
return len(self._workers)
def assign_priority(self, uid):
"Assign priority for given uid"
if not uid or self._tasks.empty():
return 0
frequency = self._uids.get(uid)
return frequency/10
def spawn(self, func, *args, **kwargs):
"""Spawn new process for given function"""
pid = kwargs.get('pid', genkey(str(args) + str(kwargs)))
evt = Event()
if not pid in self._pids:
self._pids.add(pid)
task = (evt, pid, func, args, kwargs)
if isinstance(self._tasks, PriorityQueue):
uid = kwargs.get('uid', None)
self._uids.add(uid)
priority = self.assign_priority(uid)
self._tasks.put((priority, uid, task))
else:
self._tasks.put(task)
else:
# the event was not added to task list, invoke set()
# to pass it in wait() call, see joinall
evt.set()
return evt, pid
def remove(self, pid):
"""Remove pid and associative process from the queue"""
self._pids.discard(pid)
def is_alive(self, pid):
"""Check worker queue if given pid of the process is still running"""
return pid in self._pids
def clear(self, tasks):
"""
Clear all tasks in a queue. It allows current jobs to run, but will
block all new requests till workers event flag is set again
"""
map(lambda (evt, pid): (evt.clear(), pid), tasks)
def joinall(self, tasks):
"""Join all tasks in a queue and quite"""
map(lambda (evt, pid): (evt.wait(), pid), tasks)
def quit(self):
"""Put None task to all workers and let them quit"""
map(lambda w: self._tasks.put(None), self._workers)
map(lambda w: w.join(), self._workers)
def force_exit(self):
"""Force all workers to exit"""
map(lambda w: w.force_exit(), self._workers)
from Queue import Queue #Queue.Queue类即是一个队列的同步实现 #队列的长度可为无线或者有限 #而通过Queue的构造函数的可选参数maxsize来设定队列长度 #如果maxsize小于1表示队列长度无线 myqueue=Queue(maxsize=10) #将一个值放入队列中 #put()方法有两个参数 #第一个item为必须的，为插入项目的值 #第二个block为可选参数，默认为1 #如果队列当前为空，且block为1，put方法就调用线程暂停，直到空出一个数据单元 #如果block为0，put方法引发Full异常 myqueue.put((10,9)) print "queue size=",myqueue.qsize()#返回队列的大小 print "queue data=",myqueue.get()#获得队列 print "queue is empty?",myqueue.empty()#如果队列为空，返回True,反之False print "queue is full?",myqueue.full()#如果队列满了，返回True,反之False #print "queue get_nowait=",myqueue.get_nowait()#相当Queue.get(False), #非阻塞Queue.put(item)写入队列，timeout等待时间 #print "queue put_nowait=",myqueue.put_nowait()#相当Queue.put(item.False), print "queue task_done=",myqueue.task_done()#在完成一项工作之后 #task_done函数向已经完成的队列发送一个信号 #join()实际上是意味着等到队列为空，再执行别的操作
sock_rych = zmqctx.socket(zmq.PUB)
sock_rych.setsockopt(zmq.HWM, 100)
sock_rych.bind('tcp://*:13372')
# input and output queues for worker threads that call enrycher. (zmq is only used in the main thread)
in_queue = Queue(maxsize=MAX_ENRYCHER_REQUESTS)
out_queue = Queue(maxsize=100*MAX_ENRYCHER_REQUESTS)
# prepare worker threads
for i in range(MAX_ENRYCHER_REQUESTS):
worker = threading.Thread(target=enrycher_worker, args=(in_queue,out_queue))
worker.start()
try:
while True:
if in_queue.full():
print 'sleep ... %d:%d ...' % (in_queue.qsize(), out_queue.qsize(),),
time.sleep(1)
print '!'
if not in_queue.full() and zmq.select([sock_txt], [], [], 3)[0]:
article = sock_txt.recv_pyobj()
if is_enrychable(article):
print 'enqueued %s (lang=%r)' % (article['id'], article['lang'])
print '%d:%d' % (in_queue.qsize(), out_queue.qsize(),),
in_queue.put(article)
else:
print 'ignored %s (lang=%r)' % (article['id'], article['lang'])
print '%d:%d' % (in_queue.qsize(), out_queue.qsize(),),
out_queue.put(article)
class DBPool(object):
'''''一个数据库连接池'''
def initPool(self, maxActive=5, maxWait=None, init_size=0, db_type="mysql", **config):
'''初始化数据库连接池
'''
log.msg("__init__ Pool..")
self.__freeConns = Queue(maxActive)
self.maxWait = maxWait
self.db_type = db_type
self.config = config
if init_size > maxActive:
init_size = maxActive
for i in range(init_size):
self.free(self._create_conn())
self.nowconn = None
def __del__(self):
log.msg("__del__ Pool..")
self.release()
def release(self):
'''''释放资源，关闭池中的所有连接'''
log.msg("release Pool..")
while self.__freeConns and not self.__freeConns.empty():
con = self.get()
con.release()
self.__freeConns = None
def _create_conn(self):
'''''创建连接 '''
if self.db_type in DBCS:
return DBCS[self.db_type](**self.config);
def get(self, timeout=None):
'''''获取一个连接
@param timeout:超时时间
'''
if timeout is None:
timeout = self.maxWait
conn = None
if self.__freeConns.empty():#如果容器是空的，直接创建一个连接
conn = self._create_conn()
else:
conn = self.__freeConns.get(timeout=timeout)
conn.pool = self
return conn
def cursor(self,cursorclass = None):
'''通配接口'''
conn = self.get()
self.nowconn = conn
ucur = UCursor( conn, cursorclass)
return ucur
def commit(self):
'''提交'''
try:
self.nowconn.commit()
except Exception as e:
log.err(e.message)
def rollback(self):
'''事务回滚
'''
try:
self.nowconn.rollback()
except Exception as e:
log.err(e.message)
def free(self, conn):
'''''将一个连接放回池中
@param conn: 连接对象
'''
conn.pool = None
if(self.__freeConns.full()):#如果当前连接池已满，直接关闭连接
conn.release()
return self.__freeConns.put_nowait(conn)
def execSql(self,sqlstr):
'''执行数据库的写操作(插入,修改,删除)
@param sqlstr: str 需要执行的sql语句
'''
try:
conn = self.get(5)
cursor = conn.cursor()
count = cursor.execute(sqlstr)
conn.commit()
cursor.close()
conn.close()
if count>0:
return True
return False
except Exception,err:
log.err(err)
conn.close()
return None#通过放回NONE在远程调用上抛出异常
class TaskManager(object):
"""
Task manager class based on thread module which
executes assigned tasks concurently. It uses a
pool of thread workers, queue of tasks and pid
set to monitor jobs execution.
.. doctest::
Use case:
mgr = TaskManager()
jobs = []
jobs.append(mgr.spawn(func, args))
mgr.joinall(jobs)
"""
def __init__(self, nworkers=10, name='TaskManager'):
self.name = name
self.pids = set()
self.uids = UidSet()
self.tasks = Queue()
self.workers = [Worker(name, self.tasks, self.pids, self.uids) \
for _ in range(0, nworkers)]
def status(self):
"Return status of task manager queue"
info = {'qsize':self.tasks.qsize(), 'full':self.tasks.full(),
'unfinished':self.tasks.unfinished_tasks,
'nworkers':len(self.workers)}
return {self.name: info}
def nworkers(self):
"""Return number of workers associated with this manager"""
return len(self.workers)
def spawn(self, func, *args, **kwargs):
"""Spawn new process for given function"""
pid = kwargs.get('pid', genkey(str(args) + str(kwargs)))
evt = threading.Event()
if not pid in self.pids:
self.pids.add(pid)
task = (evt, pid, func, args, kwargs)
self.tasks.put(task)
else:
# the event was not added to task list, invoke set()
# to pass it in wait() call, see joinall
evt.set()
return evt, pid
def remove(self, pid):
"""Remove pid and associative process from the queue"""
self.pids.discard(pid)
def is_alive(self, pid):
"""Check worker queue if given pid of the process is still running"""
return pid in self.pids
def clear(self, tasks):
"""
Clear all tasks in a queue. It allows current jobs to run, but will
block all new requests till workers event flag is set again
"""
_ = [t[0].clear() for t in tasks] # each task is return from spawn, i.e. a pair (evt, pid)
def joinall(self, tasks):
"""Join all tasks in a queue and quit"""
_ = [t[0].wait() for t in tasks] # each task is return from spawn, i.e. a pair (evt, pid)
def quit(self):
"""Put None task to all workers and let them quit"""
_ = [self.tasks.put(None) for _ in self.workers]
time.sleep(1) # let workers threads cool-off and quit
class _BatchWriter(object):
#: Truncate overly big items to that many bytes for the error message.
ERRMSG_DATA_TRUNCATION_LEN = 1024
def __init__(self, url, start, auth, size, interval, qsize,
maxitemsize, content_encoding, uploader, callback=None):
self.url = url
self.offset = start
self._nextid = count(start)
self.auth = auth
self.size = size
self.interval = interval
self.maxitemsize = maxitemsize
self.content_encoding = content_encoding
self.checkpoint = time.time()
self.itemsq = Queue(size * 2 if qsize is None else qsize)
self.closed = False
self.flushme = False
self.uploader = uploader
self.callback = callback
def write(self, item):
assert not self.closed, 'attempting writes to a closed writer'
data = jsonencode(item)
if len(data) > self.maxitemsize:
truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..."
temp_logger = logging.getLogger('HubstorageClient')
temp_logger.info('BatchWriter: item size is {}'.format(len(data)))
temp_logger.info('BatchWriter: data type is {}'.format(type(data)))
temp_logger.info('BatchWriter: item type {}'.format(type(item)))
js = json.loads(data)
temp_logger.info('BatchWriter: data keys {}'.format(js.keys()))
if getattr(item, 'keys', None):
temp_logger.info('BatchWriter: item keys {}'.format(item.keys()))
for key, value in item.items():
d = {}
d[key] = value
res = jsonencode(d)
temp_logger.info('BatchWriter: key <{}> JSONENCODE size is {}, value type {}'.format(key, len(res), type(value)))
if getattr(value, '__len__', None):
temp_logger.info('BatchWriter: key <{}> value size is {}'.format(key, len(value)))
raise ValueTooLarge(
'Value exceeds max encoded size of {} bytes: {!r}'
.format(self.maxitemsize, truncated_data))
self.itemsq.put(data)
if self.itemsq.full():
self.uploader.interrupt()
return self._nextid.next()
def flush(self):
self.flushme = True
self._waitforq()
self.flushme = False
def close(self, block=True):
self.closed = True
if block:
self._waitforq()
def _waitforq(self):
self.uploader.interrupt()
self.itemsq.join()
def __str__(self):
return self.url
class DatabaseAL (threading.Thread):
"database abstraction layer"
def __init__(self, server, enabled = 0):
threading.Thread.__init__(self)
self.name = "DatabaseAL worker thread"
self.setDaemon(True)
self.log = logging.getLogger('DatabaseAL')
#self.client = SensorClient()
self.server = server
self.enabled = enabled
if self.enabled == 0:
self.log.warning("OpentTSDB is disabled. Data will not be saved in OpenTSDB.")
self.dataq = Queue(maxsize = 1024*1024)
self.shutdown_evt = Event()
def append(self, timestamp, values, tags):
"append data to the db"
#self.log.debug("Got data: {} {} {}".format(timestamp, values, tags))
# append a tuple
#if type(tags) is SensorTag:
# print tags
if self.dataq.full():
self.log.error("DataQ is full! Data is lost!")
else:
self.dataq.put( (timestamp, values, tags) )
def busy(self):
return not self.dataq.empty()
def put(self, worklist):
# format is (unix timestamp, value, Tags)
m = set()
t = set()
p = []
for w in worklist:
ts, val, tags = w
if tags.metric not in m:
m.add(tags.metric)
for k,v in tags.toTagData().items():
kv_str = "{}={}".format(k,v)
if kv_str not in t:
t.add(kv_str)
p += [{'metric': tags.metric,
'timestamp': ts,
'value': val,
'tags': tags.toTagData(),
}]
#self.log.debug("putting data: "+json.dumps(p))
self.log.info("putting metrics:"+str(m))
self.log.info("with tags:"+str(t))
r = requests.post('{}/put?details'.format(self.server),
data = json.dumps(p))
if r.status_code != requests.codes.ok:
self.log.error("put failed: "+ r.text)
for ts, val, tags in worklist:
self.append( ts, val, tags)
#else:
#self.log.info("put {} samples.".format(len(worklist)))
def run(self):
while not self.shutdown_evt.wait(1):
if self.shutdown_evt.is_set():
break
if self.dataq.empty():
continue
worklist = []
while not self.dataq.empty() and len(worklist) < 2500:
worklist.append(self.dataq.get())
self.log.info("Putting a batch of {} samples".format(len(worklist)))
# now batch processes the worklist
if len(worklist) > 0:
try:
#i = self.client.multiplePut(worklist)
if self.enabled > 0:
self.put(worklist)
self.log.info("Pushed {} samples to OpenTSDB.".format(len(worklist)))
except Exception as x:
self.log.error("Push failed, error ={}; worklist = {}".format(x, worklist))
self.log.info("Shutdown")
def stop(self):
self.shutdown_evt.set()
self.join(5)
class NFQ:
##
# @param in_size number of encoded features
# @param out_size = number of possible actions
def __init__(self, in_size, out_size, learning_rate=0.75, model_path=None, weights_path=None):
assert in_size > 1
assert type(in_size) is int
assert out_size > 1
assert type(out_size) is int
self.in_size = in_size
self.out_size = out_size
self.gamma = learning_rate
if model_path is None:
self.model = Sequential()
self.model.add(Dense(64, input_dim=in_size, init='lecun_uniform'))
self.model.add(Activation('relu'))
# self.model.add(BatchNormalization())
# self.model.add(Dropout(0.2))
self.model.add(Dense(40, init='lecun_uniform'))
self.model.add(Activation('relu'))
# self.model.add(Dropout(0.2))
self.model.add(Dense(out_size, init='lecun_uniform'))
self.model.add(Activation('linear'))
else:
assert weights_path is not None
self.model = model_from_json(open(model_path).read())
self.model.load_weights(weights_path)
self.model.compile(
loss='mse', # maybe binary_crossentrpy?
optimizer='rmsprop'
)
self.transitions = Queue(25000)
##
# Calculate target Q-fun values and train a NN on it
def train(self):
queue_size = self.transitions.qsize()
np_data = list(self.transitions.queue)
np_data = np.array(np_data)
r = np.random.randint(queue_size-1, size=3000)
np_data = np_data[r, :]
# trim the input data of neural net because it also contains
# unwanted state' and reward information => need it only for target Q
in_data = np.delete(np_data, np.s_[self.in_size::], 1)
out_data = self.get_training_data(np_data)
# stop_cb = EarlyStopping(monitor='val_loss', patience=0, verbose=0, mode='auto')
print 'Learning...'
hist = self.model.fit(
in_data,
out_data,
nb_epoch=500,
batch_size=256,
verbose=0,
validation_split=0.4
)
print(
'Loss: from ',
hist.history['loss'][0],
' to ',
hist.history['loss'][-1]
)
# callbacks=[stop_cb]
##
# Predicts next action, which is the one maximizing the Q function in current state
def predict_action(self, state):
q = self.model.predict(state)
return np.argmax(q)
##
# Process training data into correct format (transitions)
def get_training_data(self, data):
out_data = list()
for row in data:
reward = row[-1]
selected_action = row[self.in_size]
next_state = row[self.in_size+1:-1]
next_state = next_state.reshape(1, next_state.size)
predicted_Q = self.model.predict(next_state)
maxQ = np.max(predicted_Q)
minQ = np.min(predicted_Q)
out = np.zeros((self.out_size,))
if reward >= 1:
out[int(selected_action)] = 1
elif reward < 0:
out[int(selected_action)] = -1
else:
out[int(selected_action)] = reward + self.gamma*maxQ
for i in xrange(self.out_size):
if i != int(selected_action):
out[i] = minQ
out_data.append(out)
return np.array(out_data)
##
# Add transition to training dataset
# transition has form [st, a, st+1, r]
def add_transition(self, transition):
# transition must be a numpy array
assert type(transition) is np.ndarray
if self.transitions.full():
self.transitions.get()
self.transitions.put(transition)
##
# Save NN to external file
# @param model_path File containing NN architecture
# @param weights_path File containing NN weights
def save_net(self, model_path=None, weights_path=None):
if model_path is None:
model_path = 'model.json'
if weights_path is None:
weights_path = 'weights.h5'
# remove old file if exists
try:
os.remove(weights_path)
except OSError:
pass
try:
self.model.save_weights(weights_path)
open(model_path, 'w').write(self.model.to_json())
except Exception as e:
print('Saving a model failed')
print(type(e))
print(e)
