class Invalidator(object):
def __init__(self, interval):
self.interval = interval
self._timers = []
self.queue = PriorityQueue()
self.repeater = Repeater(self.interval, self._worker)
atexit.register(self.stop)
self.repeater.start()
def stop(self):
self.repeater.stop()
self.queue.join()
def request(self, qst, instances):
"""Record a lookup request."""
self.queue.put((time.time(), (qst, set(i.id for i in instances))), False)
def _worker(self):
try:
_, item = self.queue.get(False)
except Queue.Empty:
return
qst, old_instances = item
instances = lookup_instance_by_name(qst.qinfo.qname_str)
if set(i.id for i in instances) != old_instances:
invalidateQueryInCache(qst, qst.qinfo)
else:
self.queue.put((time.time(), (qst, old_instances)), False)
self.queue.task_done()
class MultiQueue(object):
"""
Simple priority queue interface to push/pull tasks
Priority queue maintain the order by first element of the tuple, no futher ordering is guarantied
"""
def __init__(self):
self.queue = PriorityQueue()
def empty(self):
return self.queue.empty()
def pull_nowait(self):
task_data = self.queue.get_nowait()
if task_data:
(EnterTime, User, Task) = task_data
self.queue.task_done()
return (User, Task)
def pull_wait(self, wait):
try:
task_data = self.queue.get(block=True, timeout=wait)
(EnterTime, User, Task) = task_data
self.queue.task_done()
return (User, Task)
except Empty:
return None
def push(self, User, Tasks):
EnterTime = time()
for task in Tasks:
self.queue.put_nowait((EnterTime, User, task))
class CrawlManager(object):
def __init__(self, config):
try: self.queue_size = int(config.get('queue_size'))
except: self.queue_size = 1000
try: self.num_threads = int(config.get('num_threads'))
except: self.num_threads = 10
self.queue = PriorityQueue(maxsize=self.queue_size)
self.processors = ProcessorManager(config)
for i in range(self.num_threads):
t = Thread(target=self.queue_target,
name="CrawlDaemon-%s" % i)
t.daemon = True
t.start()
def queue_target(self):
while True:
try:
queue_entry = self.queue.get(True)
self.handle(queue_entry[1])
self.queue.task_done()
except Exception, e:
log.exception(e)
class __impl:
def __init__(self, worker_amount):
self._queue = PriorityQueue()
self._workers = [Worker(i,self) for i in range(worker_amount)]
self._locked = False
self._waiter = {}
self.running = False
self.browser_mgr = BrowserManager()
def get_waiter(self, named):
if not named in self._waiter:
self._waiter[named] = Event()
return self._waiter[named]
def wait(self, named ,seconds):
self.get_waiter(named).wait(seconds)
def get_job(self, worker):
while (self.running and (self._locked or self._queue.empty())):
self.wait('get_job_for'+str(worker._id), 0.1)
self._locked = True
job = self._queue.get()
self._queue.task_done()
self._locked = False
return job
def put_job(self, job):
self._queue.put(job)
def start_workers(self):
self.browser_mgr.bootup()
self.running = True
for worker in self._workers:
worker.start_working(self)
def stop_workers(self):
self.running = False
for worker in self._workers:
worker.stop_working()
def __del__(self):
self.browser_mgr.shutdown()
class Simulator(object):
def __init__(self, N, A, *args):
self.curTime = 0
self.maxTime = 1e6
self.A = A
self.W = 1e6
self.L = 12000
self.tickLength = 50000
self.eventStream = PriorityQueue()
self.channel = set()
self.comps = []
self.mediumSpeed = 5e-5
self.numCollisions = 0
for i in range(N):
self.comps.append(
Computer(self, self.eventStream, i, args[0], args[1]))
def simulate(self):
bar = progressbar.ProgressBar(maxval=1.0)
bar.start()
while self.curTime < self.maxTime:
if not self.eventStream.empty():
while self.eventStream.queue[0].actionTime == self.curTime:
e = self.eventStream.get()
if e.valid:
e.act()
self.eventStream.task_done()
else:
raise Exception("No events to process")
self.curTime += 1
bar.update(self.curTime / self.maxTime)
def get_stats(self):
packets_all = [packet for comp in self.comps for packet in comp.packets]
packets_all = filter(lambda p: (True if p.service_started else False), packets_all)
packets_service = map(lambda x: x.service_time(), filter(
lambda x: (True if x.service_done else False), packets_all))
packets_delay = map(lambda x: x.packet_delay(), filter(
lambda x: (True if x.service_done else False), packets_all))
packets_lost = filter(lambda x: (True if x.lost else False), packets_all)
return [
("number of all packets attempted", len(packets_all)),
("number of all packets sent successfully", len(packets_delay)),
("ratio of packets sent", float(len(packets_delay)) / len(packets_all)),
("average throughput of network (Mbps)", (self.L / 1e6) /
ticksToSecs(reduce(lambda x, y: (x + y) / 2, packets_service), self.tickLength)),
("average packet delay (mS)", ticksToSecs(
reduce(lambda x, y: (x + y) / 2, packets_delay), self.tickLength) * 1000),
("average number of collisions", self.numCollisions),
("average number of lost packets", len(packets_lost))
]
def send_socket(ws):
clock = PseudoClock()
clock.start(start_time=1415491200, time_scale=1)
logger.info("Connected to client for time %s" % clock.pseudo_start_time)
# shared memory
q = PriorityQueue()
def look_ahead_loop():
highest_time_checked = clock.pseudo_start_time
while True:
if clock.pseudo_now() >= clock.local_now():
break
if clock.pseudo_now() < highest_time_checked - LOOK_AHEAD_TIME:
sleep(0.1)
continue
start = highest_time_checked
end = highest_time_checked + CHUNK_LENGTH
logger.info('Queueing Chunk: %s to %s' % (start, end))
for sensor in sensors:
data = get_data(sensor, start, end)
for d in data:
t = dateutil.parser.parse(d['timestamp'])
v = d['value']
q.put((t.utctimetuple(), Event(sensor, t, v)))
highest_time_checked = end
t = Thread(target=look_ahead_loop)
t.daemon = True
t.start()
def read_seek():
new_start_time = ws.receive()
while True:
try:
q.get(False)
except Empty:
clock.start(start_time=new_start_time, time_scale=1)
t2 = Thread(target=read_seek)
t2.daemon = True
t2.start()
while True:
key, event = q.get()
if clock.pseudo_now() < event.time:
q.put((key, event))
sleep(0.05)
else:
logger.info('Sending: %s' % json.dumps(event.to_dict()))
ws.send(json.dumps(event.to_dict()))
q.task_done()
def outlierRejection(graph, K, percent=5.0, max_dist=5.0):
"""
Examine graph and remove some top percentage of outliers
and those outside a certain radius.
"""
# iterate through all points
pq = PriorityQueue()
marked_keys = []
for key, entry in graph["3Dmatches"].iteritems():
X = entry["3Dlocs"]
# mark and continue if too far away from the origin
if np.linalg.norm(X) > max_dist:
marked_keys.append(key)
continue
# project into each frame
errors = []
for frame, x in zip(entry["frames"], entry["2Dlocs"]):
frame -= graph["frameOffset"]
Rt = graph["motion"][frame]
proj = fromHomogenous(K * Rt * toHomogenous(X))
diff = proj - x
err = np.sqrt(np.multiply(diff, diff).sum())
#print (frame, err)
errors.append(err)
# get mean error and add to priority queue
# (priority is reciprocal of error since this is a MinPQ)
mean_error = np.array(errors).mean()
pq.put_nowait((1.0 / mean_error, key))
# remove worst keys
N = max(
0,
int((percent / 100.0) * len(graph["3Dmatches"].keys())) -
len(marked_keys))
for i in range(N):
score, key = pq.get_nowait()
del graph["3Dmatches"][key]
pq.task_done()
# remove keys out of range
for key in marked_keys:
del graph["3Dmatches"][key]
print "Removed %d outliers." % (N + len(marked_keys))
class qController (object):
def __init__ (self, default_event_handler, timer_interval):
self.controller = default_event_handler
self.timer_interval = timer_interval
self.TickTock = PriorityQueue ()
return
def stop (self):
q = qData ('stop')
self.TickTock.put ((qPriorityHigh, q))
return
def tick (self):
time.sleep (self.timer_interval)
q = qData ('tick')
self.TickTock.put ((qPriorityLow, q))
def event (self, event_type, data=None, handler=None):
q = qData (event_type, data, handler)
if handler is None:
q.event_handler = self.controller
self.TickTock.put ((qPriorityNormal, q))
def run (self):
t = threading.Thread (target=qController_tick, args=(self,)) # add a delayed event to the queue
t.start ()
while True:
p, q = self.TickTock.get ()
if q.event_type == 'stop':
break
event_handler = q.event_handler
if q.event_type == 'tick':
if event_handler is None: # assume this is our local timer
if self.controller.tock (q.event_data) == False:
break
t = threading.Thread (target=qController_tick, args=(self,)) # add a new delayed event to the queue
t.start ()
else: # another ticker wants a tock...
if event_handler.tock (q.event_data) == False:
break
elif event_handler is not None:
if event_handler.event (q.event_type, q.event_data) == False:
break
self.TickTock.task_done ()
return
class Executor(Thread):
""" This class applies the baboonsrv workflow task by task.
"""
def __init__(self):
""" Initialize the executor thread. Executor has only ONE
thread because all the task MUST BE executed one after
another.
"""
Thread.__init__(self)
# A priority queue to store all tasks. The priority is important in
# order to have an endtask with high priority. When someone puts an
# EndTask into this queue, the next task will be that and all other
# tasks will be ignored.
self.tasks = PriorityQueue()
def run(self):
""" Consume on the tasks queue and run each task until an
endtask.
"""
from baboon.baboond.task import EndTask
# The endtask is a flag to indicate if it's the end of life of
# the server or not.
endtask = False
# While the endtask is False, continue to consume on the
# queue.
while not endtask:
# Take the next task...
task = self.tasks.get()
# Verify that it's not a EndTask.
endtask = type(task) == EndTask
# Run it !
try:
self.logger.debug('Running a new task...')
task.run()
except BaboonException as err:
self.logger.error(err)
# Mark the task finished
self.logger.debug('A task has been finished...')
self.tasks.task_done()
self.logger.debug('Remaining task(s): %s' % self.tasks.qsize())
self.logger.debug('The executor thread is now finished.')
class WorkerManager(object):
def __init__(self):
self.running_threads = []
self.ideal_threads = 0
self.target = None
self.inbox = PriorityQueue()
self.running = False
self.ishandler = True
def set_target(self, t):
self.target = t
def threadloop(self):
#print "threadloop started"
while self.running:
if self.ishandler:
try:
x = self.inbox.get(False)
#print "calling handler"
self.target(x)
self.inbox.task_done()
except Empty:
pass
else:
self.target()
def start(self):
self.running = True
while (len(self.running_threads) < self.ideal_threads):
t = threading.Thread(target=self.threadloop)
t.daemon = True
t.start()
self.running_threads.append(t)
def stop(self):
self.running = False
def checkup(self):
for t in self.running_threads:
if not t.isAlive():
del t
self.start()
def putjob(self, datum):
if self.running:
self.inbox.put(datum, True)
self.checkup()
else:
raise Error("added job to dead worker")
def outlierRejection(graph, K, percent=5.0, max_dist=5.0):
"""
Examine graph and remove some top percentage of outliers
and those outside a certain radius.
"""
# iterate through all points
pq = PriorityQueue()
marked_keys = []
for key, entry in graph["3Dmatches"].iteritems():
X = entry["3Dlocs"]
# mark and continue if too far away from the origin
if np.linalg.norm(X) > max_dist:
marked_keys.append(key)
continue
# project into each frame
errors = []
for frame, x in zip(entry["frames"], entry["2Dlocs"]):
frame -= graph["frameOffset"]
Rt = graph["motion"][frame]
proj = fromHomogenous(K * Rt * toHomogenous(X))
diff = proj - x
err = np.sqrt(np.multiply(diff, diff).sum())
#print (frame, err)
errors.append(err)
# get mean error and add to priority queue
# (priority is reciprocal of error since this is a MinPQ)
mean_error = np.array(errors).mean()
pq.put_nowait((1.0 / mean_error, key))
# remove worst keys
N = max(0, int((percent/100.0) * len(graph["3Dmatches"].keys())) - len(marked_keys))
for i in range(N):
score, key = pq.get_nowait()
del graph["3Dmatches"][key]
pq.task_done()
# remove keys out of range
for key in marked_keys:
del graph["3Dmatches"][key]
print "Removed %d outliers." % (N + len(marked_keys))
class WorkerManager(object):
def __init__(self):
self.running_threads = []
self.ideal_threads = 0
self.target = None
self.inbox = PriorityQueue()
self.running = False
self.ishandler = True
def set_target(self,t):
self.target = t
def threadloop(self):
#print "threadloop started"
while self.running:
if self.ishandler:
try:
x = self.inbox.get(False)
#print "calling handler"
self.target(x)
self.inbox.task_done()
except Empty:
pass
else:
self.target()
def start(self):
self.running = True
while(len(self.running_threads) < self.ideal_threads):
t = threading.Thread(target=self.threadloop)
t.daemon = True
t.start()
self.running_threads.append(t)
def stop(self):
self.running = False
def checkup(self):
for t in self.running_threads:
if not t.isAlive():
del t
self.start()
def putjob(self, datum):
if self.running:
self.inbox.put(datum, True)
self.checkup()
else:
raise Error("added job to dead worker")
class PriorityTasks(object):
"""
This class issues a task that must be executed immediately
"""
def __init__(self):
self.peak = None
self.queue = PriorityQueue()
self.mutex = Lock()
def get_tasks(self, current_time):
with self.mutex:
if self.peak is None:
return None
elif current_time < self.peak[0]:
return None
else:
result = [self.peak[1]]
self.peak = None
while not (self.queue.empty()):
tmp = self.queue.get()
if current_time >= tmp[0]:
result.append(tmp[1])
else:
self.peak = tmp
break
return result
def put(self, item):
with self.mutex:
if self.peak is None:
self.peak = item
elif item[0] < self.peak[0]:
self.queue.put(self.peak)
self.peak = item
else:
self.queue.put(item)
def empty(self):
with self.mutex:
return (self.peak is None) and (self.queue.empty())
def clear(self):
with self.mutex:
while not self.queue.empty():
try:
self.queue.get(False)
except Empty:
continue
self.queue.task_done()
def play_video2(API, video_url = common.args.url):
try:
qbitrate = common.args.quality
except:
qbitrate = None
video_url2 = 'stack://'
threads = []
segments = []
closedcaption = []
queue = PriorityQueue()
video_data = connection.getURL(API + 'playlists/%s/videos.json' % video_url)
video_tree = simplejson.loads(video_data)
for i, video_item in enumerate(video_tree['playlist']['videos']):
worker = Thread(target = get_videos, args = (queue, i, video_item, qbitrate))
worker.setDaemon(True)
worker.start()
threads.append(worker)
[x.join() for x in threads]
while not queue.empty():
video_data2 = queue.get()
video_url2 += video_data2[1] + ' , '
segments.append(video_data2[2])
closedcaption.append((video_data2[3], video_data2[2], video_data2[0]))
player._segments_array = segments
filestring = 'XBMC.RunScript(' + os.path.join(ustvpaths.LIBPATH, 'proxy.py') + ', 12345)'
xbmc.executebuiltin(filestring)
finalurl = video_url2[:-3]
localhttpserver = True
time.sleep(20)
if (addon.getSetting('enablesubtitles') == 'true') and closedcaption:
convert_subtitles(closedcaption)
player._subtitles_Enabled = True
item = xbmcgui.ListItem(path = finalurl)
queue.task_done()
if qbitrate is not None:
item.setThumbnailImage(common.args.thumb)
item.setInfo('Video', { 'title' : common.args.name,
'season' : common.args.season_number,
'episode' : common.args.episode_number,
'TVShowTitle' : common.args.show_title })
xbmcplugin.setResolvedUrl(pluginHandle, True, item)
while player.is_active:
player.sleep(250)
def getTwoBestClusters(self):
best_clusters = PriorityQueue()
cnt = 0
for centroid, stats in self.clusters.iteritems():
avg_mass = stats["total_mass"] / stats["size"]
best_clusters.put_nowait((avg_mass, centroid))
cnt += 1
if cnt < 2:
print "Warning. Only found %d clusters." % cnt
return [], []
avg_mass, cluster1 = best_clusters.get_nowait(); best_clusters.task_done()
avg_mass, cluster2 = best_clusters.get_nowait(); best_clusters.task_done()
centroids = [cluster1, cluster2]
scores = [self.clusters[cluster1]["total_mass"],
self.clusters[cluster2]["total_mass"]]
return centroids, scores
class MonitorFactory(Thread):
def __init__(self, monitorClass, valid_paths, config):
Thread.__init__(self)
self.processor = None
self.config = config
self.valid_paths = valid_paths
self.monitorClass = monitorClass
self.children = []
self.upload_queue = PriorityQueue()
for path in self.valid_paths:
# the children emit signals to this factor to communicate back changes
self.children.append(self.monitorClass(self, path, config))
def queue(self, item):
if not self.config.tokenValid:
return
logger.debug("(%s) Queueing file %s" % (self.upload_queue.qsize(), item))
self.upload_queue.put(item)
def stop(self):
for monitor in self.children:
monitor.__del__()
monitor.join()
self.upload_queue.put((0, None))
# Not waiting for the thread isn't nice but that way we get out fast!
# FIXME
# self.join()
def Run(self):
for child in self.children:
child.Run()
self.start()
def run(self):
# On startup check the token for vadility
(valid, user) = self.testToken()
if valid:
self.config.tokenValid = True
self.config.gui.setStatus("ok", "Authenticated as %s. Ready to upload data." % user)
elif user == "error":
self.config.gui.setStatus("error", "Invalid response from server.")
else:
self.config.tokenValid = False
self.config.gui.setStatus("error", "Invalid application token.")
# Work on the upload queue
while True:
(prio, item) = self.upload_queue.get()
if item is None:
break
if not self.processor.OnNewFile(item):
logger.warn("Error uploading file, requeueing")
self.queue(item)
time.sleep(1)
self.upload_queue.task_done()
# Done. Indicate that sentinel was received and return
self.upload_queue.task_done()
return
def setProcessor(self, processor):
self.processor = processor
def testToken(self):
return self.processor.upload_client.check_token()
class RSock:
def __init__(self, sock, addr, cwnd, ploss, pcorr):
self.init = False
self.end = False
self.requesting = False
self.sock = sock
self.addr = addr
self.ploss = ploss
self.pcorr = pcorr
self.cwnd = cwnd
if self.cwnd <= 0:
self.cwnd = WINDOW_SIZE
self.attempt = 0
self.timer = None
self.lock = threading.Lock()
# PQueue will sort packets by segnum, so when inserting in queue, packets will be like:
# Acks - segnum = 0
# Failed packets (for their segnum is securely < unset packets)
# Unsent packets
# So ordering is guaranteed and ACKs should be sent immediately
self.buff = PQueue() # packets to send
self.waiting = PQueue(cwnd) # packets wating for ACK
self.seg = 0 # current packet
self.ack = 0 # last valid ack
self.nextSeg = random.randint(1, 1000) # next expected packet
def start(self):
while not self.end:
packet = self.buff.get(True) # block until another packet is added
if self.timer == None:
self.playTimer()
# only regular packets should be recent.
# acks require the other side of the socket to be sent again, so we don't need to put on waiting queue
if packet.ack == 0 and (self.requesting or packet.con == 0):
self.lock.acquire(True) # wait for timeout function if needed
self.lock.release(
) # release it BEFORE put function or it will deadlock
self.waiting.put(packet) # block after timer!
print "Sending seg " + str(packet.seg) + " to " + str(
self.addr
) # don't log ack/con for they are logged somewhere else
# inform Queue we're done with the packet we got
self.buff.task_done()
if random.randint(1, 100) < self.ploss:
print "Simulating delayed packet"
continue # simulate lost packet
self.sock.sendto(packet.wrap(), self.addr)
if packet.end:
print "Sending connection end"
if (
packet.end and packet.ack
) or self.attempt >= ATTEMPT: # should this avoid dced socket to be maintained
self.endCon()
def endCon(self):
if self.attempt >= ATTEMPT:
print "Forcing close on socket due to response lack"
else:
print "Ending connection to " + str(self.addr)
self.end = True
def playTimer(self):
if self.timer != None:
self.timer.cancel() # stop current timer
self.timer = threading.Timer(ACK_TIMEOUT, self.timeout)
self.timer.start()
def timeout(self):
if not self.end and not self.waiting.empty():
print "Timed out! Enqueuing window again..."
self.lock.acquire(True) # block other thread
self.attempt += 1 # should this avoid dced socket to be maintained
# expected ack didn't arrive... send window again!
while not self.waiting.empty():
packet = self.waiting.get()
if packet.seg >= self.ack: # no need to resend acked packets...
self.buff.put(packet)
self.waiting.task_done()
self.lock.release()
if not self.end:
self.playTimer()
def enqueuePacket(self, data):
packet = Packet(data)
if self.init:
packet.seg = self.seg
self.seg += 1
else:
self.requesting = True
packet.con = self.nextSeg # con packet: seg = 0; ack = 0; con = nextSeg
self.buff.put(packet)
def endPacket(self):
packet = Packet('', self.seg, end=1)
self.seg += 1
self.buff.put(packet)
def errPacket(self):
packet = Packet('', self.seg, err=1)
self.seg += 1
self.buff.put(packet)
def ackPacket(self, ack, endAck):
self.buff.put(Packet('', 0, ack, endAck))
def receivePacket(self, data):
packet = Packet(data)
packet.unwrap()
self.attempts = 0
if not packet.validChecksum() or random.randint(
1, 100
) < self.pcorr: # do not receive broken packets or simulate a broken packet
print "Bad checksum received on seg: " + str(
packet.seg) + " ack: " + str(packet.ack)
return None
if packet.con > 0:
self.seg = packet.con # set seg with nextSeg received in con field
self.ack = packet.con
self.init = True
if not self.requesting: # this socket received the connection request, then ack it
print "Connection request from " + str(self.addr)
self.buff.put(Packet('', end=packet.end, con=self.nextSeg)
) # ack with nextSeg expected on con field
return packet
else:
print "Connection established!"
elif (packet.ack > 0): # ack packet
self.lock.acquire(True)
if packet.ack == self.ack: # expected ack!
self.ack += 1
self.playTimer() # ack received, start timer again
print "Received expected ack " + str(
packet.ack) + " on connection " + str(self.addr)
if packet.end: # ack for end packet received
self.endCon()
self.lock.release()
elif packet.seg < self.nextSeg:
self.ackPacket(
packet.seg, packet.end
) # old packet received again, ACK might be lost.. send it again
print "Duplicated packet " + str(
packet.seg) + ". Sending ack again and ignoring"
elif self.nextSeg == packet.seg: # expected seg!
self.nextSeg += 1
self.ackPacket(packet.seg, packet.end)
print "Acking packet " + str(packet.seg)
return packet
#else:
# print "Bad packet received! Seg: " + str(packet.seg) + " expected: " + str(self.nextSeg) + " ack: " + str(packet.ack)
return None
class scanmatcher_localizer(object):
def __init__(self):
self._static_map_req = GetMapRequest()
self._scan_msg = LaserScan()
self._map_msg = OccupancyGrid()
self._initialpose_msg = PoseWithCovarianceStamped()
# parameters
self._use_static_map = rospy.get_param('~use_static_map', default=True)
self._init_x = rospy.get_param('~init_x', default=0.0)
self._init_y = rospy.get_param('~init_y', default=0.0)
self._init_phi = rospy.get_param('~init_phi', default=0.0)
# service clients
try:
rospy.wait_for_service('/static_map', timeout=3)
except ROSException as e:
print(e.message + '. Make sure that service node is running!')
self._static_map_sc = rospy.ServiceProxy('/static_map', GetMap)
# tf broadcasters
self._map_odom_tfb = tf.TransformBroadcaster()
# tf listeners
self._odom_base_link_tfl = tf.TransformListener()
self._base_link_base_laser_tfl = tf.TransformListener()
# user init
# processing queue
self._scan_queue = PriorityQueue(maxsize=2)
# get static map
if self._use_static_map:
static_map_res = self._static_map_sc(self._static_map_req)
static_map_res.map.data = list(static_map_res.map.data)
# initialize map with received one
self._map = pymrpt.maps.COccupancyGridMap2D()
self._map.from_ROS_OccupancyGrid_msg(static_map_res.map)
# print output
print 'received initial STATIC map!'
print static_map_res.map.info
else:
# initialize empty map and wait for published ones
self._map = pymrpt.maps.COccupancyGridMap2D()
# setup transformations
self._tf = tf.TransformerROS()
self._map_to_odom = pymrpt.poses.CPose2D()
self._map_to_base = pymrpt.poses.CPose2D()
self._odom_to_base = pymrpt.poses.CPose2D()
# setup initial map to base tf ("/odom -> /base_link" also should be (0,0,0))
self._map_to_base.x = self._init_x
self._map_to_base.y = self._init_y
self._map_to_base.phi = self._init_phi
# setup scanmatcher lock
self._scanmatcher_lock = Lock()
# end of user init
# subscribers
rospy.Subscriber('/scan',
LaserScan,
self._scan_callback,
queue_size=10)
rospy.Subscriber('/map',
OccupancyGrid,
self._map_callback,
queue_size=10)
rospy.Subscriber('/initialpose',
PoseWithCovarianceStamped,
self._initialpose_callback,
queue_size=10)
def get_odom_base_link_tf(self, stamp=None):
pose = PoseStamped()
if stamp is None:
stamp = rospy.Time(0)
try:
trans, rot = self._odom_base_link_tfl.lookupTransform(
'/odom', '/base_link', stamp)
except (tf.LookupException, tf.ConnectivityException,
tf.ExtrapolationException):
# TODO: implement TransformListener.lookupTransform() exception handling!
rospy.logwarn('Error while listening to /odom -> /base_link tf')
return pose, False
pose.pose.position.x = trans[0]
pose.pose.position.y = trans[1]
pose.pose.position.z = trans[2]
pose.pose.orientation.x = rot[0]
pose.pose.orientation.y = rot[1]
pose.pose.orientation.z = rot[2]
pose.pose.orientation.w = rot[3]
return pose, True
def get_base_link_base_laser_tf(self, stamp=None):
pose = PoseStamped()
if stamp is None:
stamp = rospy.Time(0)
try:
trans, rot = self._base_link_base_laser_tfl.lookupTransform(
'/base_link', '/base_laser', stamp)
except (tf.LookupException, tf.ConnectivityException,
tf.ExtrapolationException):
# TODO: implement TransformListener.lookupTransform() exception handling!
rospy.logwarn(
'Error while listening to /base_link -> /base_laser tf')
return pose, False
pose.pose.position.x = trans[0]
pose.pose.position.y = trans[1]
pose.pose.position.z = trans[2]
pose.pose.orientation.x = rot[0]
pose.pose.orientation.y = rot[1]
pose.pose.orientation.z = rot[2]
pose.pose.orientation.w = rot[3]
return pose, True
def send_map_odom_tf(self, pose):
translation = pose.pose.position
rotation = pose.pose.orientation
self._map_odom_tfb.sendTransform(
(translation.x, translation.y, translation.z),
(rotation.x, rotation.y, rotation.z, rotation.w),
pose.header.stamp, '/odom', '/map')
def _scan_callback(self, _scan_msg):
# put scans into queue
try:
self._scan_queue.put_nowait(
[_scan_msg.header.stamp.to_nsec(), _scan_msg])
except Full:
rospy.logwarn('Dropped scan. Processing queue is full!')
pass
def run(self):
# create worker thread (we do not .join() since it closes itself by ros shutdown)
scanmatcher_thread = Thread(target=self.scanmatcher_worker)
scanmatcher_thread.start()
# keep it running
rospy.spin()
def scanmatcher_worker(self):
# vars
has_initial_scan = False
# initialize ICP
icp_options = pymrpt.slam.CICP.TConfigParams()
icp = pymrpt.slam.CICP(icp_options)
# loop
while not rospy.is_shutdown():
# get scan from queue
try:
item = self._scan_queue.get(timeout=5)
except Empty:
rospy.logwarn(
'Got no scan from queue since 5 seconds. Scanmatcher will shutdown now!'
)
continue
self._scan_msg = LaserScan()
self._scan_msg = item[1]
self._scan_msg.ranges = list(self._scan_msg.ranges)
rospy.loginfo('received scan...')
# get laser scanner pose
laser_pose, ok = self.get_base_link_base_laser_tf()
self._sensor_pose = pymrpt.poses.CPose3D()
self._sensor_pose.from_ROS_Pose_msg(laser_pose.pose)
# self get odom pose
odom_pose, ok = self.get_odom_base_link_tf()
self._odom_to_base = pymrpt.poses.CPose2D()
self._odom_to_base.from_ROS_Pose_msg(odom_pose.pose)
# acquire lock
self._scanmatcher_lock.acquire()
# update current stamp for publishers
self._current_stamp = self._scan_msg.header.stamp
# convert data
observation = pymrpt.obs.CObservation2DRangeScan()
observation.from_ROS_LaserScan_msg(self._scan_msg,
self._sensor_pose)
# set current map from scan
current_map = pymrpt.maps.CSimplePointsMap()
current_map.loadFromRangeScan(observation)
# match maps
# take current maintained map to base pose as initial guess (absolut pose in map)
initial_guess = pymrpt.poses.CPosePDFGaussian(self._map_to_base)
# run ICP algorithm
aligned_pose, running_time, info = icp.AlignPDF(
self._map, current_map, initial_guess)
rospy.loginfo('init. guess: {}'.format(initial_guess.mean))
rospy.loginfo('icp goodness: {}'.format(info.goodness))
rospy.loginfo('icp run_time: {}'.format(running_time))
rospy.loginfo('aligned pose: {}'.format(aligned_pose.mean))
# check goodness
if info.goodness > .8:
rospy.loginfo('...updating pose in map...')
# update MRPT pose diff (this is actually the maintained tf)
self._map_to_base = aligned_pose.mean
# update last update time
last_update_time = self._current_stamp
else:
# warn if pose lost
rospy.logwarn('...lost pose in map...')
# update pose diff
map_to_base = pymrpt.poses.CPose2D(self._map_to_base)
map_to_base.inverse()
# map_to_base_pose = PoseStamped()
# map_to_base_pose.header.frame_id = 'base_link'
# map_to_base_pose.header.stamp = self._scan_msg.header.stamp
# map_to_base_pose.pose = map_to_base.to_ROS_Pose_msg()
# latest_tf_pose = PoseStamped()
# try:
# latest_tf_pose = self._tf.transformPose('odom', map_to_base_pose)
# except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
# rospy.logwarn('error while pose transformation')
odom_to_base = pymrpt.poses.CPose2D(self._odom_to_base)
odom_to_base.inverse()
self._map_to_odom.x = self._map_to_base.x - self._odom_to_base.x
self._map_to_odom.y = self._map_to_base.y - self._odom_to_base.y
self._map_to_odom.phi = self._map_to_base.phi - self._odom_to_base.phi
self._map_to_odom.normalizePhi()
print " /map -> /base", self._map_to_base, map_to_base
print "- /odom -> /base", self._odom_to_base, odom_to_base
print "= /map -> /odom", self._map_to_odom
map_to_odom_pose = PoseStamped()
map_to_odom_pose.pose = self._map_to_odom.to_ROS_Pose_msg()
# send "/map -> /odom" tf at scan time
map_to_odom_time = self._scan_msg.header.stamp + rospy.Duration(
2.0)
map_to_odom_pose.header.stamp = self._scan_msg.header.stamp
self.send_map_odom_tf(map_to_odom_pose)
# release lock
self._scanmatcher_lock.release()
# signalize work done
rospy.loginfo('...task done!')
self._scan_queue.task_done()
def _map_callback(self, _map_msg):
self._map_msg = _map_msg
if not self._use_static_map:
# lock scanmatcher
self._scanmatcher_lock.acquire()
# update map
self._map.from_ROS_OccupancyGrid_msg(self._map_msg)
# unlock scanmatcher
self._scanmatcher_lock.release()
# print output
rospy.loginfo('received new map!')
def _initialpose_callback(self, _initialpose_msg):
# lock scanmatcher
self._scanmatcher_lock.acquire()
# update map odom pose diff
self._map_to_base.from_ROS_Pose_msg(_initialpose_msg.pose.pose)
# unlock scanmatcher
self._scanmatcher_lock.release()
# print output
rospy.loginfo('received initial pose: {}'.format(self._map_to_base))
def play_video(BASE, video_uri=common.args.url, media_base=VIDEOURL):
video_url = media_base + video_uri
try:
qbitrate = common.args.quality
except:
qbitrate = None
video_url2 = 'stack://'
closedcaption = []
exception = False
queue = PriorityQueue()
segments = []
if 'feed' in video_uri:
feed_url = video_uri
else:
swf_url = connection.getRedirect(video_url, header={'Referer': BASE})
params = dict(
item.split("=") for item in swf_url.split('?')[1].split("&"))
uri = urllib.unquote_plus(params['uri'])
config_url = urllib.unquote_plus(params['CONFIG_URL'].replace(
'Other', DEVICE))
config_data = connection.getURL(config_url,
header={
'Referer': video_url,
'X-Forwarded-For': '12.13.14.15'
})
config_tree = BeautifulSoup(config_data, 'html.parser')
if not config_tree.error:
feed_url = config_tree.feed.string
uri = urllib.quote_plus(uri)
feed_url = feed_url.replace('{uri}', uri).replace(
'&', '&').replace('{device}', DEVICE).replace(
'{ref}', 'None').replace('{type}', 'network').strip()
else:
exception = True
error_text = config_tree.error.string.split('/')[-1].split('_')
if error_text[1] == 'loc':
params = dict(
item.split("=")
for item in config_url.split('?')[-1].split('&'))
common.show_exception('Geo', params['geo'])
if not exception:
feed_data = connection.getURL(
feed_url, header={'X-Forwarded-For': '12.13.14.15'})
video_tree = BeautifulSoup(feed_data,
'html.parser',
parse_only=SoupStrainer('media:group'))
video_segments = video_tree.find_all('media:content')
if not video_segments:
video_tree = BeautifulSoup(feed_data, 'html.parser')
common.show_exception(
video_tree.find('meta', property="og:site_name")['content'],
video_tree.find('meta', property="og:url")['content'])
exception = True
threads = []
for i, video_item in enumerate(video_segments):
try:
threads.append(
Thread(get_videos, queue, i, video_item, qbitrate, False))
except Exception, e:
print "Exception: ", e
[i.start() for i in threads]
[i.join() for i in threads]
while not queue.empty():
video_data2 = queue.get()
video_url2 += video_data2[1] + ' , '
segments.append(video_data2[2])
closedcaption.append((video_data2[3], int(video_data2[0])))
player._segments_array = segments
finalurl = video_url2[:-3]
time.sleep(20)
if (addon.getSetting('enablesubtitles')
== 'true') and closedcaption and detect_format is not None:
convert_subtitles(closedcaption)
player._subtitles_Enabled = True
item = xbmcgui.ListItem(path=finalurl)
if player._localHTTPServer:
filestring = 'XBMC.RunScript(' + os.path.join(
ustvpaths.LIBPATH, 'proxy.py') + ', 12345)'
xbmc.executebuiltin(filestring)
finalurl = video_url2[:-3]
#localhttpserver = True
time.sleep(20)
queue.task_done()
try:
item.setThumbnailImage(common.args.thumb)
except:
pass
try:
item.setInfo(
'Video', {
'title': common.args.name,
'season': common.args.season_number,
'episode': common.args.episode_number,
'TVShowTitle': common.args.show_title
})
except:
pass
xbmcplugin.setResolvedUrl(pluginHandle, True, item)
while player.is_active:
player.sleep(250)
class App(tk.Tk):
'''Tk window/label adjusts to size of image'''
def __init__(self, image_files, delay):
# the root will be self
tk.Tk.__init__(self)
# set x, y position only
self.geometry("{}x{}".format(self.winfo_screenwidth(), self.winfo_screenheight()))
self.state("normal")
self.status = True
self.delay = delay
# allows repeat cycling through the pictures
# store as (img_object, img_name) tuple
self.pictures = chain([None], cycle(image_files))
self._size = [self.winfo_screenwidth(), self.winfo_screenheight()]
self.maxqsize = 10
self._result = PriorityQueue(self.maxqsize)
self._next = 0
self._queued = 0
self.threading(self.maxqsize)
# self.pictures = cycle((tk.PhotoImage(file=image), image)
# for image in image_files)
self.picture_display = tk.Label(self)
self.picture_display.pack()
self.bind("<Control-Return>", self.toggle_fullscreen)
self.bind("<space>", self.toggle_pause)
self.bind("<Escape>", self.toggle_quit)
def threading(self, n = 1):
for _ in range(n):
thread = Thread(target = rescaled,
args = (next(self.pictures), self._size, self._result, self._queued))
self._queued += 1
thread.start()
# thread.join()
def show_slides(self):
self.set_size()
if not self.status:
self.after(self.delay, self.show_slides)
return
'''cycle through the images and show them'''
# next works with Python26 or higher
i, img_object = self._result.get()
while i != self._next:
self._result.put((i, img_object), False)
i, img_object = self._result.get()
if img_object != None:
del self.picture_display.image
img_object = ImageTk.PhotoImage(image = img_object)
self.picture_display.configure(image=img_object)
self.picture_display.image = img_object
# shows the image filename, but could be expanded
# to show an associated description of the image
self._next += 1
self.threading()
self.after(self.delay, self.show_slides)
def set_size(self):
self._size[:] = self.winfo_width(), self.winfo_height()
def run(self):
self.mainloop()
def toggle_fullscreen(self, event=None):
self.state = not self.state # Just toggling the boolean
self.attributes("-fullscreen", self.state)
return "break"
def toggle_pause(self, event=None):
self.status = not self.status
return "break"
def toggle_quit(self, event=None):
self._result.task_done()
self.destroy()
class TaskManager(QtCore.QObject):
"""The TaskManager is a stable priority queue of tasks. It takes them one by one, the
one with the highest priority first, and call its perform() method, then repeat until no tasks are left.
If two or more tasks have the same priority, it will take the one that was added first to the queue.
The TaskManager can be controlled asynchronously, as it runs the tasks in a separate thread."""
displayStatusBar = QtCore.pyqtSignal(QtCore.QString)
progressChanged = QtCore.pyqtSignal(int, int)
def __init__(self):
super(TaskManager, self).__init__()
# our main task queue
self.queue = PriorityQueue()
self.taskId = 0 # ID for the next task that will be generated
self.total = (
0
) # used to keep track of the total jobs that have been submitted (queue size decreases as we process tasks)
self.finished = [] # list of task IDs which have finished
self.lock = Lock()
log.info("Main GUI thread is: %d" % current_thread().ident)
self.shouldFinish = False
# ideally a pool of threads or sth like TBB or ThreadWeaver
self.workerThread = Thread(target=worker, args=(self,))
self.workerThread.daemon = True
self.workerThread.start()
def callback(self):
# we need to send a signal here so that it can be dealt with properly by the gui thread
# (we don't want to execute a gui callback using a worker thread)
self.progressChanged.emit(len(self.finished), self.total)
def add(self, task):
log.debug("TaskManager add task")
with self.lock:
# -task.priority because it always gets the lowest one first
# we need to put the time as well, because Queue uses heap sort which is not stable, so we
# had to find a way to make it look stable ;-)
self.queue.put(((-task.priority, time.time()), task))
self.total += 1
self.callback()
def taskDone(self, taskId):
log.debug("TaskManager task done")
with self.lock:
self.finished.append(taskId)
log.info("Task %d/%d completed!" % (len(self.finished), self.total))
# if we finished all the tasks, reset the current total
if self.queue.empty():
self.finished = []
self.total = 0
self.queue.task_done()
self.callback()
def finishNow(self):
log.info("TaskManager should finish ASAP, waiting for currently running tasks to finish")
self.shouldFinish = True
with self.lock:
if self.total == 0:
# worker thread is already waiting on an empty queue, we can't wait for him
log.info("No currently running jobs")
return
# FIXME: need to stop worker, we can't always wait for him
self.workerThread.join()
log.info("TaskManager: last running task finished")
class SessionPool:
""" A simple thread pool implementation for parsing log files of sessions.
A job is added by adding a session.
Jobs are ordered by the session_id(priority), so new sessions have a higher
priority.
The pool allocates and starts threads on demand. It allocates no more then
CPU_COUNT - 2 threads, but at least 1.
"""
__metaclass__ = Singleton
def __init__(self):
self.sessions = PriorityQueue()
self.pool = []
self.__start_jobs = False
self.MAX_THREADS = multiprocessing.cpu_count() - 2
if self.MAX_THREADS <= 0:
self.MAX_THREADS = 1
def addSession(self, session):
""" Add a session to to queue and start a thread.
"""
self.sessions.put(session)
if self.__start_jobs:
self.__startJob()
def activate(self, emptyJob):
""" Activates this pool by starting threads.
"""
self.__start_jobs = True
self.emptyJob = emptyJob
qs = self.sessions.qsize()
starts = min(qs, self.MAX_THREADS)
for i in range(0, starts):
self.__startJob()
# private methods
def __startJob(self):
""" Remove inactive threads from the pool and create a new for the new
job.
"""
if self.__start_jobs is False:
return
# clean thread pool
jc = len(self.pool)
for ri in range(jc, 0, -1):
i = ri - 1
thread = self.pool[i]
if thread is not None:
if thread.is_alive() is False:
self.pool.pop(i)
# create a new thread
jc = len(self.pool)
if jc < self.MAX_THREADS:
t = Thread(target=self.__executeJob)
t.start()
self.pool.append(t)
def __executeJob(self):
""" The target method of a thread.
Poll for sessions and execute their parser.
"""
try:
while True:
session = self.sessions.get(True, 1)
if session:
session.parseOldLogs()
parser = session.getParser()
parser.parseLog()
self.sessions.task_done()
else:
break
except Empty:
if self.emptyJob:
self.emptyJob()
return
except Exception as e:
print(str(e))
class RTS(object):
def __init__(self):
self.queue = PriorityQueue()
self.threads = []
for i in range(1):
t = Thread(target=self.worker)
t.start()
self.threads.append(t)
self.processed_events = 0
self.dropped_events = 0
self.log = file("gantt.dat", "w")
self.log2 = file("wait.dat", "w")
self.log3 = file("work_time.dat", "w")
self.log4 = file("processed_items.dat", "w")
self.log5 = file("dropped_items.dat", "w")
self.start = time.time()
def worker(self):
while True:
item = self.queue.get()
priority, time_sch, function, args, callback, deadline, fail_callback, fail_policy, time_policy = item
time_q = time.time()
thread = Thread(target=function, args=(time_q, ) + args)
thread.start()
# res = function(time_q, *args, **kwargs)
thread.join(deadline)
if thread.isAlive():
# overtime
thread._Thread__stop()
print("KILLED")
if fail_policy == "callback":
if fail_callback:
fail_callback(None)
elif fail_policy == "reschedule":
print("rescheduled")
self.schedule(*item[1:])
self.dropped_events += 1
self.log.write("{name} {time_s} {time_e} fail\n".format(name=function.func_name,
time_s=time_q - self.start,
time_e=time.time() - self.start))
self.log5.write("{name} {ctime} 1\n".format(name=function.func_name,
ctime=time.time() - self.start))
else:
time_end = time.time() - self.start
self.log.write("{name} {time_s} {time_e} ok\n".format(name=function.func_name,
time_s=time_q - self.start,
time_e=time_end))
if time_policy == "constant":
sleep_time = max((time_q + deadline) - time.time(), 0)
print("sleeping additionally %s for constant interval" % sleep_time)
time.sleep(sleep_time)
self.log.write("{name} {time_s} {time_e} placeholder\n".format(name=function.func_name,
time_s=time_end,
time_e=time_end + sleep_time))
if callback:
callback(None)
self.log3.write("{name} {ctime} {time}\n".format(name=function.func_name,
ctime=time.time() - self.start, time=time_end + self.start - time_q))
self.log4.write("{name} {ctime} 1\n".format(name=function.func_name,
ctime=time.time() - self.start))
self.log2.write("{name} {ctime} {time}\n".format(name=function.func_name,
ctime=time.time() - self.start, time=time_q - time_sch))
self.log.flush()
self.log2.flush()
self.log3.flush()
self.log4.flush()
self.log5.flush()
self.processed_events += 1
self.queue.task_done()
print("processed, dropped: ", self.processed_events, self.dropped_events)
def schedule(self, function, args, callback=None, deadline=None,
fail_callback=None, fail_policy="callback", time_policy="free"):
priority = time.time() + deadline if deadline else time.time() + 1000
self.queue.put((priority, time.time(), function, args, callback, deadline, fail_callback, fail_policy, time_policy))
class RSock:
def __init__ (self, sock, addr, cwnd, ploss, pcorr):
self.init = False
self.end = False
self.requesting = False
self.sock = sock
self.addr = addr
self.ploss = ploss
self.pcorr = pcorr
self.cwnd = cwnd
if self.cwnd <= 0:
self.cwnd = WINDOW_SIZE
self.attempt = 0
self.timer = None
self.lock = threading.Lock()
# PQueue will sort packets by segnum, so when inserting in queue, packets will be like:
# Acks - segnum = 0
# Failed packets (for their segnum is securely < unset packets)
# Unsent packets
# So ordering is guaranteed and ACKs should be sent immediately
self.buff = PQueue() # packets to send
self.waiting = PQueue(cwnd) # packets wating for ACK
self.seg = 0 # current packet
self.ack = 0 # last valid ack
self.nextSeg = random.randint(1,1000) # next expected packet
def start(self):
while not self.end:
packet = self.buff.get(True) # block until another packet is added
if self.timer == None:
self.playTimer()
# only regular packets should be recent.
# acks require the other side of the socket to be sent again, so we don't need to put on waiting queue
if packet.ack == 0 and (self.requesting or packet.con == 0):
self.lock.acquire(True) # wait for timeout function if needed
self.lock.release() # release it BEFORE put function or it will deadlock
self.waiting.put(packet) # block after timer!
print "Sending seg " + str(packet.seg) + " to " + str(self.addr) # don't log ack/con for they are logged somewhere else
# inform Queue we're done with the packet we got
self.buff.task_done()
if random.randint(1, 100) < self.ploss:
print "Simulating delayed packet"
continue # simulate lost packet
self.sock.sendto(packet.wrap(), self.addr)
if packet.end:
print "Sending connection end"
if (packet.end and packet.ack) or self.attempt >= ATTEMPT: # should this avoid dced socket to be maintained
self.endCon()
def endCon(self):
if self.attempt >= ATTEMPT:
print "Forcing close on socket due to response lack"
else:
print "Ending connection to " + str(self.addr)
self.end = True
def playTimer(self):
if self.timer != None:
self.timer.cancel() # stop current timer
self.timer = threading.Timer(ACK_TIMEOUT, self.timeout)
self.timer.start()
def timeout(self):
if not self.end and not self.waiting.empty():
print "Timed out! Enqueuing window again..."
self.lock.acquire(True) # block other thread
self.attempt += 1 # should this avoid dced socket to be maintained
# expected ack didn't arrive... send window again!
while not self.waiting.empty():
packet = self.waiting.get()
if packet.seg >= self.ack: # no need to resend acked packets...
self.buff.put(packet)
self.waiting.task_done()
self.lock.release()
if not self.end:
self.playTimer()
def enqueuePacket(self, data):
packet = Packet(data)
if self.init:
packet.seg = self.seg
self.seg += 1
else:
self.requesting = True
packet.con = self.nextSeg # con packet: seg = 0; ack = 0; con = nextSeg
self.buff.put(packet)
def endPacket(self):
packet = Packet('', self.seg, end=1)
self.seg += 1
self.buff.put(packet)
def errPacket(self):
packet = Packet('', self.seg, err=1)
self.seg += 1
self.buff.put(packet)
def ackPacket(self, ack, endAck):
self.buff.put(Packet('', 0, ack, endAck))
def receivePacket(self, data):
packet = Packet(data)
packet.unwrap()
self.attempts = 0
if not packet.validChecksum() or random.randint(1, 100) < self.pcorr: # do not receive broken packets or simulate a broken packet
print "Bad checksum received on seg: " + str(packet.seg) + " ack: " + str(packet.ack)
return None
if packet.con > 0:
self.seg = packet.con # set seg with nextSeg received in con field
self.ack = packet.con
self.init = True
if not self.requesting: # this socket received the connection request, then ack it
print "Connection request from " + str(self.addr)
self.buff.put(Packet('', end = packet.end, con = self.nextSeg)) # ack with nextSeg expected on con field
return packet
else:
print "Connection established!"
elif (packet.ack > 0): # ack packet
self.lock.acquire(True)
if packet.ack == self.ack: # expected ack!
self.ack += 1
self.playTimer() # ack received, start timer again
print "Received expected ack " + str(packet.ack) + " on connection " + str(self.addr)
if packet.end: # ack for end packet received
self.endCon()
self.lock.release()
elif packet.seg < self.nextSeg:
self.ackPacket(packet.seg, packet.end) # old packet received again, ACK might be lost.. send it again
print "Duplicated packet " + str(packet.seg) + ". Sending ack again and ignoring"
elif self.nextSeg == packet.seg: # expected seg!
self.nextSeg += 1
self.ackPacket(packet.seg, packet.end)
print "Acking packet " + str(packet.seg)
return packet
#else:
# print "Bad packet received! Seg: " + str(packet.seg) + " expected: " + str(self.nextSeg) + " ack: " + str(packet.ack)
return None
class scanmatcher_odom(object):
def __init__(self):
self._scan_msg = LaserScan()
self._odom_msg = Odometry()
# parameters
self._send_odom_base_tf = rospy.get_param('~send_odom_base_tf', default=True)
# publishers
self._odom_pub = rospy.Publisher('/odom', Odometry, queue_size=10)
# tf broadcasters
self._odom_base_link_tfb = tf.TransformBroadcaster()
# tf listeners
self._base_link_base_laser_tfl = tf.TransformListener()
# user init
# processing queue
self._scan_queue = PriorityQueue(maxsize=2)
# absolute odometry pose and current velocities
self._current_pose = pymrpt.poses.CPose2D()
self._v = .0
self._w = .0
# initialize odometry
self._odom_msg.header.frame_id = 'odom'
self._odom_msg.child_frame_id = 'base_link'
# end of user init
# subscribers
rospy.Subscriber('/scan', LaserScan, self._scan_callback, queue_size=10)
def get_base_link_base_laser_tf(self, stamp=None):
pose = PoseStamped()
if stamp is None:
stamp = rospy.Time(0)
try:
trans, rot = self._base_link_base_laser_tfl.lookupTransform('/base_link', '/base_laser', stamp)
except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
# TODO: implement TransformListener.lookupTransform() exception handling!
rospy.logwarn('Error while listening to /base_link -> /base_laser tf')
return pose, False
pose.pose.position.x = trans[0]
pose.pose.position.y = trans[1]
pose.pose.position.z = trans[2]
pose.pose.orientation.x = rot[0]
pose.pose.orientation.y = rot[1]
pose.pose.orientation.z = rot[2]
pose.pose.orientation.w = rot[3]
return pose, True
def send_odom_base_link_tf(self, pose):
translation = pose.pose.position
rotation = pose.pose.orientation
self._odom_base_link_tfb.sendTransform(
(translation.x, translation.y, translation.z),
(rotation.x, rotation.y, rotation.z, rotation.w),
pose.header.stamp,
'/base_link',
'/odom')
def run(self):
# create worker thread (we do not .join() since it closes itself by ros shutdown)
scanmatcher_thread = Thread(target=self.scanmatcher_worker)
scanmatcher_thread.start()
# keep it running
rospy.spin()
def _scan_callback(self, _scan_msg):
# put scans into queue
try:
self._scan_queue.put_nowait([_scan_msg.header.stamp.to_nsec(), _scan_msg])
except Full:
rospy.logwarn('Dropped scan. Processing queue is full!')
pass
def scanmatcher_worker(self):
# vars
has_initial_scan = False
# initialize ICP
icp_options = pymrpt.slam.CICP.TConfigParams()
icp = pymrpt.slam.CICP(icp_options)
# create last and current laser maps
last_map = pymrpt.maps.CSimplePointsMap()
# setup last update time
last_update_time = rospy.Time.now()
# loop
while not rospy.is_shutdown():
self._scan_msg = LaserScan()
# get scan from queue
try:
item = self._scan_queue.get(timeout=5)
except Empty:
rospy.logwarn('Got no scan from queue since 5 seconds. Scanmatcher will shutdown now!')
continue
self._scan_msg = item[1]
self._scan_msg.ranges = list(self._scan_msg.ranges)
rospy.loginfo('received scan...')
# update current stamp for publishers
self._current_stamp = self._scan_msg.header.stamp
# get laser scanner pose
laser_pose, ok = self.get_base_link_base_laser_tf()
self._sensor_pose = pymrpt.poses.CPose3D()
self._sensor_pose.from_ROS_Pose_msg(laser_pose.pose)
# convert data
observation = pymrpt.obs.CObservation2DRangeScan()
observation.from_ROS_LaserScan_msg(self._scan_msg, self._sensor_pose)
# set current map from scan
current_map = pymrpt.maps.CSimplePointsMap()
current_map.loadFromRangeScan(observation)
# match maps
if has_initial_scan:
# no initial guess (pure incremental)
initial_guess = pymrpt.poses.CPosePDFGaussian()
# run ICP algorithm
pose_change, running_time, info = icp.AlignPDF(last_map, current_map, initial_guess)
rospy.loginfo('icp goodness: {}'.format(info.goodness))
rospy.loginfo('icp run_time: {}'.format(running_time))
rospy.loginfo('pose change: {}'.format(pose_change.mean))
# check goodness
# if info.goodness > .8:
rospy.loginfo('...updating odometry.')
# get time delta
d_t = (last_update_time - self._current_stamp).to_sec()
# update current pose and velocities
self._current_pose += pose_change.mean
dist = pose_change.mean.norm()
self._v = dist / d_t
self._w = pose_change.mean.phi / d_t
self.publish_odom()
# update last update time
last_update_time = self._current_stamp
# update last map
last_map = current_map
# else:
# rospy.logwarn('...lost odometry...')
else:
rospy.loginfo('...is inital one!')
# load initial scan to last map
last_map = current_map
# mark initial as received
has_initial_scan = True
# rospy.loginfo('...task done!')
# signalize work done
self._scan_queue.task_done()
def publish_odom(self):
"""
Publish odometry and optionally "odom -> base_link" tf.
"""
# setup odometry message
self._odom_msg.header.stamp = self._current_stamp
self._odom_msg.pose.pose = self._current_pose.to_ROS_Pose_msg()
self._odom_msg.twist.twist.linear.x = self._v
self._odom_msg.twist.twist.angular.z = self._w
# publish odom
self._odom_pub.publish(self._odom_msg)
# publish "odom -> base_link" tf on demand
if self._send_odom_base_tf:
pose_msg = PoseStamped()
pose_msg.header.stamp = self._current_stamp
pose_msg.pose = self._odom_msg.pose.pose
self.send_odom_base_link_tf(pose_msg)
class TaskManager(object):
"""The TaskManager is a stable priority queue of tasks. It takes them one by one, the
one with the highest priority first, and call its perform() method, then repeat until no tasks are left.
If two or more tasks have the same priority, it will take the one that was added first to the queue.
The TaskManager can be controlled asynchronously, as it runs the tasks in a separate thread."""
def __init__(self):
super(TaskManager, self).__init__()
# our main task queue
self.queue = PriorityQueue()
self.taskId = 0 # ID for the next task that will be generated
self.total = 0 # used to keep track of the total jobs that have been submitted (queue size decreases as we process tasks)
self.finished = [] # list of task IDs which have finished
self.taskDesc = '' # description of the task started last
self.lock = Lock()
log.debug('Main GUI thread is: 0x%x' % current_thread().ident)
self.shouldFinish = False
# ideally a pool of threads or sth like TBB or ThreadWeaver
self.workerThread = Thread(target = worker, args = (self,))
self.workerThread.daemon = True
self.workerThread.start()
def add(self, task):
log.info('TaskManager add task: %s' % task.description)
with self.lock:
# -task.priority because it always gets the lowest one first
# we need to put the time as well, because Queue uses heap sort which is not stable, so we
# had to find a way to make it look stable ;-)
self.queue.put(( (-task.priority, time.time()), task ))
self.total += 1
def taskDone(self, taskId):
with self.lock:
self.finished.append(taskId)
log.info('Task %d/%d completed!' % (len(self.finished), self.total))
# if we finished all the tasks, reset the current total
if self.queue.empty():
self.finished = []
self.total = 0
self.queue.task_done()
def finishNow(self):
log.info('TaskManager should finish ASAP, waiting for currently running tasks to finish')
self.shouldFinish = True
with self.lock:
if self.total == 0:
# worker thread is already waiting on an empty queue, we can't wait for him
log.info('No currently running jobs')
return
# FIXME: need to stop worker, we can't always wait for it
self.workerThread.join()
log.info('TaskManager: last running task finished')
class TaskManager(object):
"""The TaskManager is a stable priority queue of tasks. It takes them one by one, the
one with the highest priority first, and call its perform() method, then repeat until no tasks are left.
If two or more tasks have the same priority, it will take the one that was added first to the queue.
The TaskManager can be controlled asynchronously, as it runs the tasks in a separate thread."""
def __init__(self):
super(TaskManager, self).__init__()
# our main task queue
self.queue = PriorityQueue()
self.taskId = 0 # ID for the next task that will be generated
self.total = 0 # used to keep track of the total jobs that have been submitted (queue size decreases as we process tasks)
self.finished = [] # list of task IDs which have finished
self.taskDesc = '' # description of the task started last
self.lock = Lock()
log.debug('Main GUI thread is: 0x%x' % current_thread().ident)
self.shouldFinish = False
# ideally a pool of threads or sth like TBB or ThreadWeaver
self.workerThread = Thread(target=worker, args=(self, ))
self.workerThread.daemon = True
self.workerThread.start()
def add(self, task):
log.info('TaskManager add task: %s' % task.description)
with self.lock:
# -task.priority because it always gets the lowest one first
# we need to put the time as well, because Queue uses heap sort which is not stable, so we
# had to find a way to make it look stable ;-)
self.queue.put(((-task.priority, time.time()), task))
self.total += 1
def taskDone(self, taskId):
with self.lock:
self.finished.append(taskId)
log.info('Task %d/%d completed!' %
(len(self.finished), self.total))
# if we finished all the tasks, reset the current total
if self.queue.empty():
self.finished = []
self.total = 0
self.queue.task_done()
def finishNow(self):
log.info(
'TaskManager should finish ASAP, waiting for currently running tasks to finish'
)
self.shouldFinish = True
with self.lock:
if self.total == 0:
# worker thread is already waiting on an empty queue, we can't wait for him
log.info('No currently running jobs')
return
# FIXME: need to stop worker, we can't always wait for it
self.workerThread.join()
log.info('TaskManager: last running task finished')
class Pool:
"""Represents a thread pool"""
def __init__(self, workers = max_workers, rate_limit = 1000):
self.max_workers = workers
self.mutex = Semaphore()
self.results = {}
self.retries = defaultdict(int)
self.queue = PriorityQueue()
self.threads = []
self.rate_limit = rate_limit
def _tick(self):
time.sleep(1.0/self.rate_limit)
# clean up finished threads
self.threads = [t for t in self.threads if t.isAlive()]
return (not self.queue.empty()) or (len(self.threads) > 0)
def _loop(self):
"""Handle task submissions"""
def run_task(priority, f, uuid, retries, args, kwargs):
"""Run a single task"""
try:
t.name = getattr(f, '__name__', None)
result = f(*args, **kwargs)
except Exception as e:
# Retry the task if applicable
if log:
log.error(traceback.format_exc())
if retries > 0:
with self.mutex:
self.retries[uuid] += 1
# re-queue the task with a lower (i.e., higher-valued) priority
self.queue.put((priority+1, dumps((f, uuid, retries - 1, args, kwargs))))
self.queue.task_done()
return
result = e
with self.mutex:
self.results[uuid] = dumps(result)
self.retries[uuid] += 1
self.queue.task_done()
while self._tick():
# spawn more threads to fill free slots
log.warn("Running %d/%d threads" % (len(self.threads),self.max_workers))
if len(self.threads) < self.max_workers:
log.debug("Queue Length: %d" % self.queue.qsize())
try:
priority, data = self.queue.get(True, 1.0/self.rate_limit)
except Empty:
continue
f, uuid, retries, args, kwargs = loads(data)
t = Thread(target=run_task, args=[priority, f, uuid, retries, args, kwargs])
t.setDaemon(True)
self.threads.append(t)
t.start()
log.debug("Exited loop.")
for t in self.threads:
t.join()
def stop(self):
"""Flush the job queue"""
self.queue = PriorityQueue()
def start(self, daemonize=False):
"""Pool entry point"""
self.results = {}
self.retries = defaultdict(int)
if daemonize:
t = Thread(target = self._loop, args=[self])
t.setDaemon(True)
t.start()
return
else:
self._loop()
def main(argv=None):
if argv is None:
argv = sys.argv
id_base = int(sys.argv[1])
manager = Manager()
wait_q = PriorityQueue()
out_file = open("latency.csv", "a")
load_thread = Thread(target=check_load)
load_thread.start()
logger.info("PID: %d" % os.getpid())
logger.info("Running the gauntlet:")
id = id_base
num_clients = 0
for i in range(2):
wait_q.put((time.time(), False, id))
id += 1
num_clients += 1
for i in range(2):
wait_q.put((time.time(), True, id))
id += 1
num_clients += 1
pool = Pool(processes=num_clients)
while True:
t, attacker, id = wait_q.get()
delta = t - time.time()
delay = max(delta, 0)
if delay > 0:
Timer(delay, pool.apply_async, (fetch, (attacker, id))).start()
else:
pool.apply_async(fetch, (attacker, id))
wait_q.task_done()
while not wait_q_q.empty():
wait_q.put(wait_q_q.get())
# we have to make sure there is at least one element in wait_q before the end of the loop
if wait_q.empty():
r = wait_q_q.get()
wait_q.put(r)
while not res_q.empty():
id, n, t, error = res_q.get()
logger.info("id: %d got a response for n = %d in %f error: %d" % (id, n, t, error))
out_file.write("%d, %d, %f, %d\n" % (id, n, t, error))
# if time.time() - start_time > 120:
# break
out_file.flush()
while not res_q.empty():
res = res_q.get()
out_file.write("%d, %f, %d\n" % (res[0], res[1], res[2]))
return 0
l3 = ListNode(2)
l3.next = ListNode(6)
lists = [l1, l2, l3]
# foreachListNode(mergeKLists1(lists))
foreachListNode(mergeKLists2(lists))
def HeapSort(list):
# 将 list 构建成堆
heapq.heapify(list)
heap = []
while list:
heap.append(heapq.heappop(lists))
list[:] = heap
return list
from Queue import PriorityQueue
q = PriorityQueue()
q.put(1)
q.put(5)
q.put(4)
q.put(3)
print q.task_done()
class Pool:
"""Represents a thread pool"""
def __init__(self, workers = max_workers, rate_limit = 1000):
self.max_workers = workers
self.mutex = Semaphore()
self.results = {}
self.retries = defaultdict(int)
self.queue = PriorityQueue()
self.threads = []
self.rate_limit = rate_limit
def _tick(self):
time.sleep(1.0/self.rate_limit)
# clean up finished threads
self.threads = [t for t in self.threads if t.isAlive()]
return (not self.queue.empty()) or (len(self.threads) > 0)
def _loop(self):
"""Handle task submissions"""
def run_task(priority, f, uuid, retries, args, kwargs):
"""Run a single task"""
try:
t.name = getattr(f, '__name__', None)
result = f(*args, **kwargs)
except Exception as e:
# Retry the task if applicable
if log:
log.error(traceback.format_exc())
if retries > 0:
with self.mutex:
self.retries[uuid] += 1
# re-queue the task with a lower (i.e., higher-valued) priority
self.queue.put((priority+1, dumps((f, uuid, retries - 1, args, kwargs))))
self.queue.task_done()
return
result = e
with self.mutex:
self.results[uuid] = dumps(result)
self.retries[uuid] += 1
self.queue.task_done()
while self._tick():
# spawn more threads to fill free slots
log.warn("Running %d/%d threads" % (len(self.threads),self.max_workers))
if len(self.threads) < self.max_workers:
log.debug("Queue Length: %d" % self.queue.qsize())
try:
priority, data = self.queue.get(True, 1.0/self.rate_limit)
except Empty:
continue
f, uuid, retries, args, kwargs = loads(data)
t = Thread(target=run_task, args=[priority, f, uuid, retries, args, kwargs])
t.setDaemon(True)
self.threads.append(t)
t.start()
log.debug("Exited loop.")
for t in self.threads:
t.join()
def stop(self):
"""Flush the job queue"""
self.queue = PriorityQueue()
def start(self, daemonize=False):
"""Pool entry point"""
self.results = {}
self.retries = defaultdict(int)
if daemonize:
t = Thread(target = self._loop, args=[self])
t.setDaemon(True)
t.start()
return
else:
self._loop()
class qController(object):
def __init__(self, event_handler, timer_interval):
self.handler = event_handler
self.timer_interval = timer_interval
self.TickTock = PriorityQueue()
self.lock = threading.Lock()
self.bListening = False
return
def stop(self):
if self.bListening:
q = qData('stop')
self.TickTock.put((qPriorityHigh, q))
return
def event(self, event_type, data=None):
if self.bListening:
q = qData(event_type, data)
self.TickTock.put((qPriorityNormal, q))
return
def tick(self):
time.sleep(self.timer_interval)
if self.bListening:
q = qData('tick')
self.TickTock.put((qPriorityLow, q))
return
def tick_start(self):
t = threading.Thread(
target=qController_tick,
args=(self, )) # add a new delayed event to the queue
t.start()
return
def run(self):
if not self.lock.acquire(False):
return False
# clear the queue, just in case of multiple calls to run()
while not self.TickTock.empty():
try:
self.TickTock.get(False)
except Empty:
continue
self.TickTock.task_done()
self.bListening = True
self.tick_start()
while True:
p, q = self.TickTock.get()
if q.event_type == 'stop':
break
if q.event_type == 'tick':
if self.handler.tock(q.event_data) == False:
break
self.tick_start()
elif self.handler.event(q.event_type, q.event_data) == False:
break
self.TickTock.task_done()
self.bListening = False
self.lock.release()
return True
class Testsuite:
verbose = False
def __init__(self, name, directory_scenarii,
clean_func, state_file, cmd_display_func):
self.name = name
self.list_scenario = []
self.selected_scenario = []
self.directory_scenarii = directory_scenarii
self.clean_system = clean_func
self.cmd_display_func = cmd_display_func
self.state_file = state_file
# save the current context to restaure it at the end of the testsuite
backends = [ line for line in process.execute(['get', 'config',
'backends'])[0].split('\n') if 'U' in line
and line != '' ]
self.user_context = backends[0].split('(')[0]
self.current_context = self.user_context
self.to_run = PriorityQueue()
self.failed = PriorityQueue()
self.passed = []
self.best_state = 1
self.working = Event()
# used to modify the best state behaviour when running one one test
# or the whole TS.
self.best_state_only_one = 0
# used to build initial testsuite data
self.batch_run = False
self.interactive = False
def restore_user_context(self):
""" Restore user active backend before testsuite runs. """
if self.user_context == 'shadow' and self.current_context != 'shadow':
process.execute([ 'mod', 'config', '-B', 'openldap'])
if self.user_context == 'openldap' \
and self.current_context != 'openldap':
process.execute([ 'mod', 'config', '-b', 'openldap'])
test_message(_(u'Restored initial backend to %s.') % self.user_context)
def add_scenario(self,scenario):
""" add a scenario to the testsuite and set a number for it """
scenario.set_number(len(self.list_scenario) + 1)
# link us to the scenario
scenario.testsuite = self
self.list_scenario.append(scenario)
def run(self):
""" Run selected scenarii. """
self.total_sce = len(self.selected_scenario)
# we will start display 'context setup' if this is the case.
self.last_setup_display = -1
self.last_status_display = 0
if self.interactive:
self.run_interactive()
else:
self.run_threaded()
def run_interactive(self):
for sce in self.selected_scenario:
sce.full_interactive = True
while sce.status in (sce_status.FAILED, sce_status.NOT_STARTED):
try:
sce.Run(interactive=True)
except KeyboardInterrupt:
test_message(_(u"Keyboard Interrupt received; "
u"cleaning scenario, please wait…"))
sce.clean()
raise
self.passed.append(sce.counter)
self.save_best_state()
def run_threaded(self):
# As contexts are shared with one only licorn, we cannot run more
# than 1 parallel job for now. But this still allows to continue
# to run jobs while the tester inspects job results, and this is
# still important to save time on success jobs which don't wait.
self.workers_scheduler = TestsuiteRunnerThread.setup(licornd=None,
input_queue=self.to_run,
peers_min=1, peers_max=1, daemon=True)
""" Feed the queue with things to do. """
if self.selected_scenario != []:
# clean the system from previus run
self.clean_system()
for scenario in self.selected_scenario:
# save the state of the testsuite (number of current scenario)
# in order to be abble to restart from here if the TS stopped.
self.to_run.put((scenario.counter, self.run_scenario,
(scenario,), {}))
# reset selection
self.selected_scenario = []
test_message(_(u'Started background tests, '
u'waiting for any failed scenario…'))
# wait for the first job to be run, else counter displays '-1'.
time.sleep(2.0)
while True:
try:
self.display_status()
#time.sleep(1.0)
prio, sce = self.failed.get(True, 1.0)
if not self.best_state_only_one \
and sce.counter > self.best_state:
# we delay the checks for failed jobs which are not
# the next to check. This permits the user to check
# scenario to the end, before checking a new one, at
# the expense of a little more loop cycles.
#print '>> downgrading sce %s %s' % (sce.counter, sce.name)
self.failed.task_done()
self.failed.put((sce.counter, sce))
# if we got another job, our current failed one is still
# not finished, wait a little.
time.sleep(1.0)
continue
#print '>> run sce %s %s ' % (sce.counter, sce.name)
self.display_status()
sce.Run(interactive=True)
self.failed.task_done()
# if it failed again, reput it in the queue (until it doesn't
# fail anymore).
if sce.status in (sce_status.FAILED, sce_status.NOT_STARTED):
self.to_run.put(
(sce.counter, self.run_scenario, (sce,), {}))
test_message(_(u'Re-put scenario in the run queue '
u'for further execution.'))
# wait a short while, to avoid displaying next status
# working on the end of the last failed scenario, which
# is a false-positive.
#time.sleep(1.0)
except KeyboardInterrupt:
#best state is already saved.
test_message(_(u'Stopping the run queue worker '
u'and cleaning, please wait…'))
self.workers_scheduler.stop()
self.display_status()
#self.threads[0].join()
raise
except Empty:
#print '>> empty'
if self.to_run.qsize() == 0 and self.failed.qsize() == 0 \
and not self.working.is_set():
self.workers_scheduler.stop()
self.save_best_state()
logging.notice(_(u'no more jobs, ending testsuite.'))
break
self.workers_scheduler.stop()
def save_best_state(self):
""" find the highest number of a consecutive suite of non failed ones,
and save it as the best state we could reach in the current session.
"""
old_best = self.best_state
for elem in sorted(self.passed):
if elem < self.best_state:
self.passed.remove(elem)
elif elem == self.best_state:
self.passed.remove(elem)
self.best_state = elem + 1
elif self.best_state_only_one and elem == self.best_state + 1:
self.passed.remove(elem)
self.best_state = elem
else:
break
if self.best_state != old_best:
# note we got at least until there, save it.
self.save_state(self.best_state)
test_message(_(u'Saved best state #%s (not inclusive).') %
self.best_state)
def run_scenario(self, scenario):
self.current_sce = scenario
self.working.set()
scenario.batch_run = self.batch_run
scenario.Run()
if scenario.status == sce_status.PASSED:
#print '>> got one passed'
self.passed.append(scenario.counter)
self.save_best_state()
elif scenario.status == sce_status.FAILED:
#print '>> got one failed -> put(failed)'
self.failed.put((scenario.counter, scenario))
elif scenario.status == sce_status.NOT_STARTED:
#print '>> got not started -> put(re_run)'
# after a corrected fail, the scenario must be rerun.
# with high priority, because we are probably waiting for it.
self.to_run.put(
(scenario.counter, self.run_scenario, (scenario,), {}))
self.current_sce = None
self.working.clear()
def display_status(self):
""" display the current status of the TS, in the following conditions:
* if not working on anything, don't display any status (this
happens in some rare cases when jobs are migrated from one
queue to another (which is blocking, or while cleanings occur).
* if no command is running, display the fact that the scenario
is setting up its context.
This can take time because we need to wait for the daemon to
restart. In this case, the status will be displayed only ONCE
for a given scenario.
It may not be displayed at all, if context is fast enough to
check and setup (or not set up at all when not needed).
* while running scenario commands, always display the status.
"""
# "1" counts for the job currently under test: it is not yet done.
if self.working.is_set():
if time.time() - self.last_status_display < 1.0:
# don't display the status more than once per second,
# in any case.
return
self.last_status_display = time.time()
if self.current_sce.current_cmd != None \
or self.last_setup_display != self.current_sce.counter:
ran_sce = self.total_sce - (
(1 if self.working.is_set() else 0)
+ self.to_run.qsize()
+ self.failed.qsize())
failed_sce = self.failed.qsize()
logging.notice(_(u'TS global status: on #{current}, '
'{ran}/{total} passed '
'({percent_done:.2}%) {passed_content}- '
'{failed} failed '
'({percent_failed:.2}%) - '
'best score until now: #{best}').format(
current=stylize(ST_BUSY, '%s (%s)' % (
self.current_sce.counter,
'%s/%s %.1f%%' % (
self.current_sce.current_cmd,
self.current_sce.total_cmds,
100.0
* self.current_sce.current_cmd
/ self.current_sce.total_cmds,
)
if self.current_sce.current_cmd
else _(u'context setup')
)
if self.current_sce
else '-'),
ran=ran_sce,
total=self.total_sce,
percent_done=100.0*ran_sce/self.total_sce,
failed=failed_sce,
percent_failed=100.0*failed_sce/self.total_sce,
best=self.best_state,
passed_content=_(u'last: {passed}{passed_more} ').format(
passed=','.join([ str(i)
for i in sorted(self.passed)[:5] ]),
passed_more=_(u'…')
if len(self.passed) > 5 else '')
if len(self.passed) > 0 else ''
)
)
# always mark the context setup displayed if we reach here.
# either we displayed it, or we jumped directly to a command
# display, which implies the context setup is already done.
#
# taking in account the second reason avoids displaying
# 'context setup' while the scenario is cleaning after a fail.
self.last_setup_display = self.current_sce.counter
def get_scenarii(self):
""" display the list of scenarii in the TS """
# display stats
self.get_stats()
logging.notice('''List of scenarii in %s TS: (%s=no trace;%s=not '''
'''completely tested;%s=completety tested)''' % (self.name,
stylize(ST_BAD,'*'), stylize(ST_NAME,'*'), stylize(ST_OK,'*')))
for scenario in self.list_scenario:
try:
nbr_cmd_in_dir = len(os.listdir('%s/%s' %
(self.directory_scenarii, scenario.hash)))
except OSError:
nbr_cmd_in_dir = 0
nbr_cmd = len(scenario.cmds)
if nbr_cmd_in_dir < nbr_cmd: color=ST_NAME
if nbr_cmd_in_dir == 0: color=ST_BAD
if nbr_cmd_in_dir == nbr_cmd: color=ST_OK
logging.notice('%s: %s %s' % (
stylize(color, '%3d' % scenario.counter),
scenario.descr,
'[%s]' % stylize(ST_LINK,scenario.context)
if self.name=='CLI' else ''))
if Testsuite.verbose:
for cmd in scenario.cmds:
logging.notice("--> cmd %2d: %s" % (cmd,
self.cmd_display_func(scenario.cmds[cmd])))
def get_stats(self):
""" display some statistics of the TS (number of scenario, number
of commands) """
sce_ = len(self.list_scenario)
cmd_ = sum(len(sce.cmds) for sce in self.list_scenario)
logging.notice(_(u'The {0} testsuite holds {1} scenarii, counting {2} '
'commands (avg of {3} cmds per scenario).').format(
stylize(ST_NAME, self.name), stylize(ST_OK, sce_),
stylize(ST_OK, cmd_), stylize(ST_UGID, '%.1f' % (cmd_*1.0/sce_))))
def select(self, scenario_number=None, all=False, mode=None):
""" select some scenarii to be executed """
if all:
# select all scenarii
self.selected_scenario = self.list_scenario[:]
self.best_state = 1
elif scenario_number != None and mode == None:
try:
# select only one scenario
self.selected_scenario.append(
self.list_scenario[scenario_number])
self.best_state = self.get_state() or 1
self.best_state_only_one = scenario_number
except IndexError, e:
test_message(_(u"No scenario selected"))
elif scenario_number != None and mode == 'start':
# start selection from a scenario to the end of the list
for scenario in self.list_scenario[scenario_number-1:]:
self.selected_scenario.append(scenario)
self.best_state = self.get_state() or 1
if self.best_state < scenario_number:
raise RuntimeError(_('Cannot select a higher scenario '
'than the current best_state (%s)' % self.best_state))
if self.selected_scenario == []:
test_message(_(u"No scenario selected"))
def play_video(BASE, video_url = _common.args.url, media_base = VIDEOURL):
if media_base not in video_url:
video_url = media_base + video_url
try:
qbitrate = _common.args.quality
except:
qbitrate = None
video_url2 = 'stack://'
threads = []
closedcaption = []
exception = False
queue = PriorityQueue()
segments = []
if 'feed' not in video_url:
swf_url = _connection.getRedirect(video_url, header = {'Referer' : BASE})
try:
params = dict(item.split("=") for item in swf_url.split('?')[1].split("&"))
uri = urllib.unquote_plus(params['uri'])
config_url = urllib.unquote_plus(params['CONFIG_URL'].replace('Other', DEVICE))
config_data = _connection.getURL(config_url, header = {'Referer' : video_url, 'X-Forwarded-For' : '12.13.14.15'})
config_tree = BeautifulSoup(config_data, 'html.parser')
if not config_tree.error:
feed_url = config_tree.feed.string
feed_url = feed_url.replace('{uri}', uri).replace('&', '&').replace('{device}', DEVICE).replace('{ref}', 'None').strip()
else:
exception = True
error_text = config_tree.error.string.split('/')[-1].split('_')
_common.show_exception(error_text[1], error_text[2])
except:
_common.show_exception("Viacomm", swf_url)
else:
feed_url = video_url
if not exception:
feed_data = _connection.getURL(feed_url)
video_tree = BeautifulSoup(feed_data, 'html.parser', parse_only = SoupStrainer('media:group'))
video_segments = video_tree.find_all('media:content')
for i, video_item in enumerate(video_segments):
worker = Thread(target = get_videos, args = (queue, i, video_item, qbitrate))
worker.setDaemon(True)
worker.start()
threads.append(worker)
[x.join() for x in threads]
while not queue.empty():
video_data2 = queue.get()
video_url2 += video_data2[1] + ' , '
segments.append(video_data2[2])
closedcaption.append((video_data2[3], video_data2[2], video_data2[0]))
player._segments_array = segments
filestring = 'XBMC.RunScript(' + os.path.join(_common.LIBPATH,'_proxy.py') + ', 12345)'
xbmc.executebuiltin(filestring)
finalurl = video_url2[:-3]
localhttpserver = True
time.sleep(20)
if (_addoncompat.get_setting('enablesubtitles') == 'true') and closedcaption:
convert_subtitles(closedcaption)
player._subtitles_Enabled = True
item = xbmcgui.ListItem(path = finalurl)
queue.task_done()
if qbitrate is not None:
item.setThumbnailImage(_common.args.thumb)
item.setInfo('Video', { 'title' : _common.args.name,
'season' : _common.args.season_number,
'episode' : _common.args.episode_number,
'TVShowTitle' : _common.args.show_title })
xbmcplugin.setResolvedUrl(pluginHandle, True, item)
while player.is_active:
player.sleep(250)
def play_video(BASE, video_uri = common.args.url, media_base = VIDEOURL):
video_url = media_base + video_uri
try:
qbitrate = common.args.quality
except:
qbitrate = None
video_url2 = 'stack://'
closedcaption = []
exception = False
queue = PriorityQueue()
segments = []
if 'feed' in video_uri:
feed_url = video_uri
else:
swf_url = connection.getRedirect(video_url, header = {'Referer' : BASE})
params = dict(item.split("=") for item in swf_url.split('?')[1].split("&"))
uri = urllib.unquote_plus(params['uri'])
config_url = urllib.unquote_plus(params['CONFIG_URL'].replace('Other', DEVICE))
config_data = connection.getURL(config_url, header = {'Referer' : video_url, 'X-Forwarded-For' : '12.13.14.15'})
config_tree = BeautifulSoup(config_data, 'html.parser')
if not config_tree.error:
feed_url = config_tree.feed.string
uri = urllib.quote_plus(uri)
feed_url = feed_url.replace('{uri}', uri).replace('&', '&').replace('{device}', DEVICE).replace('{ref}', 'None').replace('{type}', 'network').strip()
else:
exception = True
error_text = config_tree.error.string.split('/')[-1].split('_')
if error_text[1] == 'loc':
params = dict(item.split("=") for item in config_url.split('?')[-1].split('&'))
common.show_exception('Geo', params['geo'])
if not exception:
feed_data = connection.getURL(feed_url, header = {'X-Forwarded-For' : '12.13.14.15'})
video_tree = BeautifulSoup(feed_data, 'html.parser', parse_only = SoupStrainer('media:group'))
video_segments = video_tree.find_all('media:content')
if not video_segments:
video_tree = BeautifulSoup(feed_data, 'html.parser')
common.show_exception(video_tree.find('meta', property = "og:site_name")['content'], video_tree.find('meta', property = "og:url")['content'])
exception = True
threads = []
for i, video_item in enumerate(video_segments):
try:
threads.append(Thread(get_videos, queue, i, video_item, qbitrate, False))
except Exception, e:
print "Exception: ", e
[i.start() for i in threads]
[i.join() for i in threads]
while not queue.empty():
video_data2 = queue.get()
video_url2 += video_data2[1] + ' , '
segments.append(video_data2[2])
closedcaption.append((video_data2[3], int(video_data2[0])))
player._segments_array = segments
finalurl = video_url2[:-3]
time.sleep(20)
if (addon.getSetting('enablesubtitles') == 'true') and closedcaption and detect_format is not None:
convert_subtitles(closedcaption)
player._subtitles_Enabled = True
item = xbmcgui.ListItem(path = finalurl)
if player._localHTTPServer:
filestring = 'XBMC.RunScript(' + os.path.join(ustvpaths.LIBPATH,'proxy.py') + ', 12345)'
xbmc.executebuiltin(filestring)
finalurl = video_url2[:-3]
#localhttpserver = True
time.sleep(20)
queue.task_done()
try:
item.setThumbnailImage(common.args.thumb)
except:
pass
try:
item.setInfo('Video', { 'title' : common.args.name,
'season' : common.args.season_number,
'episode' : common.args.episode_number,
'TVShowTitle' : common.args.show_title })
except:
pass
xbmcplugin.setResolvedUrl(pluginHandle, True, item)
while player.is_active:
player.sleep(250)
class PGoApi:
def __init__(self, signature_lib_path, hash_lib_path, hash_key):
self.set_logger()
self._signature_lib_path = signature_lib_path
self._hash_lib_path = hash_lib_path
self._work_queue = PriorityQueue()
self._auth_queue = PriorityQueue()
self._workers = []
self._api_endpoint = 'https://pgorelease.nianticlabs.com/plfe/rpc'
self._hashKey = hash_key
self.log.info('%s v%s - %s', __title__, __version__, __copyright__)
self.log.info('%s', __patchedBy__)
def create_workers(self, num_workers):
for i in xrange(num_workers):
worker = PGoApiWorker(self._signature_lib_path,
self._hash_lib_path, self._work_queue,
self._auth_queue, self._hashKey)
worker.daemon = True
worker.start()
self._workers.append(worker)
def resize_workers(self, num_workers):
workers_now = len(self._workers)
if workers_now < num_workers:
self.create_workers(num_workers - workers_now)
elif workers_now > num_workers:
for i in xrange(workers_now - num_workers):
worker = self._workers.pop()
worker.stop()
def set_accounts(self, accounts):
old_accounts = []
new_accounts = []
accounts_todo = {}
for account in accounts:
accounts_todo[account['username']] = accounts['password']
while not self._auth_queue.empty():
# Go through accounts in auth queue and only add those back
# that we still want to use
next_call, auth_provider = self._auth_queue.get()
if auth_provider.username in accounts_todo:
old_accounts.append((next_call, auth_provider))
del accounts_todo[auth_provider.username]
while old_accounts:
self._auth_queue.put(old_accounts.pop())
# Add new accounts
for username, password in accounts_todo.iteritems():
new_accounts.append({'username': username, 'password': password})
add_accounts(new_accounts)
def add_accounts(self, accounts):
for account in accounts:
username, password = account['username'], account['password']
if not isinstance(username, six.string_types) or not isinstance(
password, six.string_types):
raise AuthException(
"Username/password not correctly specified")
provider = account.get('provider', 'ptc')
if provider == 'ptc':
auth_provider = AuthPtc(username, password)
elif provider == 'google':
auth_provider = AuthGoogle(username, password)
else:
raise AuthException(
"Invalid authentication provider - only ptc/google available."
)
self._auth_queue.put((time.time(), auth_provider))
def set_logger(self, logger=None):
self.log = logger or logging.getLogger(__name__)
def get_api_endpoint(self):
return self._api_endpoint
def __getattr__(self, func):
def function(**kwargs):
name = func.upper()
position = kwargs.pop('position')
callback = kwargs.pop('callback')
priority = kwargs.pop('priority') if 'priority' in kwargs else 10.0
if kwargs:
method = {RequestType.Value(name): kwargs}
self.log.debug(
"Adding '%s' to RPC request including arguments", name)
self.log.debug("Arguments of '%s': \n\r%s", name, kwargs)
else:
method = RequestType.Value(name)
self.log.debug("Adding '%s' to RPC request", name)
self.call_method(method, position, callback, priority)
if func.upper() in RequestType.keys():
return function
else:
raise AttributeError
def call_method(self, method, position, callback, priority=10.0):
self._work_queue.put((priority, method, position, callback))
def empty_work_queue(self):
while not self._work_queue.empty():
try:
self._work_queue.get(False)
self._work_queue.task_done()
except Queue.Empty:
return
def is_work_queue_empty(self):
return self._work_queue.empty()
def wait_until_done(self):
self._work_queue.join()
def main(argv=None):
if argv is None:
argv = sys.argv
id_base = int(sys.argv[1])
manager = Manager()
wait_q = PriorityQueue()
out_file = open('latency.csv', 'a')
load_thread = Thread(target=check_load)
load_thread.start()
logger.info("PID: %d" % os.getpid())
logger.info("Running the gauntlet:")
id = id_base
num_clients = 0
for i in range(2):
wait_q.put((time.time(), False, id))
id += 1
num_clients += 1
for i in range(2):
wait_q.put((time.time(), True, id))
id += 1
num_clients += 1
pool = Pool(processes=num_clients)
while True:
t, attacker, id = wait_q.get()
delta = t - time.time()
delay = max(delta, 0)
if delay > 0:
Timer(delay, pool.apply_async, (fetch, (attacker, id))).start()
else:
pool.apply_async(fetch, (attacker, id))
wait_q.task_done()
while not wait_q_q.empty():
wait_q.put(wait_q_q.get())
# we have to make sure there is at least one element in wait_q before the end of the loop
if wait_q.empty():
r = wait_q_q.get()
wait_q.put(r)
while not res_q.empty():
id, n, t, error = res_q.get()
logger.info("id: %d got a response for n = %d in %f error: %d" %
(id, n, t, error))
out_file.write("%d, %d, %f, %d\n" % (id, n, t, error))
#if time.time() - start_time > 120:
# break
out_file.flush()
while not res_q.empty():
res = res_q.get()
out_file.write("%d, %f, %d\n" % (res[0], res[1], res[2]))
return 0
class scanmatcher_odom(object):
def __init__(self):
self._scan_msg = LaserScan()
self._odom_msg = Odometry()
# parameters
self._send_odom_base_tf = rospy.get_param('~send_odom_base_tf',
default=True)
# publishers
self._odom_pub = rospy.Publisher('/odom', Odometry, queue_size=10)
# tf broadcasters
self._odom_base_link_tfb = tf.TransformBroadcaster()
# tf listeners
self._base_link_base_laser_tfl = tf.TransformListener()
# user init
# processing queue
self._scan_queue = PriorityQueue(maxsize=2)
# absolute odometry pose and current velocities
self._current_pose = pymrpt.poses.CPose2D()
self._v = .0
self._w = .0
# initialize odometry
self._odom_msg.header.frame_id = 'odom'
self._odom_msg.child_frame_id = 'base_link'
# end of user init
# subscribers
rospy.Subscriber('/scan',
LaserScan,
self._scan_callback,
queue_size=10)
def get_base_link_base_laser_tf(self, stamp=None):
pose = PoseStamped()
if stamp is None:
stamp = rospy.Time(0)
try:
trans, rot = self._base_link_base_laser_tfl.lookupTransform(
'/base_link', '/base_laser', stamp)
except (tf.LookupException, tf.ConnectivityException,
tf.ExtrapolationException):
# TODO: implement TransformListener.lookupTransform() exception handling!
rospy.logwarn(
'Error while listening to /base_link -> /base_laser tf')
return pose, False
pose.pose.position.x = trans[0]
pose.pose.position.y = trans[1]
pose.pose.position.z = trans[2]
pose.pose.orientation.x = rot[0]
pose.pose.orientation.y = rot[1]
pose.pose.orientation.z = rot[2]
pose.pose.orientation.w = rot[3]
return pose, True
def send_odom_base_link_tf(self, pose):
translation = pose.pose.position
rotation = pose.pose.orientation
self._odom_base_link_tfb.sendTransform(
(translation.x, translation.y, translation.z),
(rotation.x, rotation.y, rotation.z, rotation.w),
pose.header.stamp, '/base_link', '/odom')
def run(self):
# create worker thread (we do not .join() since it closes itself by ros shutdown)
scanmatcher_thread = Thread(target=self.scanmatcher_worker)
scanmatcher_thread.start()
# keep it running
rospy.spin()
def _scan_callback(self, _scan_msg):
# put scans into queue
try:
self._scan_queue.put_nowait(
[_scan_msg.header.stamp.to_nsec(), _scan_msg])
except Full:
rospy.logwarn('Dropped scan. Processing queue is full!')
pass
def scanmatcher_worker(self):
# vars
has_initial_scan = False
# initialize ICP
icp_options = pymrpt.slam.CICP.TConfigParams()
icp = pymrpt.slam.CICP(icp_options)
# create last and current laser maps
last_map = pymrpt.maps.CSimplePointsMap()
# setup last update time
last_update_time = rospy.Time.now()
# loop
while not rospy.is_shutdown():
self._scan_msg = LaserScan()
# get scan from queue
try:
item = self._scan_queue.get(timeout=5)
except Empty:
rospy.logwarn(
'Got no scan from queue since 5 seconds. Scanmatcher will shutdown now!'
)
continue
self._scan_msg = item[1]
self._scan_msg.ranges = list(self._scan_msg.ranges)
rospy.loginfo('received scan...')
# update current stamp for publishers
self._current_stamp = self._scan_msg.header.stamp
# get laser scanner pose
laser_pose, ok = self.get_base_link_base_laser_tf()
self._sensor_pose = pymrpt.poses.CPose3D()
self._sensor_pose.from_ROS_Pose_msg(laser_pose.pose)
# convert data
observation = pymrpt.obs.CObservation2DRangeScan()
observation.from_ROS_LaserScan_msg(self._scan_msg,
self._sensor_pose)
# set current map from scan
current_map = pymrpt.maps.CSimplePointsMap()
current_map.loadFromRangeScan(observation)
# match maps
if has_initial_scan:
# no initial guess (pure incremental)
initial_guess = pymrpt.poses.CPosePDFGaussian()
# run ICP algorithm
pose_change, running_time, info = icp.AlignPDF(
last_map, current_map, initial_guess)
rospy.loginfo('icp goodness: {}'.format(info.goodness))
rospy.loginfo('icp run_time: {}'.format(running_time))
rospy.loginfo('pose change: {}'.format(pose_change.mean))
# check goodness
# if info.goodness > .8:
rospy.loginfo('...updating odometry.')
# get time delta
d_t = (last_update_time - self._current_stamp).to_sec()
# update current pose and velocities
self._current_pose += pose_change.mean
dist = pose_change.mean.norm()
self._v = dist / d_t
self._w = pose_change.mean.phi / d_t
self.publish_odom()
# update last update time
last_update_time = self._current_stamp
# update last map
last_map = current_map
# else:
# rospy.logwarn('...lost odometry...')
else:
rospy.loginfo('...is inital one!')
# load initial scan to last map
last_map = current_map
# mark initial as received
has_initial_scan = True
# rospy.loginfo('...task done!')
# signalize work done
self._scan_queue.task_done()
def publish_odom(self):
"""
Publish odometry and optionally "odom -> base_link" tf.
"""
# setup odometry message
self._odom_msg.header.stamp = self._current_stamp
self._odom_msg.pose.pose = self._current_pose.to_ROS_Pose_msg()
self._odom_msg.twist.twist.linear.x = self._v
self._odom_msg.twist.twist.angular.z = self._w
# publish odom
self._odom_pub.publish(self._odom_msg)
# publish "odom -> base_link" tf on demand
if self._send_odom_base_tf:
pose_msg = PoseStamped()
pose_msg.header.stamp = self._current_stamp
pose_msg.pose = self._odom_msg.pose.pose
self.send_odom_base_link_tf(pose_msg)
