def test_gvim_damage_performance(rectangles):
start = time.time()
for _ in range(N):
rects = []
for x,y,width,height in rectangles:
r = rectangle(x, y, width, height)
rects.append(r)
end = time.time()
print("created %s rectangles %s times in %.2fms" % (len(rectangles), N, (end-start)*1000.0/N))
#now try add rectangle:
start = time.time()
for _ in range(N):
rects = []
for x,y,width,height in rectangles:
r = rectangle(x, y, width, height)
add_rectangle(rects, r)
end = time.time()
print("add_rectangle %s rectangles %s times in %.2fms" % (len(rectangles), N, (end-start)*1000.0/N))
#now try remove rectangle:
start = time.time()
for _ in range(N):
rects = []
for x,y,width,height in rectangles:
r = rectangle(x+width//4, y+height//3, width//2, height//2)
remove_rectangle(rects, r)
end = time.time()
print("remove_rectangle %s rectangles %s times in %.2fms" % (len(rectangles), N, (end-start)*1000.0/N))
start = time.time()
n = N*1000
for _ in range(n):
for r in rects:
contains_rect(rects, r)
end = time.time()
print("contains_rect %s rectangles %s times in %.2fms" % (len(rectangles), n, (end-start)*1000.0/N))
def test_gvim_damage_performance(rectangles):
start = time.time()
for _ in range(N):
rects = []
for x, y, width, height in rectangles:
r = rectangle(x, y, width, height)
rects.append(r)
end = time.time()
print("created %s rectangles %s times in %.2fms" % (len(rectangles), N, (end - start) * 1000.0 / N))
# now try add rectangle:
start = time.time()
for _ in range(N):
rects = []
for x, y, width, height in rectangles:
r = rectangle(x, y, width, height)
add_rectangle(rects, r)
end = time.time()
print("add_rectangle %s rectangles %s times in %.2fms" % (len(rectangles), N, (end - start) * 1000.0 / N))
# now try remove rectangle:
start = time.time()
for _ in range(N):
rects = []
for x, y, width, height in rectangles:
r = rectangle(x + width // 4, y + height // 3, width // 2, height // 2)
remove_rectangle(rects, r)
end = time.time()
print("remove_rectangle %s rectangles %s times in %.2fms" % (len(rectangles), N, (end - start) * 1000.0 / N))
start = time.time()
n = N * 1000
for _ in range(n):
for r in rects:
contains_rect(rects, r)
end = time.time()
print("contains_rect %s rectangles %s times in %.2fms" % (len(rectangles), n, (end - start) * 1000.0 / N))
def test_cases(self):
from xpra.server.window.video_subregion import scoreinout #, sslog
from xpra.server.window.region import rectangle #@UnresolvedImport
#sslog.enable_debug()
r = rectangle(35, 435, 194, 132)
score = scoreinout(1200, 1024, r, 1466834, 21874694)
assert score < 100
r = rectangle(100, 600, 320, 240)
score = scoreinout(1200, 1024, r, 320 * 240 * 10, 320 * 240 * 25)
assert score < 100
def test_cases(self):
from xpra.server.window.video_subregion import scoreinout #, sslog
from xpra.server.window.region import rectangle #@UnresolvedImport
#sslog.enable_debug()
r = rectangle(35, 435, 194, 132)
score = scoreinout(1200, 1024, r, 1466834, 21874694)
assert score<100
r = rectangle(100, 600, 320, 240)
score = scoreinout(1200, 1024, r, 320*240*10, 320*240*25)
assert score<100
def set_exclusion_zones(self, zones):
rects = []
for (x, y, w, h) in zones:
rects.append(rectangle(int(x), int(y), int(w), int(h)))
self.exclusion_zones = rects
#force expire:
self.counter = 0
def set_exclusion_zones(self, zones):
rects = []
for (x, y, w, h) in zones:
rects.append(rectangle(int(x), int(y), int(w), int(h)))
self.exclusion_zones = rects
#force expire:
self.counter = 0
def test_merge_all():
start = time.time()
R = [rectangle(*v) for v in R1 + R2]
n = N * 10
for _ in range(n):
v = merge_all(R)
end = time.time()
print("merged %s rectangles %s times in %.2fms" % (len(R), n, (end - start) * 1000.0 / N))
def set_region(self, x, y, w, h):
sslog("set_region%s", (x, y, w, h))
if self.detection:
sslog("video region detection is on - the given region may or may not stick")
if x==0 and y==0 and w==0 and h==0:
self.novideoregion()
else:
self.rectangle = rectangle(x, y, w, h)
def set_region(self, x, y, w, h):
sslog("set_region%s", (x, y, w, h))
if self.detection:
sslog("video region detection is on - the given region may or may not stick")
if x==0 and y==0 and w==0 and h==0:
self.novideoregion()
else:
self.rectangle = rectangle(x, y, w, h)
def test_merge_all():
start = time.time()
R = [rectangle(*v) for v in R1+R2]
n = N*10
for _ in range(n):
v = merge_all(R)
end = time.time()
print("merged %s rectangles %s times in %.2fms" % (len(R), n, (end-start)*1000.0/N))
def do_screen_refresh(self, rlist):
#TODO: improve damage method to handle lists directly:
from xpra.server.window.region import rectangle #@UnresolvedImport
model_rects = {}
for model in self._id_to_window.values():
model_rects[model] = rectangle(*model.geometry)
for x, y, w, h in rlist:
for model, rect in model_rects.items():
mrect = rect.intersection(x, y, w, h)
#log("screen refresh intersection of %s and %24s: %s", model, (x, y, w, h), mrect)
if mrect:
self._damage(model, mrect.x - rect.x, mrect.y - rect.y,
mrect.width, mrect.height)
def damaged_ratio(rect):
if all_damaged:
return 1
rects = [rect]
for _,x,y,w,h in lde:
r = rectangle(x,y,w,h)
new_rects = []
for cr in rects:
new_rects += cr.substract_rect(r)
if not new_rects:
#nothing left: damage covered the whole rect
return 1.0
rects = new_rects
not_damaged_pixels = sum((r.width*r.height) for r in rects)
rect_pixels = rect.width*rect.height
#sslog("damaged_ratio: not damaged pixels(%s)=%i, rect pixels(%s)=%i", rects, not_damaged_pixels, rect, rect_pixels)
return max(0, min(1, 1.0-float(not_damaged_pixels)/float(rect_pixels)))
def damaged_ratio(rect):
if all_damaged:
return 1
rects = [rect]
for _,x,y,w,h in lde:
r = rectangle(x,y,w,h)
new_rects = []
for cr in rects:
new_rects += cr.substract_rect(r)
if not new_rects:
#nothing left: damage covered the whole rect
return 1.0
rects = new_rects
not_damaged_pixels = sum((r.width*r.height) for r in rects)
rect_pixels = rect.width*rect.height
#sslog("damaged_ratio: not damaged pixels(%s)=%i, rect pixels(%s)=%i", rects, not_damaged_pixels, rect, rect_pixels)
return max(0, min(1, 1.0-float(not_damaged_pixels)/float(rect_pixels)))
def identify_video_subregion(self,
ww,
wh,
damage_events_count,
last_damage_events,
starting_at=0):
if not self.enabled or not self.supported:
self.novideoregion("disabled")
return
if not self.detection:
return
sslog("%s.identify_video_subregion(..)", self)
sslog("identify_video_subregion(%s, %s, %s, %s)", ww, wh,
damage_events_count, last_damage_events)
if damage_events_count < self.set_at:
#stats got reset
self.set_at = 0
#validate against window dimensions:
rect = self.rectangle
if rect and (rect.width > ww or rect.height > wh):
#region is now bigger than the window!
return self.novideoregion(
"window is now smaller than current region")
#arbitrary minimum size for regions we will look at:
#(we don't want video regions smaller than this - too much effort for little gain)
if ww < MIN_W or wh < MIN_H:
return self.novideoregion("window is too small: %sx%s", MIN_W,
MIN_H)
def update_markers():
self.counter = damage_events_count
self.time = time.time()
def few_damage_events(event_types, event_count):
elapsed = time.time() - self.time
#how many damage events occurred since we chose this region:
event_count = max(0, damage_events_count - self.set_at)
#make the timeout longer when the region has worked longer:
slow_region_timeout = 2 + math.log(2 + event_count, 1.5)
if rect and elapsed >= slow_region_timeout:
update_markers()
return self.novideoregion(
"too much time has passed (%is for %s %s events)", elapsed,
event_types, event_count)
sslog(
"identify video: waiting for more %s damage events (%s) counters: %s / %s",
event_types, event_count, self.counter, damage_events_count)
if self.counter + 10 > damage_events_count:
#less than 10 events since last time we called update_markers:
event_count = damage_events_count - self.counter
few_damage_events("total", event_count)
return
from_time = max(starting_at, time.time() - MAX_TIME)
#create a list (copy) to work on:
lde = [x for x in list(last_damage_events) if x[0] >= from_time]
dc = len(lde)
if dc <= MIN_EVENTS:
return self.novideoregion("not enough damage events yet (%s)", dc)
#structures for counting areas and sizes:
wc = {}
hc = {}
dec = {}
#count how many times we see each area, each width/height and where,
#after removing any exclusion zones:
for _, x, y, w, h in lde:
r = rectangle(x, y, w, h)
rects = [r]
if self.exclusion_zones:
for e in self.exclusion_zones:
new_rects = []
for r in rects:
ex, ey, ew, eh = e.get_geometry()
if ex < 0 or ey < 0:
#negative values are relative to the width / height of the window:
if ex < 0:
ex = max(0, ww - ew)
if ey < 0:
ey = max(0, wh - eh)
new_rects += r.substract(ex, ey, ew, eh)
rects = new_rects
for r in rects:
dec.setdefault(r, MutableInteger()).increase()
if w >= MIN_W:
wc.setdefault(w, dict()).setdefault(x, set()).add(r)
if h >= MIN_H:
hc.setdefault(h, dict()).setdefault(y, set()).add(r)
#we can shortcut the damaged ratio if the whole window got damaged at least once:
all_damaged = dec.get(rectangle(0, 0, ww, wh), 0) > 0
def inoutcount(region, ignore_size=0):
#count how many pixels are in or out if this region
incount, outcount = 0, 0
for r, count in dec.items():
inregion = r.intersection_rect(region)
if inregion:
incount += inregion.width * inregion.height * int(count)
outregions = r.substract_rect(region)
for x in outregions:
if ignore_size > 0 and x.width * x.height < ignore_size:
#skip small region outside rectangle
continue
outcount += x.width * x.height * int(count)
return incount, outcount
def damaged_ratio(rect):
if all_damaged:
return 1
rects = [rect]
for _, x, y, w, h in lde:
r = rectangle(x, y, w, h)
new_rects = []
for cr in rects:
new_rects += cr.substract_rect(r)
if not new_rects:
#nothing left: damage covered the whole rect
return 1.0
rects = new_rects
not_damaged_pixels = sum((r.width * r.height) for r in rects)
rect_pixels = rect.width * rect.height
#sslog("damaged_ratio: not damaged pixels(%s)=%i, rect pixels(%s)=%i", rects, not_damaged_pixels, rect, rect_pixels)
return max(
0, min(1,
1.0 - float(not_damaged_pixels) / float(rect_pixels)))
scores = {None: 0}
def score_region(info, region, ignore_size=0, d_ratio=0):
score = scores.get(region)
if score is not None:
return score
#check if the region given is a good candidate, and if so we use it
#clamp it:
if region.width < MIN_W or region.height < MIN_H:
#too small, ignore it:
return 0
#and make sure this does not end up much bigger than needed:
if ww * wh < (region.width * region.height):
return 0
incount, outcount = inoutcount(region, ignore_size)
total = incount + outcount
score = scoreinout(ww, wh, region, incount, outcount)
#discount score if the region contains areas that were not damaged:
#(apply sqrt to limit the discount: 50% damaged -> multiply by 0.7)
if d_ratio == 0:
d_ratio = damaged_ratio(region)
score = int(score * math.sqrt(d_ratio))
sslog(
"testing %12s video region %34s: %3i%% in, %3i%% out, %3i%% of window, damaged ratio=%.2f, score=%2i",
info, region, 100 * incount // total, 100 * outcount // total,
100 * region.width * region.height / ww / wh, d_ratio, score)
scores[region] = score
return score
def updateregion(rect):
self.rectangle = rect
self.time = time.time()
self.inout = inoutcount(rect)
self.score = scoreinout(ww, wh, rect, *self.inout)
self.fps = int(self.inout[0] / (rect.width * rect.height) /
(time.time() - from_time))
self.damaged = int(100 * damaged_ratio(self.rectangle))
self.last_scores = scores
sslog("score(%s)=%s, damaged=%i%%", self.inout, self.score,
self.damaged)
def setnewregion(rect, msg="", *args):
if not self.rectangle or self.rectangle != rect:
sslog("setting new region %s: " + msg, rect, *args)
self.set_at = damage_events_count
self.counter = damage_events_count
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
return
if not self.detection:
return
updateregion(rect)
update_markers()
if len(dec) == 1:
rect, count = dec.items()[0]
return setnewregion(rect, "only region damaged")
#see if we can keep the region we already have (if any):
cur_score = 0
if rect:
cur_score = score_region("current", rect)
if cur_score >= KEEP_SCORE:
sslog("keeping existing video region %s with score %s", rect,
cur_score)
return
#split the regions we really care about (enough pixels, big enough):
damage_count = {}
min_count = max(2, len(lde) / 40)
for r, count in dec.items():
#ignore small regions:
if count > min_count and r.width >= MIN_W and r.height >= MIN_H:
damage_count[r] = count
c = sum([int(x) for x in damage_count.values()])
most_damaged = -1
most_pct = 0
if c > 0:
most_damaged = int(sorted(damage_count.values())[-1])
most_pct = 100 * most_damaged / c
sslog("identify video: most=%s%% damage count=%s", most_pct,
damage_count)
#is there a region that stands out?
#try to use the region which is responsible for most of the large damage requests:
most_damaged_regions = [
r for r, v in damage_count.items() if v == most_damaged
]
if len(most_damaged_regions) == 1:
r = most_damaged_regions[0]
score = score_region("most-damaged", r, d_ratio=1.0)
sslog("identify video: score most damaged area %s=%i%%", r,
score)
if score > 120:
setnewregion(r, "%s%% of large damage requests, score=%s",
most_pct, score)
return
elif score >= 100:
scores[r] = score
#try harder: try combining regions with the same width or height:
#(some video players update the video region in bands)
for w, d in wc.items():
for x, regions in d.items():
if len(regions) >= 2:
#merge regions of width w at x
min_count = max(2, len(regions) / 25)
keep = [
r for r in regions if int(dec.get(r, 0)) >= min_count
]
sslog(
"vertical regions of width %i at %i with at least %i hits: %s",
w, x, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("vertical", merged,
48 * 48)
for h, d in hc.items():
for y, regions in d.items():
if len(regions) >= 2:
#merge regions of height h at y
min_count = max(2, len(regions) / 25)
keep = [
r for r in regions if int(dec.get(r, 0)) >= min_count
]
sslog(
"horizontal regions of height %i at %i with at least %i hits: %s",
h, y, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("horizontal", merged,
48 * 48)
sslog("merged regions scores: %s", scores)
highscore = max(scores.values())
#a score of 100 is neutral
if highscore >= 120:
region = [r for r, s in scores.items() if s == highscore][0]
return setnewregion(region, "very high score: %s", highscore)
#retry existing region, tolerate lower score:
if cur_score >= 90 and (highscore < 100 or cur_score >= highscore):
sslog("keeping existing video region %s with score %s", rect,
cur_score)
return setnewregion(self.rectangle,
"existing region with score: %i" % cur_score)
if highscore >= 100:
region = [r for r, s in scores.items() if s == highscore][0]
return setnewregion(region, "high score: %s", highscore)
#FIXME: re-add some scrolling detection
#try harder still: try combining all the regions we haven't discarded
#(flash player with firefox and youtube does stupid unnecessary repaints)
if len(damage_count) >= 2:
merged = merge_all(damage_count.keys())
score = score_region("merged", merged)
if score >= 110:
return setnewregion(merged, "merged all regions, score=%s",
score)
self.novideoregion("failed to identify a video region")
self.last_scores = scores
def identify_video_subregion(self, ww, wh, damage_events_count, last_damage_events, starting_at=0):
if not self.enabled or not self.supported:
self.novideoregion("disabled")
return
if not self.detection:
return
sslog("%s.identify_video_subregion(..)", self)
sslog("identify_video_subregion(%s, %s, %s, %s)", ww, wh, damage_events_count, last_damage_events)
if damage_events_count < self.set_at:
#stats got reset
self.set_at = 0
#validate against window dimensions:
rect = self.rectangle
if rect and (rect.width>ww or rect.height>wh):
#region is now bigger than the window!
return self.novideoregion("window is now smaller than current region")
#arbitrary minimum size for regions we will look at:
#(we don't want video regions smaller than this - too much effort for little gain)
if ww<MIN_W or wh<MIN_H:
return self.novideoregion("window is too small: %sx%s", MIN_W, MIN_H)
def update_markers():
self.counter = damage_events_count
self.time = time.time()
def few_damage_events(event_types, event_count):
elapsed = time.time()-self.time
#how many damage events occurred since we chose this region:
event_count = max(0, damage_events_count - self.set_at)
#make the timeout longer when the region has worked longer:
slow_region_timeout = 2 + math.log(2+event_count, 1.5)
if rect and elapsed>=slow_region_timeout:
update_markers()
return self.novideoregion("too much time has passed (%is for %s %s events)", elapsed, event_types, event_count)
sslog("identify video: waiting for more %s damage events (%s) counters: %s / %s", event_types, event_count, self.counter, damage_events_count)
if self.counter+10>damage_events_count:
#less than 10 events since last time we called update_markers:
event_count = damage_events_count-self.counter
few_damage_events("total", event_count)
return
from_time = max(starting_at, time.time()-MAX_TIME)
#create a list (copy) to work on:
lde = [x for x in list(last_damage_events) if x[0]>=from_time]
dc = len(lde)
if dc<=MIN_EVENTS:
return self.novideoregion("not enough damage events yet (%s)", dc)
#structures for counting areas and sizes:
wc = {}
hc = {}
dec = {}
#count how many times we see each area, each width/height and where,
#after removing any exclusion zones:
for _,x,y,w,h in lde:
rects = self.excluded_rectangles(rectangle(x,y,w,h), ww, wh)
for r in rects:
dec.setdefault(r, MutableInteger()).increase()
if w>=MIN_W:
wc.setdefault(w, dict()).setdefault(x, set()).add(r)
if h>=MIN_H:
hc.setdefault(h, dict()).setdefault(y, set()).add(r)
#we can shortcut the damaged ratio if the whole window got damaged at least once:
all_damaged = dec.get(rectangle(0, 0, ww, wh), 0) > 0
def inoutcount(region, ignore_size=0):
#count how many pixels are in or out if this region
incount, outcount = 0, 0
for r, count in dec.items():
inregion = r.intersection_rect(region)
if inregion:
incount += inregion.width*inregion.height*int(count)
outregions = r.substract_rect(region)
for x in outregions:
if ignore_size>0 and x.width*x.height<ignore_size:
#skip small region outside rectangle
continue
outcount += x.width*x.height*int(count)
return incount, outcount
def damaged_ratio(rect):
if all_damaged:
return 1
rects = [rect]
for _,x,y,w,h in lde:
r = rectangle(x,y,w,h)
new_rects = []
for cr in rects:
new_rects += cr.substract_rect(r)
if not new_rects:
#nothing left: damage covered the whole rect
return 1.0
rects = new_rects
not_damaged_pixels = sum((r.width*r.height) for r in rects)
rect_pixels = rect.width*rect.height
#sslog("damaged_ratio: not damaged pixels(%s)=%i, rect pixels(%s)=%i", rects, not_damaged_pixels, rect, rect_pixels)
return max(0, min(1, 1.0-float(not_damaged_pixels)/float(rect_pixels)))
scores = {None : 0}
def score_region(info, region, ignore_size=0, d_ratio=0):
score = scores.get(region)
if score is not None:
return score
#check if the region given is a good candidate, and if so we use it
#clamp it:
if region.width<MIN_W or region.height<MIN_H:
#too small, ignore it:
return 0
#and make sure this does not end up much bigger than needed:
if ww*wh<(region.width*region.height):
return 0
incount, outcount = inoutcount(region, ignore_size)
total = incount+outcount
score = scoreinout(ww, wh, region, incount, outcount)
#discount score if the region contains areas that were not damaged:
#(apply sqrt to limit the discount: 50% damaged -> multiply by 0.7)
if d_ratio==0:
d_ratio = damaged_ratio(region)
score = int(score * math.sqrt(d_ratio))
sslog("testing %12s video region %34s: %3i%% in, %3i%% out, %3i%% of window, damaged ratio=%.2f, score=%2i",
info, region, 100*incount//total, 100*outcount//total, 100*region.width*region.height/ww/wh, d_ratio, score)
scores[region] = score
return score
def updateregion(rect):
self.rectangle = rect
self.time = time.time()
self.inout = inoutcount(rect)
self.score = scoreinout(ww, wh, rect, *self.inout)
self.fps = int(self.inout[0]/(rect.width*rect.height) / (time.time()-from_time))
self.damaged = int(100*damaged_ratio(self.rectangle))
self.last_scores = scores
sslog("score(%s)=%s, damaged=%i%%", self.inout, self.score, self.damaged)
def setnewregion(rect, msg="", *args):
rects = self.excluded_rectangles(rect, ww, wh)
if len(rects)==0:
self.novideoregion("no match after removing excluded regions")
return
if len(rects)==1:
rect = rects[0]
else:
#use the biggest one of what remains:
def get_rect_size(rect):
return -rect.width * rect.height
biggest_rects = sorted(rects, key=get_rect_size)
rect = biggest_rects[0]
if rect.width<MIN_W or rect.height<MIN_H:
self.novideoregion("match is too small after removing excluded regions")
return
if not self.rectangle or self.rectangle!=rect:
sslog("setting new region %s: "+msg, rect, *args)
self.set_at = damage_events_count
self.counter = damage_events_count
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
return
if not self.detection:
return
updateregion(rect)
update_markers()
if len(dec)==1:
rect, count = dec.items()[0]
return setnewregion(rect, "only region damaged")
#see if we can keep the region we already have (if any):
cur_score = 0
if rect:
cur_score = score_region("current", rect)
if cur_score>=KEEP_SCORE:
sslog("keeping existing video region %s with score %s", rect, cur_score)
return
#split the regions we really care about (enough pixels, big enough):
damage_count = {}
min_count = max(2, len(lde)/40)
for r, count in dec.items():
#ignore small regions:
if count>min_count and r.width>=MIN_W and r.height>=MIN_H:
damage_count[r] = count
c = sum([int(x) for x in damage_count.values()])
most_damaged = -1
most_pct = 0
if c>0:
most_damaged = int(sorted(damage_count.values())[-1])
most_pct = 100*most_damaged/c
sslog("identify video: most=%s%% damage count=%s", most_pct, damage_count)
#is there a region that stands out?
#try to use the region which is responsible for most of the large damage requests:
most_damaged_regions = [r for r,v in damage_count.items() if v==most_damaged]
if len(most_damaged_regions)==1:
r = most_damaged_regions[0]
score = score_region("most-damaged", r, d_ratio=1.0)
sslog("identify video: score most damaged area %s=%i%%", r, score)
if score>120:
setnewregion(r, "%s%% of large damage requests, score=%s", most_pct, score)
return
elif score>=100:
scores[r] = score
#try harder: try combining regions with the same width or height:
#(some video players update the video region in bands)
for w, d in wc.items():
for x,regions in d.items():
if len(regions)>=2:
#merge regions of width w at x
min_count = max(2, len(regions)//25)
keep = [r for r in regions if int(dec.get(r, 0))>=min_count]
sslog("vertical regions of width %i at %i with at least %i hits: %s", w, x, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("vertical", merged, 48*48)
for h, d in hc.items():
for y,regions in d.items():
if len(regions)>=2:
#merge regions of height h at y
min_count = max(2, len(regions)//25)
keep = [r for r in regions if int(dec.get(r, 0))>=min_count]
sslog("horizontal regions of height %i at %i with at least %i hits: %s", h, y, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("horizontal", merged, 48*48)
sslog("merged regions scores: %s", scores)
highscore = max(scores.values())
#a score of 100 is neutral
if highscore>=120:
region = [r for r,s in scores.items() if s==highscore][0]
return setnewregion(region, "very high score: %s", highscore)
#retry existing region, tolerate lower score:
if cur_score>=90 and (highscore<100 or cur_score>=highscore):
sslog("keeping existing video region %s with score %s", rect, cur_score)
return setnewregion(self.rectangle, "existing region with score: %i" % cur_score)
if highscore>=100:
region = [r for r,s in scores.items() if s==highscore][0]
return setnewregion(region, "high score: %s", highscore)
#TODO:
# * re-add some scrolling detection: the region may have moved
# * re-try with a higher "from_time" and a higher score threshold
#try harder still: try combining all the regions we haven't discarded
#(flash player with firefox and youtube does stupid unnecessary repaints)
if len(damage_count)>=2:
merged = merge_all(damage_count.keys())
score = score_region("merged", merged)
if score>=110:
return setnewregion(merged, "merged all regions, score=%s", score)
self.novideoregion("failed to identify a video region")
self.last_scores = scores
def identify_video_subregion(self,
ww,
wh,
damage_events_count,
last_damage_events,
starting_at=0):
if not self.detection:
return
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
sslog("%s.identify_video_subregion(..)", self)
sslog("identify_video_subregion(%s, %s, %s, %s)", ww, wh,
damage_events_count, last_damage_events)
def setnewregion(rect, msg="", *args):
if rect.x <= 0 and rect.y <= 0 and rect.width >= ww and rect.height >= wh:
#same size as the window, don't use a region!
self.novideoregion("region is full window")
return
sslog("setting new region %s: " + msg, rect, *args)
self.set_at = damage_events_count
self.counter = damage_events_count
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
return
if not self.detection:
return
self.rectangle = rect
if damage_events_count < self.set_at:
#stats got reset
self.set_at = 0
#validate against window dimensions:
rect = self.rectangle
if rect and (rect.width > ww or rect.height > wh):
#region is now bigger than the window!
return self.novideoregion(
"window is now smaller than current region")
#arbitrary minimum size for regions we will look at:
#(we don't want video regions smaller than this - too much effort for little gain)
if ww < MIN_W or wh < MIN_H:
return self.novideoregion("window is too small: %sx%s", MIN_W,
MIN_H)
def update_markers():
self.counter = damage_events_count
self.time = time.time()
def few_damage_events(event_types, event_count):
elapsed = time.time() - self.time
#how many damage events occurred since we chose this region:
event_count = max(0, damage_events_count - self.set_at)
#make the timeout longer when the region has worked longer:
slow_region_timeout = 2 + math.log(2 + event_count, 1.5)
if rect and elapsed >= slow_region_timeout:
update_markers()
return self.novideoregion(
"too much time has passed (%is for %s %s events)", elapsed,
event_types, event_count)
sslog(
"identify video: waiting for more %s damage events (%s) counters: %s / %s",
event_types, event_count, self.counter, damage_events_count)
if self.counter + 10 > damage_events_count:
#less than 10 events since last time we called update_markers:
event_count = damage_events_count - self.counter
few_damage_events("total", event_count)
return
from_time = max(starting_at, time.time() - MAX_TIME)
#create a list (copy) to work on:
lde = [x for x in list(last_damage_events) if x[0] >= from_time]
dc = len(lde)
if dc <= MIN_EVENTS:
return self.novideoregion("not enough damage events yet (%s)", dc)
#structures for counting areas and sizes:
wc = {}
hc = {}
dec = {}
#count how many times we see each area, each width/height and where:
for _, x, y, w, h in lde:
r = rectangle(x, y, w, h)
dec.setdefault(r, MutableInteger()).increase()
if w >= MIN_W:
wc.setdefault(w, dict()).setdefault(x, set()).add(r)
if h >= MIN_H:
hc.setdefault(h, dict()).setdefault(y, set()).add(r)
def score_region(info, region, ignore_size=0):
#check if the region given is a good candidate, and if so we use it
#clamp it:
width = min(ww, region.width)
height = min(wh, region.height)
if width < MIN_W or height < MIN_H:
#too small, ignore it:
return 0
#and make sure this does not end up much bigger than needed:
insize = width * height
if ww * wh < insize:
return 0
#count how many pixels are in or out if this region
incount, outcount = 0, 0
for r, count in dec.items():
inregion = r.intersection_rect(region)
if inregion:
incount += inregion.width * inregion.height * int(count)
outregions = r.substract_rect(region)
for x in outregions:
if ignore_size > 0 and x.width * x.height < ignore_size:
#skip small region outside rectangle
continue
outcount += x.width * x.height * int(count)
total = incount + outcount
assert total > 0
inpct = 100 * incount / total
outpct = 100 * outcount / total
#devaluate by taking into account the number of pixels in the area
#so that a large video region only wins if it really
#has a larger proportion of the pixels
#(offset the "insize" to even things out a bit:
# if we have a series of vertical or horizontal bands that we merge,
# we would otherwise end up excluding the ones on the edge
# if they ever happen to have a slightly lower hit count)
score = inpct * ww * wh * 2 / (ww * wh + insize)
sslog(
"testing %12s video region %34s: %3i%% in, %3i%% out, %3i%% of window, score=%2i",
info, region, inpct, outpct, 100 * width * height / ww / wh,
score)
return score
update_markers()
#see if we can keep the region we already have (if any):
cur_score = 0
if rect:
cur_score = score_region("current", rect)
if cur_score >= 125:
sslog("keeping existing video region %s with score %s", rect,
cur_score)
return
scores = {None: 0}
#split the regions we really care about (enough pixels, big enough):
damage_count = {}
min_count = max(2, len(lde) / 40)
for r, count in dec.items():
#ignore small regions:
if count > min_count and r.width >= MIN_W and r.height >= MIN_H:
damage_count[r] = count
c = sum([int(x) for x in damage_count.values()])
most_damaged = -1
most_pct = 0
if c > 0:
most_damaged = int(sorted(damage_count.values())[-1])
most_pct = 100 * most_damaged / c
sslog("identify video: most=%s%% damage count=%s", most_pct,
damage_count)
#is there a region that stands out?
#try to use the region which is responsible for most of the large damage requests:
most_damaged_regions = [
r for r, v in damage_count.items() if v == most_damaged
]
if len(most_damaged_regions) == 1:
r = most_damaged_regions[0]
score = score_region("most-damaged", r)
sslog("identify video: score most damaged area %s=%s%%", r,
score)
if score > 120:
setnewregion(r, "%s%% of large damage requests, score=%s",
most_pct, score)
return
elif score >= 100:
scores[r] = score
#try harder: try combining regions with the same width or height:
#(some video players update the video region in bands)
for w, d in wc.items():
for x, regions in d.items():
if len(regions) >= 2:
#merge regions of width w at x
min_count = max(2, len(regions) / 25)
keep = [
r for r in regions if int(dec.get(r, 0)) >= min_count
]
sslog(
"vertical regions of width %i at %i with at least %i hits: %s",
w, x, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("vertical", merged,
48 * 48)
for h, d in hc.items():
for y, regions in d.items():
if len(regions) >= 2:
#merge regions of height h at y
min_count = max(2, len(regions) / 25)
keep = [
r for r in regions if int(dec.get(r, 0)) >= min_count
]
sslog(
"horizontal regions of height %i at %i with at least %i hits: %s",
h, y, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("horizontal", merged,
48 * 48)
sslog("merged regions scores: %s", scores)
highscore = max(scores.values())
#a score of 100 is neutral
if highscore >= 120:
region = [r for r, s in scores.items() if s == highscore][0]
return setnewregion(region, "very high score: %s", highscore)
#retry existing region, tolerate lower score:
if cur_score >= 90:
sslog("keeping existing video region %s with score %s", rect,
cur_score)
return
if highscore >= 100:
region = [r for r, s in scores.items() if s == highscore][0]
return setnewregion(region, "high score: %s", highscore)
#FIXME: re-add some scrolling detection
#try harder still: try combining all the regions we haven't discarded
#(flash player with firefox and youtube does stupid unnecessary repaints)
if len(damage_count) >= 2:
merged = merge_all(damage_count.keys())
score = score_region("merged", merged)
if score >= 110:
return setnewregion(merged, "merged all regions, score=%s",
score, 48 * 48)
self.novideoregion("failed to identify a video region")
def test_eq(self):
log = video_subregion.sslog
def refresh_cb(window, regions):
log("refresh_cb(%s, %s)", window, regions)
r = video_subregion.VideoSubregion(glib.timeout_add,
glib.source_remove, refresh_cb, 150,
True)
ww = 1024
wh = 768
def assertiswin():
assert r.rectangle and r.rectangle.get_geometry() == (
0, 0, ww,
wh), "rectangle %s does not match whole window %ix%i" % (
r.rectangle, ww, wh)
log("* checking that we need some events")
last_damage_events = []
for x in range(video_subregion.MIN_EVENTS):
last_damage_events.append((0, 0, 0, 1, 1))
r.identify_video_subregion(ww, wh, video_subregion.MIN_EVENTS,
last_damage_events)
assert r.rectangle is None
vr = (monotonic_time(), 100, 100, 320, 240)
log("* easiest case: all updates in one region")
last_damage_events = []
for _ in range(50):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 50, last_damage_events)
assert r.rectangle
assert r.rectangle == region.rectangle(*vr[1:])
log("* checking that empty damage events does not cause errors")
r.reset()
r.identify_video_subregion(ww, wh, 0, [])
assert r.rectangle is None
log("* checking that full window can be a region")
vr = (monotonic_time(), 0, 0, ww, wh)
last_damage_events = []
for _ in range(50):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 50, last_damage_events)
assert r.rectangle is not None
log("* checking that regions covering the whole window give the same result"
)
last_damage_events = deque(maxlen=150)
for x in range(4):
for y in range(4):
vr = (monotonic_time(), ww * x / 4, wh * y / 4, ww / 4, wh / 4)
for _ in range(3):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 150, last_damage_events)
assertiswin()
vr = (monotonic_time(), ww / 4, wh / 4, ww / 2, wh / 2)
log("* mixed with region using 1/4 of window and 1/3 of updates: %s",
vr)
for _ in range(24):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 200, last_damage_events)
assertiswin()
log("* info=%s", r.get_info())
log("* checking that two video regions quite far apart do not get merged"
)
last_damage_events = deque(maxlen=150)
r.reset()
v1 = (monotonic_time(), 100, 100, 320, 240)
v2 = (monotonic_time(), 500, 500, 320, 240)
for _ in range(50):
last_damage_events.append(v1)
last_damage_events.append(v2)
r.identify_video_subregion(ww, wh, 100, last_damage_events)
assert r.rectangle is None
log("* checking that two video regions close to each other can be merged"
)
for N1, N2 in ((50, 50), (60, 40), (50, 30)):
last_damage_events = deque(maxlen=150)
r.reset()
v1 = (monotonic_time(), 100, 100, 320, 240)
for _ in range(N1):
last_damage_events.append(v1)
v2 = (monotonic_time(), 460, 120, 320, 240)
for _ in range(N2):
last_damage_events.append(v2)
r.identify_video_subregion(ww, wh, 100, last_damage_events)
m = region.merge_all(
[region.rectangle(*v1[1:]),
region.rectangle(*v2[1:])])
assert r.rectangle and r.rectangle == m, "expected %s but got %s for N1=%i, N2=%i" % (
m, r.rectangle, N1, N2)
def identify_video_subregion(self, ww, wh, damage_events_count, last_damage_events, starting_at=0):
if not self.detection:
return
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
sslog("%s.identify_video_subregion(..)", self)
sslog("identify_video_subregion(%s, %s, %s, %s)", ww, wh, damage_events_count, last_damage_events)
def setnewregion(rect, msg="", *args):
if rect.x<=0 and rect.y<=0 and rect.width>=ww and rect.height>=wh:
#same size as the window, don't use a region!
self.novideoregion("region is full window")
return
sslog("setting new region %s: "+msg, rect, *args)
self.set_at = damage_events_count
self.counter = damage_events_count
if not self.enabled:
#could have been disabled since we started this method!
self.novideoregion("disabled")
return
if not self.detection:
return
self.rectangle = rect
if damage_events_count < self.set_at:
#stats got reset
self.set_at = 0
#validate against window dimensions:
rect = self.rectangle
if rect and (rect.width>ww or rect.height>wh):
#region is now bigger than the window!
return self.novideoregion("window is now smaller than current region")
#arbitrary minimum size for regions we will look at:
#(we don't want video regions smaller than this - too much effort for little gain)
if ww<MIN_W or wh<MIN_H:
return self.novideoregion("window is too small: %sx%s", MIN_W, MIN_H)
def update_markers():
self.counter = damage_events_count
self.time = time.time()
def few_damage_events(event_types, event_count):
elapsed = time.time()-self.time
#how many damage events occurred since we chose this region:
event_count = max(0, damage_events_count - self.set_at)
#make the timeout longer when the region has worked longer:
slow_region_timeout = 2 + math.log(2+event_count, 1.5)
if rect and elapsed>=slow_region_timeout:
update_markers()
return self.novideoregion("too much time has passed (%is for %s %s events)", elapsed, event_types, event_count)
sslog("identify video: waiting for more %s damage events (%s) counters: %s / %s", event_types, event_count, self.counter, damage_events_count)
if self.counter+10>damage_events_count:
#less than 10 events since last time we called update_markers:
event_count = damage_events_count-self.counter
few_damage_events("total", event_count)
return
from_time = max(starting_at, time.time()-MAX_TIME)
#create a list (copy) to work on:
lde = [x for x in list(last_damage_events) if x[0]>=from_time]
dc = len(lde)
if dc<=MIN_EVENTS:
return self.novideoregion("not enough damage events yet (%s)", dc)
#structures for counting areas and sizes:
wc = {}
hc = {}
dec = {}
#count how many times we see each area, each width/height and where:
for _,x,y,w,h in lde:
r = rectangle(x,y,w,h)
dec.setdefault(r, MutableInteger()).increase()
if w>=MIN_W:
wc.setdefault(w, dict()).setdefault(x, set()).add(r)
if h>=MIN_H:
hc.setdefault(h, dict()).setdefault(y, set()).add(r)
def score_region(info, region, ignore_size=0):
#check if the region given is a good candidate, and if so we use it
#clamp it:
width = min(ww, region.width)
height = min(wh, region.height)
if width<MIN_W or height<MIN_H:
#too small, ignore it:
return 0
#and make sure this does not end up much bigger than needed:
insize = width*height
if ww*wh<insize:
return 0
#count how many pixels are in or out if this region
incount, outcount = 0, 0
for r, count in dec.items():
inregion = r.intersection_rect(region)
if inregion:
incount += inregion.width*inregion.height*int(count)
outregions = r.substract_rect(region)
for x in outregions:
if ignore_size>0 and x.width*x.height<ignore_size:
#skip small region outside rectangle
continue
outcount += x.width*x.height*int(count)
total = incount+outcount
assert total>0
inpct = 100*incount/total
outpct = 100*outcount/total
#devaluate by taking into account the number of pixels in the area
#so that a large video region only wins if it really
#has a larger proportion of the pixels
#(offset the "insize" to even things out a bit:
# if we have a series of vertical or horizontal bands that we merge,
# we would otherwise end up excluding the ones on the edge
# if they ever happen to have a slightly lower hit count)
score = inpct * ww*wh*2 / (ww*wh + insize)
sslog("testing %12s video region %34s: %3i%% in, %3i%% out, %3i%% of window, score=%2i",
info, region, inpct, outpct, 100*width*height/ww/wh, score)
return score
update_markers()
#see if we can keep the region we already have (if any):
cur_score = 0
if rect:
cur_score = score_region("current", rect)
if cur_score>=125:
sslog("keeping existing video region %s with score %s", rect, cur_score)
return
scores = {None : 0}
#split the regions we really care about (enough pixels, big enough):
damage_count = {}
min_count = max(2, len(lde)/40)
for r, count in dec.items():
#ignore small regions:
if count>min_count and r.width>=MIN_W and r.height>=MIN_H:
damage_count[r] = count
c = sum([int(x) for x in damage_count.values()])
most_damaged = -1
most_pct = 0
if c>0:
most_damaged = int(sorted(damage_count.values())[-1])
most_pct = 100*most_damaged/c
sslog("identify video: most=%s%% damage count=%s", most_pct, damage_count)
#is there a region that stands out?
#try to use the region which is responsible for most of the large damage requests:
most_damaged_regions = [r for r,v in damage_count.items() if v==most_damaged]
if len(most_damaged_regions)==1:
r = most_damaged_regions[0]
score = score_region("most-damaged", r)
sslog("identify video: score most damaged area %s=%s%%", r, score)
if score>120:
setnewregion(r, "%s%% of large damage requests, score=%s", most_pct, score)
return
elif score>=100:
scores[r] = score
#try harder: try combining regions with the same width or height:
#(some video players update the video region in bands)
for w, d in wc.items():
for x,regions in d.items():
if len(regions)>=2:
#merge regions of width w at x
min_count = max(2, len(regions)/25)
keep = [r for r in regions if int(dec.get(r, 0))>=min_count]
sslog("vertical regions of width %i at %i with at least %i hits: %s", w, x, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("vertical", merged, 48*48)
for h, d in hc.items():
for y,regions in d.items():
if len(regions)>=2:
#merge regions of height h at y
min_count = max(2, len(regions)/25)
keep = [r for r in regions if int(dec.get(r, 0))>=min_count]
sslog("horizontal regions of height %i at %i with at least %i hits: %s", h, y, min_count, keep)
if keep:
merged = merge_all(keep)
scores[merged] = score_region("horizontal", merged, 48*48)
sslog("merged regions scores: %s", scores)
highscore = max(scores.values())
#a score of 100 is neutral
if highscore>=120:
region = [r for r,s in scores.items() if s==highscore][0]
return setnewregion(region, "very high score: %s", highscore)
#retry existing region, tolerate lower score:
if cur_score>=90:
sslog("keeping existing video region %s with score %s", rect, cur_score)
return
if highscore>=100:
region = [r for r,s in scores.items() if s==highscore][0]
return setnewregion(region, "high score: %s", highscore)
#FIXME: re-add some scrolling detection
#try harder still: try combining all the regions we haven't discarded
#(flash player with firefox and youtube does stupid unnecessary repaints)
if len(damage_count)>=2:
merged = merge_all(damage_count.keys())
score = score_region("merged", merged)
if score>=110:
return setnewregion(merged, "merged all regions, score=%s", score, 48*48)
self.novideoregion("failed to identify a video region")
def test_eq(self):
log = video_subregion.sslog
def refresh_cb(window, regions):
log("refresh_cb(%s, %s)", window, regions)
r = video_subregion.VideoSubregion(gobject.timeout_add, gobject.source_remove, refresh_cb, 150, True)
ww = 1024
wh = 768
def assertiswin():
assert r.rectangle and r.rectangle.get_geometry()==(0, 0, ww, wh), "rectangle %s does not match whole window %ix%i" % (r.rectangle, ww, wh)
log("* checking that we need some events")
last_damage_events = []
for x in range(video_subregion.MIN_EVENTS):
last_damage_events.append((0, 0, 0, 1, 1))
r.identify_video_subregion(ww, wh, video_subregion.MIN_EVENTS, last_damage_events)
assert r.rectangle is None
vr = (time.time(), 100, 100, 320, 240)
log("* easiest case: all updates in one region")
last_damage_events = []
for _ in range(50):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 50, last_damage_events)
assert r.rectangle
assert r.rectangle==region.rectangle(*vr[1:])
log("* checking that empty damage events does not cause errors")
r.reset()
r.identify_video_subregion(ww, wh, 0, [])
assert r.rectangle is None
log("* checking that full window can be a region")
vr = (time.time(), 0, 0, ww, wh)
last_damage_events = []
for _ in range(50):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 50, last_damage_events)
assert r.rectangle is not None
log("* checking that regions covering the whole window give the same result")
last_damage_events = deque(maxlen=150)
for x in range(4):
for y in range(4):
vr = (time.time(), ww*x/4, wh*y/4, ww/4, wh/4)
for _ in range(3):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 150, last_damage_events)
assertiswin()
vr = (time.time(), ww/4, wh/4, ww/2, wh/2)
log("* mixed with region using 1/4 of window and 1/3 of updates: %s", vr)
for _ in range(24):
last_damage_events.append(vr)
r.identify_video_subregion(ww, wh, 200, last_damage_events)
assertiswin()
log("* info=%s", r.get_info())
log("* checking that two video regions quite far apart do not get merged")
last_damage_events = deque(maxlen=150)
r.reset()
v1 = (time.time(), 100, 100, 320, 240)
v2 = (time.time(), 500, 500, 320, 240)
for _ in range(50):
last_damage_events.append(v1)
last_damage_events.append(v2)
r.identify_video_subregion(ww, wh, 100, last_damage_events)
assert r.rectangle is None
log("* checking that two video regions close to each other can be merged")
v1 = (time.time(), 100, 100, 320, 240)
v2 = (time.time(), 460, 120, 320, 240)
for N1, N2 in ((50, 50), (60, 40), (50, 30)):
last_damage_events = deque(maxlen=150)
r.reset()
for _ in range(N1):
last_damage_events.append(v1)
for _ in range(N2):
last_damage_events.append(v2)
r.identify_video_subregion(ww, wh, 100, last_damage_events)
m = region.merge_all([region.rectangle(*v1[1:]), region.rectangle(*v2[1:])])
assert r.rectangle and r.rectangle==m, "expected %s but got %s" % (m, r.rectangle)