def __init__(self, boundary, obstacle):
self.boundary = boundary
self.obstacle = obstacle
self.vertices = []
self.get_vertices()
self.current_edge_list = AVLTree()
self.vertical_extension = []
def reducer(self, key, values):
tree = AVLTree()
points = set()
for items in values:
for val in items:
y = val[2][0][1]
if val[1]:
tree.insert(y, val[2])
try:
i = line_intersection(val[2], tree.succ_item(y)[1])
if i:
points.add(i)
except KeyError:
pass
try:
i = line_intersection(val[2], tree.prev_item(y)[1])
if i:
points.add(i)
except KeyError:
pass
else:
try:
i = line_intersection(
tree.prev_item(y)[1],
tree.succ_item(y)[1])
if i:
points.add(i)
except KeyError:
pass
try:
tree.remove(y)
except KeyError:
pass
yield key, list(points)
def testRandomisedValueInsertion(self):
pytree = AVLTree()
for _ in range(self.TEST_SZ):
v = self.rand_val()
self.tree.insert(v)
pytree.insert(v, v)
self.assertEqual(len(self.tree), len(pytree))
self.assertTrue(self.tree.validateBSTInvariant(self.tree.root))
def __init__(self, logfile_full_path='logs/orderbook.log'):
self.bids = AVLTree()
self.asks = AVLTree()
# This is to retrieve an order *by order_id* from the trees in O(log(n)). (self.remove_order())
# self.bids and self.asks are indexed by Order._key(), which is (self.price, self.size).
self.order_id_key_translate = {}
self.trades = {}
self.subscribers = {}
self.logger = Logger(logfile_full_path)
def __init__(self, cache_size, min_obj_size, max_obj_size):
self._max_size = cache_size
self._used_size = 0
# dictionary: obj_id -> object with last and next caching time
self._cached_objects = {}
# AVL tree: next_time -> object with last and next caching time
self._tree = AVLTree()
self._oldest_obj_id = None
self._freshest_obj_id = None
self.stats = CacheStats.CacheStats("Belady", cache_size)
self.daily_stats = CacheStats.DailyCacheStats(cache_size)
def __init__(self, cfg=None):
self._blanket = AVLTree()
self._ffi = cffi.FFI()
if cfg is not None:
self._from_cfg(cfg)
else:
_l.debug(
"CFG is not specified. Initialize CFBlanket from the knowledge base."
)
for func in self.kb.functions.values():
self.add_function(func)
def __init__(self, is_stack):
"""
Constructor
:param is_stack: Whether this is a region map for stack frames or not. Different strategies apply for stack
regions.
"""
self.is_stack = is_stack
# An AVLTree, which maps stack addresses to region IDs
self._address_to_region_id = AVLTree()
# A dict, which maps region IDs to memory address ranges
self._region_id_to_address = { }
def grow_lambda_function2(self, wordlist):
self.word_list = wordlist
self.word_dict = {}
cnt = 0
while cnt < len(self.word_list):
self.index_dict[cnt] = cnt
cnt += 1
self.index_tree = BinaryTree(self.index_dict)
self.index_tree.foreach(self.build_lambda_comp_tree, 0)
self.lambda_comp_tree = AVLTree(self.word_dict)
print "==========================================================================="
print "Lambda Composition AVL Tree (inorder traversed) is the original text itself:"
print "==========================================================================="
self.lambda_expression = []
self.lambda_comp_tree.foreach(self.build_lambda_expression, 0)
print self.lambda_expression
print "==========================================================================="
print "Lambda Composition AVL Tree (postorder traversed - Postfix expression):"
print "Every parenthesis has two operands,operated by function outside:"
print "==============================================================="
self.lambda_expression = []
self.lambda_comp_tree.foreach(self.build_lambda_expression, 1)
self.lambda_composition = []
cnt = 0
per_random_walk_graph_tensor_neuron_network_intrinsic_merit = 0
#recursively evaluate the Graph Tensor Neuron Network for random walk composition tree bottom up as Graph Neural Network
#having Tensor Neuron activations for each subtree.
while len(self.lambda_expression) > 2:
operand2 = self.lambda_expression.pop()
operand1 = self.lambda_expression.pop()
function = self.lambda_expression.pop()
subtree_graph_tensor_neuron_network_wght = self.subtree_graph_tensor_neuron_network_weight(
operand1, function, operand2)
self.graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
per_random_walk_graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
self.lambda_composition = "(" + function + "(" + operand1 + "," + operand2 + "))"
self.lambda_expression.append(self.lambda_composition)
cnt += 1
if len(self.lambda_expression) > 1:
return (
self.lambda_expression[0] + "(" + self.lambda_expression[1] +
")",
per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
else:
return (
self.lambda_expression[0],
per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
class ParallelSBTree(object):
"""
Parallel threading structure as a Self-balancing Tree. Used to implement
the Reduce Parallel Computation algorithm. For operations over the whole
tree forming threads
"""
def __init__(self, entries, shared=None):
self.sem_lock = Semaphore(value=1)
self.psbt = AVLTree(entries)
self.shared = shared
def foreach(self, function, root):
if root == None:
return
value = root.value
# change value to args
worker_thread = Thread(target=function, args=(
self.shared,
value,
))
worker_thread.start()
# exponential spawning
if root.left != None:
worker_thread_l = Thread(target=self.foreach,
args=(function, root.left))
worker_thread_l.start()
if root.right != None:
worker_thread_r = Thread(target=self.foreach,
args=(function, root.right))
worker_thread_r.start()
def update_node(self, key, value):
print self.psbt
self.remove(key)
self.insert(key, value)
def new_node(self, key, value):
return self.psbt._new_node(key, value)
def insert(self, key, value):
self.psbt.insert(key, value)
def remove(self, key):
self.psbt.remove(key)
def merge_result(self, shared_result):
pass
def findIntersections(S):
T = AVLTree()
Q = [] # event queue with [QPointF, lineId]
for i, l in enumerate(S):
Q.append([l.p1(), i, "l"]) # l : left, r : right, c : center
Q.append([l.p2(), i, "r"])
while Q: # stops if Q = []
p = Q[0][0]
Q.pop(0)
# handleEventPoint(p)
Up = [p]
Lp = []
Segs = []
T.foreach(lambda k, v: print(k,v ) )
def getNewAVL(seq):
if has_fast_tree_support():
from bintrees import FastAVLTree
return FastAVLTree(seq)
else:
from bintrees import AVLTree
return AVLTree(seq)
def __init__(self):
self.lambda_comp_tree = AVLTree()
self.index_tree = BinaryTree()
self.word_list = []
self.word_dict = {}
self.index_dict = {}
self.index_list = []
self.lambda_expression = []
self.lambda_composition = ""
self.graph_tensor_neuron_network_intrinsic_merit = 1.0
self.entropy = 10000000000.0
self.conceptnet = ConceptNet5Client()
#self.Similarity="ConceptNet"
self.Similarity = "WordNet"
self.ClosedPaths = True
self.dictionary = PyDictionary()
def load_objects(self, index_name): # initialise empty tree self.objects = AVLTree() # try open file with objects try: ins = open(index_name + "/bin_tree_dump_objects.txt", "r") except: print "Index tree for objects doesnt exist" return # reading objects from file, one by one for line in ins: try: # load serialized data from file x = json.loads(line) # create instance of FreebaseObject o = FreebaseObject() # initialise content with object data from file o.deserialise(x) # insert loaded freebase object to objects tree self.objects.insert(o.get_id(),o) except: print "something wrong 2" print "Index tree for objects loaded"
def __init__(self, series):
self.__x_tree = AVLTree()
self.__y_tree = AVLTree()
self.__mapping = {}
for (line, (xs, ys)) in enumerate(series):
for (point, (x, y)) in enumerate(zip(xs, ys)):
self.__x_tree.insert(x, x)
self.__y_tree.insert(y, y)
if self.__mapping.get(x) is None:
self.__mapping[x] = {}
if self.__mapping[x].get(y) is None:
self.__mapping[x][y] = []
self.__mapping[x][y].append((line, point, x, y))
def grow_lambda_function1(self):
text=open("RecursiveLambdaFunctionGrowth.txt","r")
word_dict={}
index_dict={}
words_evaluated=0
word_list=text.read().split()
for cnt in range(1,len(word_list)):
index_dict[cnt-1] = len(word_list)/cnt
index_tree=AVLTree(index_dict)
print "Index AVL Tree:", repr(index_tree)
#index_tree.foreach(print_node,1)
try:
while words_evaluated < len(word_list):
#word_dict[words_evaluated]=word_list[random.randint(0,len(word_list)-1)]
#print word_list[index_tree.pop_min()[0]]
word_dict[words_evaluated]=word_list[index_tree.pop_min()[0]]
words_evaluated+=1
except:
pass
self.lambda_comp_tree=AVLTree(word_dict)
print "Lambda Composition AVL Tree:"
self.lambda_comp_tree.foreach(print_node)
iteration=0
while iteration < len(word_list):
k=self.lambda_comp_tree.get(iteration)
print "k:",k
try:
prev=self.lambda_comp_tree.prev_key(iteration)
prevk=self.lambda_comp_tree.get(prev)
print "prevk:",prevk
except:
pass
try:
succ=self.lambda_comp_tree.succ_key(iteration)
succk=self.lambda_comp_tree.get(succ)
print "succk:",succk
except:
pass
iteration+=1
def grow_lambda_function1(self):
text = open("RecursiveLambdaFunctionGrowth.txt", "r")
word_dict = {}
index_dict = {}
words_evaluated = 0
word_list = text.read().split()
for cnt in range(1, len(word_list)):
index_dict[cnt - 1] = len(word_list) / cnt
index_tree = AVLTree(index_dict)
print "Index AVL Tree:", repr(index_tree)
#index_tree.foreach(print_node,1)
try:
while words_evaluated < len(word_list):
#word_dict[words_evaluated]=word_list[random.randint(0,len(word_list)-1)]
#print word_list[index_tree.pop_min()[0]]
word_dict[words_evaluated] = word_list[index_tree.pop_min()[0]]
words_evaluated += 1
except:
pass
self.lambda_comp_tree = AVLTree(word_dict)
print "Lambda Composition AVL Tree:"
self.lambda_comp_tree.foreach(print_node)
iteration = 0
while iteration < len(word_list):
k = self.lambda_comp_tree.get(iteration)
print "k:", k
try:
prev = self.lambda_comp_tree.prev_key(iteration)
prevk = self.lambda_comp_tree.get(prev)
print "prevk:", prevk
except:
pass
try:
succ = self.lambda_comp_tree.succ_key(iteration)
succk = self.lambda_comp_tree.get(succ)
print "succk:", succk
except:
pass
iteration += 1
def __init__(self, is_stack):
"""
Constructor
:param is_stack: Whether this is a region map for stack frames or not. Different strategies apply for stack
regions.
"""
self.is_stack = is_stack
# An AVLTree, which maps stack addresses to region IDs
self._address_to_region_id = AVLTree()
# A dict, which maps region IDs to memory address ranges
self._region_id_to_address = {}
def __init__(self, name, genome):
self.name = name
self.genome = genome
self.sb_name_mapping = {}
self.start_telomer = StartTelomereSyntenyBlock(name)
self.end_telomer = EndTelomereSyntenyBlock(name)
self.blocks = AVLTree()
self.add_sb(self.start_telomer)
self.add_sb(self.end_telomer)
# maps sb into a dict of resolutions, it has mapping in
# in each particular resolution, mapping is represented as a list,
# this particular block is mapped to
self.mapping = defaultdict(lambda: defaultdict(list))
def intersections(psegs):
"""
Implementation of the Bentley-Ottmann algorithm.
Input
psegs: a list of segments
Output
intpoints: a list of intersection points
"""
eq = EventQueue(psegs)
intpoints = []
T = AVLTree()
L = []
while not eq.is_empty(): # for all events
e = eq.events.pop(0) # remove the event
p = e.p # get event point
L = e.edges # segments with p as left end
R, C = get_edges(T, p) # p: right (R) and interior (C)
if len(L + R + C) > 1: # Intersection at p among L+R+C
for s in L + R + C:
if not s.contains(p): # if p is interior
s.lp = p # change lp and
s.status = INTERIOR # status
intpoints.append(p)
R, C = get_edges(T, p)
for s in R + C:
T.discard(s)
for s in L + C:
T.insert(s, str(s))
if len(L + C) == 0:
s = R[0]
if s is not None:
sl, sr = get_lr(T, s)
find_new_event(sl, sr, p, eq)
else:
sp, spp = get_lrmost(T, L + C)
try:
sl = T.prev_key(sp)
except KeyError: # only on last key
sl = None
try:
sr = T.succ_key(spp)
except KeyError: # only on last key
sr = None
find_new_event(sl, sp, p, eq)
find_new_event(sr, spp, p, eq)
return intpoints
def get_edges(t, p):
"""
Gets the edges that contain point p as their right
endpoint or in the interior
"""
lr = []
lc = []
for s in AVLTree(t):
if s.rp == p:
lr.append(s)
elif s.lp == p and s.status == INTERIOR:
lc.append(s)
elif sideplr(p, s.lp, s.rp) == 0:
lc.append(s)
return lr, lc
class BinTree: tree = None def __init__(self): self.tree = AVLTree() def insert(self, rating, username): if not rating in self.tree: self.tree[rating] = set() self.tree[rating].add(username) def remove(self, rating, username): if rating in self.tree: self.tree[rating].remove(username) if len(self.tree[rating]) == 0: self.tree.discard(rating) def all_pairs_inbetween(self, low, high): pairs = [] for k, v in self.tree[low: high].items(): for u in v: pairs.append((k, u)) return pairs
def __init__(self):
self.lambda_comp_tree=AVLTree()
self.index_tree=BinaryTree()
self.word_list=[]
self.word_dict={}
self.index_dict={}
self.index_list=[]
self.lambda_expression=[]
self.lambda_composition=""
self.graph_tensor_neuron_network_intrinsic_merit=1.0
self.entropy=10000000000.0
self.conceptnet=ConceptNet5Client()
#self.Similarity="ConceptNet"
self.Similarity="WordNet"
self.ClosedPaths=True
self.dictionary=PyDictionary()
def intersections(psegs):
"""
Implementation of the Bentley-Ottmann algorithm.
Input
psegs: a list of segments
Output
intpoints: a list of intersection points
"""
eq = EventQueue(psegs)
intpoints = []
T = AVLTree()
L=[]
while not eq.is_empty(): # for all events
e = eq.events.pop(0) # remove the event
p = e.p # get event point
L = e.edges # segments with p as left end
R,C = get_edges(T, p) # p: right (R) and interior (C)
if len(L+R+C) > 1: # Intersection at p among L+R+C
for s in L+R+C:
if not s.contains(p): # if p is interior
s.lp = p # change lp and
s.status = INTERIOR # status
intpoints.append(p)
R,C = get_edges(T, p)
for s in R+C:
T.discard(s)
for s in L+C:
T.insert(s, str(s))
if len(L+C) == 0:
s = R[0]
if s is not None:
sl, sr = get_lr(T, s)
find_new_event(sl, sr, p, eq)
else:
sp, spp = get_lrmost(T, L+C)
try:
sl = T.prev_key(sp)
except KeyError: # only on last key
sl = None
try:
sr = T.succ_key(spp)
except KeyError: # only on last key
sr = None
find_new_event(sl, sp, p, eq)
find_new_event(sr, spp, p, eq)
return intpoints
def grow_lambda_function2(self, wordlist):
self.word_list=wordlist
self.word_dict={}
cnt=0
while cnt < len(self.word_list):
self.index_dict[cnt]=cnt
cnt+=1
self.index_tree=BinaryTree(self.index_dict)
self.index_tree.foreach(self.build_lambda_comp_tree,0)
self.lambda_comp_tree=AVLTree(self.word_dict)
print "==========================================================================="
print "Lambda Composition AVL Tree (inorder traversed) is the original text itself:"
print "==========================================================================="
self.lambda_expression=[]
self.lambda_comp_tree.foreach(self.build_lambda_expression, 0)
print self.lambda_expression
print "==========================================================================="
print "Lambda Composition AVL Tree (postorder traversed - Postfix expression):"
print "Every parenthesis has two operands,operated by function outside:"
print "==============================================================="
self.lambda_expression=[]
self.lambda_comp_tree.foreach(self.build_lambda_expression, 1)
self.lambda_composition=[]
cnt=0
per_random_walk_graph_tensor_neuron_network_intrinsic_merit = 0
#recursively evaluate the Graph Tensor Neuron Network for random walk composition tree bottom up as Graph Neural Network
#having Tensor Neuron activations for each subtree.
while len(self.lambda_expression) > 2 :
operand2=self.lambda_expression.pop()
operand1=self.lambda_expression.pop()
function=self.lambda_expression.pop()
subtree_graph_tensor_neuron_network_wght = self.subtree_graph_tensor_neuron_network_weight(operand1, function, operand2)
self.graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
per_random_walk_graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
self.lambda_composition="("+function+"("+operand1+","+operand2+"))"
self.lambda_expression.append(self.lambda_composition)
cnt+=1
if len(self.lambda_expression) > 1:
return (self.lambda_expression[0] + "("+self.lambda_expression[1]+")", per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
else:
return (self.lambda_expression[0], per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
class LinearStreamingInterpolator(StreamingInterpolatorBase):
""" Linear streaming interpolator """
def __init__(self):
self._history = AVLTree()
def insert(self, x, val):
"""
Register a datapoint
x (float) : the x coordinate of the datapoint
val (float): the rest of the datapoint
"""
self._history.insert(x, val)
def getInterpolatedVal(self, x):
"""
Get the interpolated value for the given x
x (float) : the x coordinate of the datapoint
returns (float) : interpolated value
"""
if self._history.is_empty():
return None
leftX, leftVal = None, None
rightX, rightVal = None, None
if self._history.min_key() <= x:
leftX, leftVal = self._history.floor_item(x)
if leftX == x:
return leftVal
if self._history.max_key() >= x:
rightX, rightVal = self._history.ceiling_item(x)
if rightX == x:
return rightVal
# check if on edge
if leftVal == None:
return rightVal
elif rightVal == None:
return leftVal
# find weighted average of the vals of the closest enclosing data
# points
intervalLength = abs(x - leftX) + abs(x - rightX)
value = float(abs(x - rightX) * leftVal + abs(x - leftX) * rightVal) / intervalLength
return value
class NearestNeighborStreamingInterpolator(StreamingInterpolatorBase):
""" Nearest Neighbor 1D Streaming Interpolator """
def __init__(self):
self._history = AVLTree()
def insert(self, x, val):
"""
Register a datapoint
x (float) : the x coordinate of the datapoint
val (float): the rest of the datapoint
"""
self._history.insert(x, val)
def getInterpolatedVal(self, x):
"""
Get the interpolated value for the given x
x (float) : the x coordinate of the datapoint
returns (float) : interpolated value
"""
if self._history.is_empty():
return None
# find the nearest point to the left and right
leftX, leftVal = None, None
rightX, rightVal = None, None
if self._history.min_key() <= x:
leftX, leftVal = self._history.floor_item(x)
if self._history.max_key() >= x:
rightX, rightVal = self._history.ceiling_item(x)
# if there is only one neighbor, return it
if leftVal == None:
return rightVal
elif rightVal == None:
return leftVal
# return the nearest neighbor
if abs(x - leftX) < abs(x - rightX):
return leftVal
else:
return rightVal
def testRandomRemove(self):
pytree = AVLTree()
for i in range(self.TEST_SZ):
size = i
lst = self.gen_rand_list(size)
for value in lst:
self.tree.insert(value)
pytree.insert(value, value)
random.shuffle(lst)
for j, value in enumerate(lst):
pytree.remove(value)
self.tree.remove(value)
self.assertEqual(len(pytree), len(self.tree))
self.assertEqual(size - j - 1, len(self.tree))
self.assertTrue(not self.tree)
def populate_distance(self):
added = 0
for i in range(self.N):
E_temp = self.Dist[i]
V_temp = {}
Q_temp = AVLTree()
for key in range(len(E_temp)):
j = key
D_temp = Distance(E_temp[key], key)
V_temp[key] = D_temp
if key != i:
Q_temp[D_temp] = D_temp
added += 1
self.C[i] = V_temp
self.II[i] = 1
self.P[i] = Q_temp
A_i = []
A_i.append(i)
self.A[i] = A_i
def __init__(self, workspace, disasm_view, parent=None):
super(QLinearViewer, self).__init__(parent)
self.workspace = workspace
self.disasm_view = disasm_view
self.objects = [] # Objects that will be painted
self.cfg = None
self.cfb = None
self._offset_to_addr = AVLTree()
self._addr_to_offset = AVLTree()
self._offset_to_object = AVLTree()
self._offset = 0
self._paint_start_offset = 0
self._linear_view = None # type: QLinearGraphicsView
self._disasms = {}
self._init_widgets()
setup_AVLTree = """
from __main__ import avl_build_delete, avl_build, avl_search
"""
setup_FastAVLTree = """
from __main__ import cavl_build_delete, cavl_build, cavl_search
"""
try:
fp = open('testkeys.txt')
keys = eval(fp.read())
fp.close()
except IOError:
print("create 'testkeys.txt' with profile_bintree.py\n")
sys.exit()
py_searchtree = AVLTree.from_keys(keys)
cy_searchtree = FastAVLTree.from_keys(keys)
def avl_build_delete():
tree = AVLTree.from_keys(keys)
for key in keys:
del tree[key]
def cavl_build_delete():
tree = FastAVLTree.from_keys(keys)
for key in keys:
del tree[key]
class RegionMap(object):
"""
Mostly used in SimAbstractMemory, RegionMap stores a series of mappings between concrete memory address ranges and
memory regions, like stack frames and heap regions.
"""
def __init__(self, is_stack):
"""
Constructor
:param is_stack: Whether this is a region map for stack frames or not. Different strategies apply for stack
regions.
"""
self.is_stack = is_stack
# An AVLTree, which maps stack addresses to region IDs
self._address_to_region_id = AVLTree()
# A dict, which maps region IDs to memory address ranges
self._region_id_to_address = {}
#
# Properties
#
def __repr__(self):
return "RegionMap<%s>" % ("S" if self.is_stack else "H")
@property
def is_empty(self):
return len(self._address_to_region_id) == 0
@property
def stack_base(self):
if not self.is_stack:
raise SimRegionMapError('Calling "stack_base" on a non-stack region map.')
return self._address_to_region_id.max_key()
@property
def region_ids(self):
return self._region_id_to_address.keys()
#
# Public methods
#
def copy(self):
r = RegionMap(is_stack=self.is_stack)
# A shallow copy should be enough, since we never modify any RegionDescriptor object in-place
if len(self._address_to_region_id) > 0:
# TODO: There is a bug in bintrees 2.0.2 that prevents us from copying a non-empty AVLTree object
# TODO: Consider submit a pull request
r._address_to_region_id = self._address_to_region_id.copy()
r._region_id_to_address = self._region_id_to_address.copy()
return r
def map(self, absolute_address, region_id, related_function_address=None):
"""
Add a mapping between an absolute address and a region ID. If this is a stack region map, all stack regions
beyond (lower than) this newly added regions will be discarded.
:param absolute_address: An absolute memory address.
:param region_id: ID of the memory region.
:param related_function_address: A related function address, mostly used for stack regions.
"""
if self.is_stack:
# Sanity check
if not region_id.startswith("stack_"):
raise SimRegionMapError('Received a non-stack memory ID "%d" in a stack region map' % region_id)
# Remove all stack regions that are lower than the one to add
while True:
try:
addr = self._address_to_region_id.floor_key(absolute_address)
descriptor = self._address_to_region_id[addr]
# Remove this mapping
del self._address_to_region_id[addr]
# Remove this region ID from the other mapping
del self._region_id_to_address[descriptor.region_id]
except KeyError:
break
else:
if absolute_address in self._address_to_region_id:
descriptor = self._address_to_region_id[absolute_address]
# Remove this mapping
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[descriptor.region_id]
# Add this new region mapping
desc = RegionDescriptor(region_id, absolute_address, related_function_address=related_function_address)
self._address_to_region_id[absolute_address] = desc
self._region_id_to_address[region_id] = desc
def unmap_by_address(self, absolute_address):
"""
Removes a mapping based on its absolute address.
:param absolute_address: An absolute address
"""
desc = self._address_to_region_id[absolute_address]
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[desc.region_id]
def absolutize(self, region_id, relative_address):
"""
Convert a relative address in some memory region to an absolute address.
:param region_id: The memory region ID
:param relative_address: The relative memory offset in that memory region
:return: An absolute address if converted, or an exception is raised when region id does not
exist.
"""
if region_id == "global":
# The global region always bases 0
return relative_address
if region_id not in self._region_id_to_address:
raise SimRegionMapError('Non-existent region ID "%s"' % region_id)
base_address = self._region_id_to_address[region_id].base_address
return base_address + relative_address
def relativize(self, absolute_address, target_region_id=None):
"""
Convert an absolute address to the memory offset in a memory region.
Note that if an address belongs to heap region is passed in to a stack region map, it will be converted to an
offset included in the closest stack frame, and vice versa for passing a stack address to a heap region.
Therefore you should only pass in address that belongs to the same category (stack or non-stack) of this region
map.
:param absolute_address: An absolute memory address
:return: A tuple of the closest region ID, the relative offset, and the related function
address.
"""
if target_region_id is None:
if self.is_stack:
# Get the base address of the stack frame it belongs to
base_address = self._address_to_region_id.ceiling_key(absolute_address)
else:
try:
base_address = self._address_to_region_id.floor_key(absolute_address)
except KeyError:
# Not found. It belongs to the global region then.
return "global", absolute_address, None
descriptor = self._address_to_region_id[base_address]
else:
if target_region_id == "global":
# Just return the absolute address
return "global", absolute_address, None
if target_region_id not in self._region_id_to_address:
raise SimRegionMapError('Trying to relativize to a non-existent region "%s"' % target_region_id)
descriptor = self._region_id_to_address[target_region_id]
base_address = descriptor.base_address
return descriptor.region_id, absolute_address - base_address, descriptor.related_function_address
def __init__(self, tree=None):
self._storage = AVLTree() if tree is None else tree # type: AVLTree
def __init__(self, segments, x):
self.segments = segments
self.x = x
self.tree = AVLTree()
class SortedTypedList(object):
"""This data structure allows to sort and maintain sorted
a list of objects inheriting from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
It uses an AVLTree proposed by :mod:`bintrees` to represent elements in the list.
:TODO: The inner object __tree stores the given element using an AVLTree. Thus
exposing new methods only requires the creation of a wrapper method in this class.
>>> from netzob.all import *
>>> from netzob.Common.Utils.SortedTypedList import SortedTypedList
>>> msg1 = RawMessage(b"msg1", date=25.0)
>>> msg2 = RawMessage(b"msg2", date=2.0)
>>> msg3 = RawMessage(b"msg3", date=1456487548.0)
>>> l = SortedTypedList(RawMessage, [msg2, msg3, msg1])
>>> print(list(l.values())[0])
[0;32m[2.0 [0;m[1;32mNone[1;m[0;32m->[0;m[1;32mNone[1;m[0;32m][0;m 'msg2'
>>> msg4 = RawMessage(b"msg4", date=145548.0)
>>> l.add(msg4)
>>> msg5 = RawMessage(b"msg5", date=14.0)
>>> msg6 = RawMessage(b"msg6", date=1745645548.0)
>>> l.addAll([msg5, msg6])
>>> print(list(l.values())[5])
[0;32m[1745645548.0 [0;m[1;32mNone[1;m[0;32m->[0;m[1;32mNone[1;m[0;32m][0;m 'msg6'
>>> print(len(l))
6
"""
def __init__(self, membersTypes, elements=None):
self.membersTypes = membersTypes
self.__treePriorities = AVLTree()
self.__mapMessages = dict()
if elements is not None and len(elements) > 0:
self._extend(elements)
def add(self, element):
"""Insert in the proper place the specified element.
:type: any object that comply with the typed of the current list and inherits from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
:raises: a TypeError if element is None or if its type doesn't comply with
the definition of the list.
"""
if element is None:
raise TypeError("Element cannot be None")
self._extend([element])
def addAll(self, elements):
"""Insert in their proper place all the specified element.
:type: a list of any object that comply with the typed of the current list and inherits from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
:raises: a TypeError if element is None or if its type doesn't comply with
the definition of the list.
"""
if elements is None:
raise TypeError("Element cannot be None")
self._extend(elements)
def values(self):
"""Return a list sorted with the values of the current SortedTypedList.
:warning: modifying this list has no effect on the SortedTypedList.
:rtype: :mod:list
"""
l = []
for x in list(self.__treePriorities.keys()):
l.extend(self.__mapMessages[x])
return l
def clear(self):
"""remove all items from the list.
It's a O(n) operation"""
self.__treePriorities.clear()
def _extend(self, elements):
"""Add all the elements in the current list.
:parameter elements: a list of :class:`netzob.Common.Utils.SortableObject.SortableObject` to insert.
:raises: TypeError if something is wrong with the given elements
"""
for e in elements:
self._check(e)
d = dict()
for e in elements:
d[e.priority()] = None
if e.priority() in self.__mapMessages:
self.__mapMessages[e.priority()].append(e)
else:
self.__mapMessages[e.priority()] = [e]
self.__treePriorities.update(d)
def _check(self, v):
if not isinstance(v, self.membersTypes):
raise TypeError(
"Invalid type for argument, expecting: {0}, received : {1}".
format(self.membersTypes, v.__class__.__name__))
if not isinstance(v, SortableObject):
raise TypeError(
"Objects inserted in a SortedTypedList must inherits from SortableObject class"
)
def __len__(self):
"""Returns the number of elements in the sorted list which takes
O(1) operation :)"""
return len(self.__treePriorities)
def __str__(self):
return ', \n'.join([str(v) for v in list(self.values())])
def __repr__(self):
return repr(str(self))
def __iter__(self):
"""SortedTypedList is an iterable over its values (and not its keys)."""
return list(self.__treePriorities.values()).__iter__()
class KeyedRegion(object):
"""
KeyedRegion keeps a mapping between stack offsets and all variables covering that offset. It assumes no variable in
this region overlap with another variable in this region.
Registers and function frames can all be viewed as a keyed region.
"""
def __init__(self, tree=None):
self._storage = AVLTree() if tree is None else tree # type: AVLTree
def __contains__(self, offset):
"""
Test if there is at least one varaible covering the given offset.
:param offset:
:return:
"""
try: base_offset, item = self._storage.floor_item(offset) #pylint:disable=unused-variable
except KeyError: return False
if item.includes(offset):
return True
return False
def __len__(self):
return len(self._storage)
def __iter__(self):
for _, item in self._storage.items():
yield item
def __eq__(self, other):
if set(self._storage.keys()) != set(other._storage.keys()):
return False
for k, v in self._storage.iter_items():
if v != other._storage[k]:
return False
return True
def copy(self):
if not self._storage:
return KeyedRegion()
kr = KeyedRegion()
for key, ro in self._storage.iter_items():
kr._storage[key] = ro.copy()
return kr
def merge(self, other, make_phi_func=None):
"""
Merge another KeyedRegion into this KeyedRegion.
:param KeyedRegion other: The other instance to merge with.
:return: None
"""
# TODO: is the current solution not optimal enough?
for _, item in other._storage.iter_items(): # type: RegionObject
for loc_and_var in item.objects:
self.__store(loc_and_var, overwrite=False, make_phi_func=make_phi_func)
return self
def dbg_repr(self):
"""
Get a debugging representation of this keyed region.
:return: A string of debugging output.
"""
keys = self._storage.keys()
offset_to_vars = { }
for key in sorted(keys):
ro = self._storage[key]
variables = [ obj.variable for obj in ro.objects ]
offset_to_vars[key] = variables
s = [ ]
for offset, variables in offset_to_vars.iteritems():
s.append("Offset %#x: %s" % (offset, variables))
return "\n".join(s)
def add_variable(self, start, variable):
"""
Add a variable to this region at the given offset.
:param int start:
:param SimVariable variable:
:return: None
"""
self._store(start, variable, overwrite=False)
def set_variable(self, start, variable):
"""
Add a variable to this region at the given offset, and remove all other variables that are fully covered by
this variable.
:param int start:
:param SimVariable variable:
:return: None
"""
self._store(start, variable, overwrite=True)
def get_base_addr(self, addr):
"""
Get the base offset (the key we are using to index variables covering the given offset) of a specific offset.
:param int addr:
:return:
:rtype: int or None
"""
try: base_addr, item = self._storage.floor_item(addr)
except KeyError: return None
if item.includes(addr):
return base_addr
return None
def get_variables_by_offset(self, start):
"""
Find variables covering the given region offset.
:param int start:
:return: A list of stack variables.
:rtype: set
"""
try: base_addr = self._storage.floor_key(start)
except KeyError: return [ ]
item = self._storage[base_addr] # type: RegionObject
if item.includes(start):
return item.variables
return [ ]
#
# Private methods
#
def _store(self, start, variable, overwrite=False):
"""
Store a variable into the storage.
:param int start: The beginning address of the variable.
:param variable: The variable to store.
:param bool overwrite: Whether existing variables should be overwritten or not.
:return: None
"""
loc_and_var = LocationAndVariable(start, variable)
self.__store(loc_and_var, overwrite=overwrite)
def __store(self, loc_and_var, overwrite=False, make_phi_func=None):
"""
Store a variable into the storage.
:param LocationAndVariable loc_and_var: The descriptor describing start address and the variable.
:param bool overwrite: Whether existing variables should be overwritten or not.
:return: None
"""
start = loc_and_var.start
variable = loc_and_var.variable
variable_size = variable.size if variable.size is not None else 1
end = start + variable_size
# region items in the middle
overlapping_items = list(self._storage.item_slice(start, end))
# is there a region item that begins before the start and overlaps with this variable?
try:
floor_key, floor_item = self._storage.floor_item(start) # type: RegionObject
if floor_item.includes(start):
item = (floor_key, floor_item)
if item not in overlapping_items:
# insert it into the beginningq
overlapping_items.insert(0, (floor_key, floor_item))
except KeyError:
# no there isn't
pass
# scan through the entire list of region items, split existing regions and insert new regions as needed
to_update = { start: RegionObject(start, variable_size, { loc_and_var }) }
last_end = start
for _, item in overlapping_items: # type: RegionObject
if item.start < start:
# we need to break this item into two
a, b = item.split(start)
if overwrite:
b.set_object(loc_and_var)
else:
self._add_object_or_make_phi(b, loc_and_var, make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
elif item.start > last_end:
# there is a gap between the last item and the current item
# fill in the gap
new_item = RegionObject(last_end, item.start - last_end, { loc_and_var })
to_update[new_item.start] = new_item
last_end = new_item.end
elif item.end > end:
# we need to split this item into two
a, b = item.split(end)
if overwrite:
a.set_object(loc_and_var)
else:
self._add_object_or_make_phi(a, loc_and_var, make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
else:
if overwrite:
item.set_object(loc_and_var)
else:
self._add_object_or_make_phi(item, loc_and_var, make_phi_func=make_phi_func)
to_update[loc_and_var.start] = item
self._storage.update(to_update)
def _is_overlapping(self, start, variable):
if variable.size is not None:
# make sure this variable does not overlap with any other variable
end = start + variable.size
try:
prev_offset = self._storage.floor_key(end - 1)
except KeyError:
prev_offset = None
if prev_offset is not None:
if start <= prev_offset < end:
return True
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if start < prev_offset + prev_item_size < end:
return True
else:
try:
prev_offset = self._storage.floor_key(start)
except KeyError:
prev_offset = None
if prev_offset is not None:
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if prev_offset <= start < prev_offset + prev_item_size:
return True
return False
def _add_object_or_make_phi(self, item, loc_and_var, make_phi_func=None): #pylint:disable=no-self-use
if not make_phi_func or len({loc_and_var.variable} | item.variables) == 1:
item.add_object(loc_and_var)
else:
# make a phi node
item.set_object(LocationAndVariable(loc_and_var.start,
make_phi_func(loc_and_var.variable, *item.variables)
)
)
def __init__(self):
self._history = AVLTree()
def separateFilesToClusters(self, sDirectory, size_threshold):
start = 0
for root, dirs, files in os.walk(sDirectory):
start += len(dirs)
for i in range(self.N):
if self.II[i] == 1:
if len(self.A[i]) >= size_threshold:
sDst = join(sDirectory, str(start))
os.makedirs(sDst)
for j in range(len(self.A[i])):
shutil.copy(self.fileList[self.A[i][j]], sDst)
start += 1
if __name__ == '__main__':
t = AVLTree()
d1 = Distance(1.12, 1)
d2 = Distance(0.9, 2)
d3 = Distance(0.9, 3)
t[d1] = d1
t[d2] = d2
t[d3] = d3
del t[d3]
print len(t)
def __init__(self, membersTypes, elements=None):
self.membersTypes = membersTypes
self.__treePriorities = AVLTree()
self.__mapMessages = dict()
if elements is not None and len(elements) > 0:
self._extend(elements)
def __init__(self, entries, shared=None):
self.sem_lock = Semaphore(value=1)
self.psbt = AVLTree(entries)
self.shared = shared
def ABguilhermeberger(reds, blus): """ Input: red and blue segments with no red/red or blue/blue crossings, and no endpoints inside segments Output: For each flag, the segment above and below of the same and the opposite color. """ red_bot_sentinel = ColoredSegment(Point(MIN_COORD-1, MIN_COORD-1), Point(MAX_COORD+1, MIN_COORD-1), 'Red') red_top_sentinel = ColoredSegment(Point(MIN_COORD-1, MAX_COORD+1), Point(MAX_COORD+1, MAX_COORD+1), 'Red') red_bot_sentinel.num = 0 red_top_sentinel.num = len(reds)+1 blu_bot_sentinel = ColoredSegment(Point(MIN_COORD-1, MIN_COORD-1), Point(MAX_COORD+1, MIN_COORD-1), 'Blue') blu_top_sentinel = ColoredSegment(Point(MIN_COORD-1, MAX_COORD+1), Point(MAX_COORD+1, MAX_COORD+1), 'Blue') blu_bot_sentinel.num = 0 blu_top_sentinel.num = len(blus)+1 blu_start_flags = [seg.a for seg in blus] blu_termi_flags = [seg.b for seg in blus] red_start_flags = [seg.a for seg in reds] red_termi_flags = [seg.b for seg in reds] red_flags = red_start_flags + red_termi_flags blu_flags = blu_start_flags + blu_termi_flags # Lists ordered by the flag sorting conditions defined on SI3 red_flags = sorted(red_flags) blu_flags = sorted(blu_flags) flags = sorted(red_flags + blu_flags) # This list is ordered by the Y coordinate of the Start flag red_sweep = AVLTree() red_sweep.insert(red_bot_sentinel, red_bot_sentinel) red_sweep.insert(red_top_sentinel, red_top_sentinel) blu_sweep = AVLTree() blu_sweep.insert(blu_bot_sentinel, blu_bot_sentinel) blu_sweep.insert(blu_top_sentinel, blu_top_sentinel) def get_same_sweep(flag): if flag.color() == 'Blue': return blu_sweep else: return red_sweep def get_other_sweep(flag): if flag.color() == 'Red': return blu_sweep else: return red_sweep # By iterating through `flags`, we are effectively sweeping a vertical line from left to right for flag in flags: segment = flag.parent same_sweep = get_same_sweep(flag) other_sweep = get_other_sweep(flag) if flag.kind == 'Start': same_sweep.insert(segment, segment) flag.sb = same_sweep.prev_item(segment)[0].num flag.sa = same_sweep.succ_item(segment)[0].num else: flag.sb = segment.num flag.sa = segment.num same_sweep.remove(segment) other_sweep.insert(segment, segment) flag.ob = other_sweep.prev_item(segment)[0].num flag.oa = other_sweep.succ_item(segment)[0].num other_sweep.remove(segment) return red_flags, blu_flags
class RecursiveLambdaFunctionGrowth(object):
def __init__(self):
self.lambda_comp_tree=AVLTree()
self.index_tree=BinaryTree()
self.word_list=[]
self.word_dict={}
self.index_dict={}
self.index_list=[]
self.lambda_expression=[]
self.lambda_composition=""
self.graph_tensor_neuron_network_intrinsic_merit=1.0
self.entropy=10000000000.0
self.conceptnet=ConceptNet5Client()
#self.Similarity="ConceptNet"
self.Similarity="WordNet"
self.ClosedPaths=True
self.dictionary=PyDictionary()
def get_next_tree_traversal_id(self,x,y):
if y-x == 1 or x-y == 1:
return 1
print "x,y:",x,y
self.index_list.append((x+y)/2)
self.get_next_tree_traversal_id(x,(x+y)/2)
self.get_next_tree_traversal_id((x+y)/2,y)
def build_lambda_expression(self,key,value):
#print value,
self.lambda_expression.append(value)
def build_lambda_comp_tree(self,k,v):
if k < len(self.word_list):
self.word_dict[k]=self.word_list[k]
def return_next(self,k,v):
return (k,v)
def grow_lambda_function2(self, wordlist):
self.word_list=wordlist
self.word_dict={}
cnt=0
while cnt < len(self.word_list):
self.index_dict[cnt]=cnt
cnt+=1
self.index_tree=BinaryTree(self.index_dict)
self.index_tree.foreach(self.build_lambda_comp_tree,0)
self.lambda_comp_tree=AVLTree(self.word_dict)
print "==========================================================================="
print "Lambda Composition AVL Tree (inorder traversed) is the original text itself:"
print "==========================================================================="
self.lambda_expression=[]
self.lambda_comp_tree.foreach(self.build_lambda_expression, 0)
print self.lambda_expression
print "==========================================================================="
print "Lambda Composition AVL Tree (postorder traversed - Postfix expression):"
print "Every parenthesis has two operands,operated by function outside:"
print "==============================================================="
self.lambda_expression=[]
self.lambda_comp_tree.foreach(self.build_lambda_expression, 1)
self.lambda_composition=[]
cnt=0
per_random_walk_graph_tensor_neuron_network_intrinsic_merit = 0
#recursively evaluate the Graph Tensor Neuron Network for random walk composition tree bottom up as Graph Neural Network
#having Tensor Neuron activations for each subtree.
while len(self.lambda_expression) > 2 :
operand2=self.lambda_expression.pop()
operand1=self.lambda_expression.pop()
function=self.lambda_expression.pop()
subtree_graph_tensor_neuron_network_wght = self.subtree_graph_tensor_neuron_network_weight(operand1, function, operand2)
self.graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
per_random_walk_graph_tensor_neuron_network_intrinsic_merit += subtree_graph_tensor_neuron_network_wght
self.lambda_composition="("+function+"("+operand1+","+operand2+"))"
self.lambda_expression.append(self.lambda_composition)
cnt+=1
if len(self.lambda_expression) > 1:
return (self.lambda_expression[0] + "("+self.lambda_expression[1]+")", per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
else:
return (self.lambda_expression[0], per_random_walk_graph_tensor_neuron_network_intrinsic_merit)
def grow_lambda_function1(self):
text=open("RecursiveLambdaFunctionGrowth.txt","r")
word_dict={}
index_dict={}
words_evaluated=0
word_list=text.read().split()
for cnt in range(1,len(word_list)):
index_dict[cnt-1] = len(word_list)/cnt
index_tree=AVLTree(index_dict)
print "Index AVL Tree:", repr(index_tree)
#index_tree.foreach(print_node,1)
try:
while words_evaluated < len(word_list):
#word_dict[words_evaluated]=word_list[random.randint(0,len(word_list)-1)]
#print word_list[index_tree.pop_min()[0]]
word_dict[words_evaluated]=word_list[index_tree.pop_min()[0]]
words_evaluated+=1
except:
pass
self.lambda_comp_tree=AVLTree(word_dict)
print "Lambda Composition AVL Tree:"
self.lambda_comp_tree.foreach(print_node)
iteration=0
while iteration < len(word_list):
k=self.lambda_comp_tree.get(iteration)
print "k:",k
try:
prev=self.lambda_comp_tree.prev_key(iteration)
prevk=self.lambda_comp_tree.get(prev)
print "prevk:",prevk
except:
pass
try:
succ=self.lambda_comp_tree.succ_key(iteration)
succk=self.lambda_comp_tree.get(succ)
print "succk:",succk
except:
pass
iteration+=1
def get_tensor_neuron_potential_for_relation(self,synset_vertex,synset_r):
smt=0.0
similarity=0.0
for s1, s2 in product(synset_vertex, synset_r):
if self.Similarity=="WordNet":
smt=wn.wup_similarity(s1,s2)
if self.Similarity=="ConceptNet":
s1_lemma_names=s1.lemma_names()
s2_lemma_names=s2.lemma_names()
smt=self.conceptnet.conceptnet_distance(s1_lemma_names[0], s2_lemma_names[0])
#print "similarity=",smt
if smt > similarity and smt != 1.0:
similarity = float(smt)
return similarity
def subtree_graph_tensor_neuron_network_weight(self, e1, r, e2):
#relation_tensor_neuron_potential=self.get_tensor_neuron_potential_for_relation(r)
if e1[0]=="(":
e1_parsed=e1.split("(")
#print "operand1:", e1_parsed[1]
synset_e1 = wn.synsets(e1_parsed[1])
else:
synset_e1 = wn.synsets(e1)
#print "operand1:", e1
#print "Relation: ",r
synset_r = wn.synsets(r)
if e2[0]=="(":
e2_parsed=e2.split("(")
#print "operand2:", e2_parsed[1]
synset_e2 = wn.synsets(e2_parsed[1])
else:
#print "operand2:", e2
synset_e2 = wn.synsets(e2)
similarity1 = 0.0
similarity2 = 0.0
#Children of each subtree are the Tensor Neuron inputs to the subtree root
#Each subtree is evaluated as a graph neural network with weights for
#each neural input to the subtree root. WordNet similarity is computed
#between each child and subtree root and is presently assumed as Tensor Neuron
#relation potential for the lack of better metric to measure word-word EEG potential.
#If a dataset for tensor neuron potential
#is available, it has to to be looked-up and numeric
#potential has to be returned from here.
similarity1 = self.get_tensor_neuron_potential_for_relation(synset_e1,synset_r)
similarity2 = self.get_tensor_neuron_potential_for_relation(synset_e2,synset_r)
if similarity1 == 0.0:
similarity1 = 1.0
if similarity2 == 0.0:
similarity2 = 1.0
weight1=0.5
weight2=0.5
bias=0.1
#Finally a neuron activation function (simple 1-dimensional tensor) is computed and
#returned to the subtree root for next level.
return (weight1*similarity1 + weight2*similarity2 + bias)
def randomwalk_lambda_function_composition_tree(self,randomwalk):
randomwalk_lambdacomposition=self.grow_lambda_function2(randomwalk)
return randomwalk_lambdacomposition
def create_summary(self,text,corenumber=3,pathsimilarity=0.8,graphtraversedsummary=False,shortestpath=True):
if graphtraversedsummary==True:
definitiongraph=RecursiveGlossOverlap_Classifier.RecursiveGlossOverlapGraph(text)
#This has to be replaced by a Hypergraph Transversal but NetworkX does not have Hypergraphs yet.
#Hence approximating the transversal with a k-core which is the Graph counterpart of
#Hypergraph transversal. Other measures create a summary too : Vertex Cover is NP-hard while Edge Cover is Polynomial Time.
richclubcoeff=nx.rich_club_coefficient(definitiongraph.to_undirected())
print "Rich Club Coefficient of the Recursive Gloss Overlap Definition Graph:",richclubcoeff
kcore=nx.k_core(definitiongraph,corenumber)
print "Text Summarized by k-core(subgraph having vertices of degree atleast k) on the Recursive Gloss Overlap graph:"
print "=========================="
print "Dense subgraph edges:"
print "=========================="
print kcore.edges()
print "=========================="
if shortestpath == False:
for e in kcore.edges():
for s1 in wn.synsets(e[0]):
for s2 in wn.synsets(e[1]):
if s1.path_similarity(s2) > pathsimilarity:
lowestcommonhypernyms=s1.lowest_common_hypernyms(s2)
for l in lowestcommonhypernyms:
for ln in l.lemma_names():
print e[0]," and ",e[1]," are ",ln,".",
else:
#Following is the slightly modified version of shortest_path_distance() function
#in NLTK wordnet - traverses the synset path between 2 synsets instead of distance
summary={}
intermediates=[]
for e in kcore.edges():
for s1 in wn.synsets(e[0]):
for s2 in wn.synsets(e[1]):
s1dict = s1._shortest_hypernym_paths(False)
s2dict = s2._shortest_hypernym_paths(False)
s2dictkeys=s2dict.keys()
for s,d in s1dict.iteritems():
if s in s2dictkeys:
slemmanames=s.lemma_names()
if slemmanames[0] not in intermediates:
intermediates.append(slemmanames[0])
if len(intermediates) > 3:
sentence1=e[0] + " is a " + intermediates[0]
summary[sentence1]=self.relevance_to_text(sentence1,text)
for i in xrange(len(intermediates)-2):
sentence2= intermediates[i] + " is a " + intermediates[i+1] + "."
if sentence2 not in summary:
summary[sentence2]=self.relevance_to_text(sentence2,text)
sentence3=intermediates[len(intermediates)-1] + " is a " + e[1]
summary[sentence3]=self.relevance_to_text(sentence3,text)
intermediates=[]
sorted_summary=sorted(summary,key=operator.itemgetter(1), reverse=True)
print "==================================================================="
print "Sorted summary created from k-core dense subgraph of text RGO"
print "==================================================================="
for s in sorted_summary:
print s,
return (sorted_summary, len(sorted_summary))
else:
definitiongraph_merit=RecursiveGlossOverlap_Classifier.RecursiveGlossOverlapGraph(text)
definitiongraph=definitiongraph_merit[0]
richclubcoeff=nx.rich_club_coefficient(definitiongraph.to_undirected(),normalized=False)
print "Rich Club Coefficient of the Recursive Gloss Overlap Definition Graph:",richclubcoeff
textsentences=text.split(".")
lensummary=0
summary=[]
definitiongraphclasses=RecursiveGlossOverlap_Classifier.RecursiveGlossOverlap_Classify(text)
print "Text Summarized based on the Recursive Gloss Overlap graph classes the text belongs to:"
prominentclasses=int(len(definitiongraphclasses[0])/2)
print "Total number of classes:",len(definitiongraphclasses[0])
print "Number of prominent classes:",prominentclasses
for c in definitiongraphclasses[0][:prominentclasses]:
if len(summary) > len(textsentences) * 0.5:
return (summary,lensummary)
for s in textsentences:
classsynsets=wn.synsets(c[0])
for classsynset in classsynsets:
if self.relevance_to_text(classsynset.definition(), s) > 0.41:
if s not in summary:
summary.append(s)
lensummary += len(s)
print s,
return (summary,lensummary)
def relevance_to_text(self, sentence, text):
#Ratcliff/Obershelp gestalt string pattern matching
textset=set(text.split("."))
relevancescore=0.0
for t in textset:
rs=difflib.SequenceMatcher(None,sentence,t).ratio()
relevancescore=max(rs,relevancescore)
return relevancescore
def instrument_relations(self, rw_words_list):
word_list_len=len(rw_words_list)
instrumented_rw_words_list=[]
if word_list_len==2:
path=path_between(rw_words_list[0], rw_words_list[1])
for p in path:
instrumented_rw_words_list.append(p)
else:
for n in range(0,word_list_len-2):
path=path_between(rw_words_list[n], rw_words_list[n+1])
for p in path:
instrumented_rw_words_list.append(p)
if len(instrumented_rw_words_list) > 0:
return instrumented_rw_words_list
else:
return rw_words_list
def grow_lambda_function3(self,text,level=3):
stpairs=[]
maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit=("",0.0)
definitiongraph_merit=RecursiveGlossOverlap_Classifier.RecursiveGlossOverlapGraph(text,level)
definitiongraph=definitiongraph_merit[0]
sentiment=SentimentAnalyzer.SentimentAnalysis_RGO_Belief_Propagation_MarkovRandomFields(definitiongraph)
apsp=nx.all_pairs_shortest_path(definitiongraph)
for a in definitiongraph.nodes():
for b in definitiongraph.nodes():
stpairs.append((a,b))
rw_ct=""
if self.ClosedPaths==False:
for k,v in stpairs:
try:
print "==================================================================="
print "Random Walk between :",k," and ",v,":",apsp[k][v]
instrumented_apspkv=self.instrument_relations(apsp[k][v])
rw_ct=self.randomwalk_lambda_function_composition_tree(instrumented_apspkv)
print "Random Walk Composition Tree for walk between :",k," and ",v,":",rw_ct
print "maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit=",maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit
print "==================================================================="
if rw_ct[1] > maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit[1]:
maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit=rw_ct
except KeyError:
pass
rw_ct=""
if self.ClosedPaths==True:
allsimplecycles=nx.simple_cycles(definitiongraph)
#allsimplecycles=nx.cycle_basis(definitiongraph)
number_of_cycles=0
for cycle in allsimplecycles:
number_of_cycles += 1
if number_of_cycles > 500:
break
try:
print "==================================================================="
print "Cycle :",cycle
instrumented_cycle=self.instrument_relations(cycle)
print "instrumented_cycle:",instrumented_cycle
rw_ct=self.randomwalk_lambda_function_composition_tree(instrumented_cycle)
print "Cycle Composition Tree for this cycle :",rw_ct
print "maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit=",maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit
print "==================================================================="
if rw_ct[1] > maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit[1]:
maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit=rw_ct
except KeyError:
pass
rw_ct=""
intrinsic_merit_dict={}
print "grow_lambda_function3(): Graph Tensor Neuron Network Intrinsic Merit for this text:",self.graph_tensor_neuron_network_intrinsic_merit
print "grow_lambda_function3(): Machine Translation Example - English to Kannada:"
self.machine_translation(definitiongraph, "kn")
self.korner_entropy(definitiongraph)
print "grow_lambda_function3(): Korner Entropy Intrinsic Merit for this text:",self.entropy
density = self.density(definitiongraph)
print "grow_lambda_function3(): Graph Density (Regularity Lemma):",density
bose_einstein_intrinsic_fitness=self.bose_einstein_intrinsic_fitness(definitiongraph)
print "grow_lambda_function3(): Bose-Einstein Intrinsic Fitness:",bose_einstein_intrinsic_fitness
print "grow_lambda_function3(): Maximum Per Random Walk Graph Tensor Neuron Network Intrinsic Merit :",maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit
print "grow_lambda_function3(): Recursive Gloss Overlap Classifier classes for text:",RecursiveGlossOverlap_Classifier.RecursiveGlossOverlap_Classify(text)
intrinsic_merit_dict["graph_tensor_neuron_network_intrinsic_merit"]=self.graph_tensor_neuron_network_intrinsic_merit
intrinsic_merit_dict["maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit"]=maximum_per_random_walk_graph_tensor_neuron_network_intrinsic_merit
intrinsic_merit_dict["korner_entropy"]=self.entropy
intrinsic_merit_dict["density"]=density
intrinsic_merit_dict["bose_einstein_intrinsic_fitness"]=bose_einstein_intrinsic_fitness
intrinsic_merit_dict["recursive_gloss_overlap_intrinsic_merit"]=definitiongraph_merit[1]
intrinsic_merit_dict["empath_sentiment"]=sentiment
write_dot(definitiongraph,"RecursiveLambdaFunctionGrowth.dot")
self.graph_tensor_neuron_network_intrinsic_merit=1.0
print "intrinsic_merit_dict:",intrinsic_merit_dict
return intrinsic_merit_dict
def machine_translation(self, definitiongraph, languagecode):
nodes=definitiongraph.nodes()
edges=definitiongraph.edges()
translationgraph=nx.DiGraph()
for k, v in edges:
ktrans=self.dictionary.translate(k,languagecode)
vtrans=self.dictionary.translate(v,languagecode)
print "k=",k,",v=",v,",ktrans=",ktrans,",vtrans=",vtrans
translationgraph.add_edge(ktrans, vtrans)
translationgraph.add_edge(vtrans, ktrans)
print "TextGraph Translated to ",languagecode,":",translationgraph
#KornerEntropy(G) = minimum [- sum_v_in_V(G) {1/|V(G)| * log(Pr[v in Y])}] for each independent set Y
def korner_entropy(self, definitiongraph):
nodes=definitiongraph.nodes()
stable_sets=[]
for v in nodes:
stable_sets.append(nx.maximal_independent_set(definitiongraph.to_undirected(),[v]))
print "korner_entropy(): Stable Independent Sets:",stable_sets
entropy=0.0
prob_v_in_stableset=0.0
for v in nodes:
for s in stable_sets:
if v in s:
prob_v_in_stableset=math.log(0.999999)
else:
prob_v_in_stableset=math.log(0.000001)
entropy += (-1.0) * float(1.0/len(nodes)) * prob_v_in_stableset
if entropy < self.entropy:
self.entropy = entropy
entropy=0.0
return self.entropy
#Graph Density - Regularity Lemma
def density(self, definitiongraph):
dty=nx.classes.function.density(definitiongraph)
return dty
#Bose-Einstein Bianconi intrinsic fitness
def bose_einstein_intrinsic_fitness(self, definitiongraph):
#Bose-Einstein fitness presently assumes energy of a document vertex in a link graph to be
#the entropy or extent of chaos in the definition graph of document text
#This has to be replaced by a more suitable fitness measure
#Bose-Einstein Condensation function value is hardcoded
entropy = self.korner_entropy(definitiongraph)
becf = 0.3
bei_fitness = math.pow(2, -1 * becf * entropy)
return bei_fitness
def __init__(self): self.tree = AVLTree()
def __init__(self, membersTypes, elements=None):
self.membersTypes = membersTypes
self.__treePriorities = AVLTree()
self.__mapMessages = dict()
if elements is not None and len(elements) > 0:
self._extend(elements)
def avl_build_delete():
tree = AVLTree.from_keys(keys)
for key in keys:
del tree[key]
setup_AVLTree = """
from __main__ import avl_build_delete, avl_build, avl_search
"""
setup_FastAVLTree = """
from __main__ import cavl_build_delete, cavl_build, cavl_search
"""
try:
fp = open('testkeys.txt')
keys = eval(fp.read())
fp.close()
except IOError:
print("create 'testkeys.txt' with profile_bintree.py\n")
sys.exit()
py_searchtree = AVLTree.from_keys(keys)
cy_searchtree = FastAVLTree.from_keys(keys)
def avl_build_delete():
tree = AVLTree.from_keys(keys)
for key in keys:
del tree[key]
def cavl_build_delete():
tree = FastAVLTree.from_keys(keys)
for key in keys:
del tree[key]
def avl_build():
tree = AVLTree.from_keys(keys)
def avl_build_delete():
tree = AVLTree.from_keys(keys)
for key in keys:
del tree[key]
from bintrees import AVLTree tree = AVLTree() tree.insert(4, "hehe4") tree.insert(2, "hehe2") tree.insert(6, "hehe6") tree.insert(1, "hehe1") tree.insert(7, "hehe7") print(tree)
def avl_build():
tree = AVLTree.from_keys(keys)
class State(object):
def __init__(self, segments, x):
self.segments = segments
self.x = x
self.tree = AVLTree()
def segment_crossing_point(self, segment_id):
segment = self.segments[segment_id.label]
segment_dx = segment.x2 - segment.x1
sweep_dx = self.x - segment.x1
segment_dy = segment.y2 - segment.y1
sweep_dy = segment_dy * (sweep_dx / segment_dx)
return segment.y1 + sweep_dy
def insert(self, new_segment):
""" Returns new pair of neighbours """
new_neighbours = []
self.tree.insert(new_segment, new_segment)
try:
new_neighbours.append((self.tree.prev_item(new_segment)[0], new_segment))
except KeyError:
pass
try:
new_neighbours.append((self.tree.succ_item(new_segment)[0], new_segment))
except KeyError:
pass
return new_neighbours
def delete(self, segment):
""" Returns new pair of neighbours """
new_neighbours = []
predecessor = None
successor = None
try:
predecessor = self.tree.prev_item(segment)[0]
except KeyError:
pass
try:
successor = self.tree.succ_item(segment)[0]
except KeyError:
pass
if predecessor and successor:
# make predecessor and successor new neighbours
new_neighbours.append((predecessor, successor))
self.tree.remove(segment)
return new_neighbours
def swap(self, segment1, segment2):
""" Returns new pair of neighbours """
pos_x = self.x
self.x = pos_x - epsilon
new_neighbours1 = self.delete(segment1)
self.x = pos_x + epsilon
new_neighbours2 = self.insert(segment1)
self.x = pos_x
all_neighbours = new_neighbours1 + new_neighbours2
new_neighbours = []
for neighbours in all_neighbours:
if neighbours != (segment1, segment2) and neighbours != (segment2, segment1):
new_neighbours.append(neighbours)
return new_neighbours
def get_intersections(spheres: List[Sphere]):
start_points_map = {}
end_points_map = {}
for index, sphere in enumerate(spheres):
start_points_map[(sphere.center.x - sphere.radius,
sphere.center.y)] = index
end_points_map[(sphere.center.x + sphere.radius,
sphere.center.y)] = index
event_points = [x for x in start_points_map.keys()]
event_points.extend(x for x in end_points_map.keys())
event_points.sort()
start_points = sorted(start_points_map.keys(), reverse=True)
end_points = sorted(end_points_map.keys(), reverse=True)
tree = AVL()
s = set()
while len(start_points) > 0 and len(end_points) > 0:
if start_points[-1] <= end_points[-1]:
sphere = spheres[start_points_map[start_points[-1]]]
tree.insert(sphere.center, sphere)
try:
key = sphere.center
while True:
prev = tree.prev_item(key)[1]
if prev.intersects(sphere):
# print('Intersection found: {} and {}'.format(prev, sphere))
s.add(prev.center.to_tuple())
s.add(sphere.center.to_tuple())
key = prev.center
except KeyError:
pass
try:
key = sphere.center
while True:
prev = tree.succ_item(key)[1]
if prev.intersects(sphere):
# print('Intersection found: {} and {}'.format(prev, sphere))
s.add(prev.center.to_tuple())
s.add(sphere.center.to_tuple())
key = prev.center
except KeyError:
pass
start_points.pop()
else:
tree.remove(spheres[end_points_map[end_points[-1]]].center)
end_points.pop()
while len(end_points) > 0:
tree.remove(spheres[end_points_map[end_points[-1]]].center)
end_points.pop()
return s
class Index():
# balanced trees to store index references (name -> ID), each letter of alphabet and digit have one tree
hash_index_tree = {}
# tree for storing objects by ID
objects = AVLTree()
def __init__(self, index_name):
if index_name != None:
# initialise index from folder
self.load_persistant_index(index_name)
else:
# initialise new index
self.inilialise()
def inilialise(self):
hashKeys = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','x','y','z','0','1','2','3','4','5','6','7','8','9','+']
#for each key create separate balanced tree
for k in hashKeys:
self.hash_index_tree[k] = AVLTree()
# append object to index
def index_object(self, obj):
# store whole object in objec tree
self.append_to_objects(obj)
# store index reference to object for each name and each word in name of object
object_id = obj.get_id()
names = obj.get_names()
# foreach names
for n in names.keys():
exploded_name = names[n].split(" ")
# foreach word in name
for e in exploded_name:
# appnd to index tree
self.append_to_index(e, object_id)
def get_object(self, key):
return self.objects.get(key)
def append_to_objects(self, obj):
self.objects.insert(obj.get_id(), obj)
def append_to_index(self, key, object_id):
# select desired tree from hash
tree = self.select_tree(key)
if tree == None:
print "Error adding new entry, key was empty string. Obj.ID: " + object_id + "\n"
return
# update existing object_id's for key
entry = tree.get(key);
if entry == None:
# append first value
entry = str(object_id)
else:
# append N-th value
entry += "," + object_id
# update entry for key
tree.insert(key, entry)
# select indexing tree accoriding key first letter
def select_tree(self, key):
if(len(key) == 0):
return None
# get first letter of key, lowercase
k = key[0].lower()
# select tree
if k in self.hash_index_tree:
# if firt letter is list of trees, return tree
return self.hash_index_tree[k]
else:
# otherwise return universal tree
return self.hash_index_tree["+"]
def __str__(self):
return str(self.hash_index_tree)
# get items generator for index trees by key, or objects tree (k == None)
def get_items_generator(self, k):
if k == None:
return self.objects.items()
return self.hash_index_tree[k].items()
# load index from existing index in folder
def load_persistant_index(self, index_name):
# load objects
self.load_objects(index_name)
# load all index trees
self.load_index_trees(index_name)
# load objects to memmory
def load_objects(self, index_name):
# initialise empty tree
self.objects = AVLTree()
# try open file with objects
try:
ins = open(index_name + "/bin_tree_dump_objects.txt", "r")
except:
print "Index tree for objects doesnt exist"
return
# reading objects from file, one by one
for line in ins:
try:
# load serialized data from file
x = json.loads(line)
# create instance of FreebaseObject
o = FreebaseObject()
# initialise content with object data from file
o.deserialise(x)
# insert loaded freebase object to objects tree
self.objects.insert(o.get_id(),o)
except:
print "something wrong 2"
print "Index tree for objects loaded"
#load all index trees
def load_index_trees(self, index_name):
hashKeys = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','x','y','z','0','1','2','3','4','5','6','7','8','9','+']
# for each key in hashKeys try load file with data for index tree
for key in hashKeys:
# create empty tree
self.hash_index_tree[key] = AVLTree()
# try open appropriate file
try:
ins = open(index_name + "/bin_tree_dump_" + key + ".txt", "r")
except:
print "Index tree for " + key + " doesnt exist"
continue
# try reads entries from file (entry = line)
for line in ins:
try:
# deserialise data from file
x = json.loads(line)
# get key and value
k = x[0]
v = x[1]
# insert data to actual index tree
self.hash_index_tree[key].insert(k,v)
except:
print "something wrong"
print "Index tree for " + key + " loaded"
def __init__(self, segments, pos_x):
self.segments = segments
self.pos_x = pos_x
self.tree = AVLTree()
class SortedTypedList(object):
"""This data structure allows to sort and maintain sorted
a list of objects inheriting from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
It uses an AVLTree proposed by :mod:`bintrees` to represent elements in the list.
:TODO: The inner object __tree stores the given element using an AVLTree. Thus
exposing new methods only requires the creation of a wrapper method in this class.
>>> from netzob.all import *
>>> from netzob.Common.Utils.SortedTypedList import SortedTypedList
>>> msg1 = RawMessage(b"msg1", date=25.0)
>>> msg2 = RawMessage(b"msg2", date=2.0)
>>> msg3 = RawMessage(b"msg3", date=1456487548.0)
>>> l = SortedTypedList(RawMessage, [msg2, msg3, msg1])
>>> print(list(l.values())[0])
[0;32m[2.0 [0;m[1;32mNone[1;m[0;32m->[0;m[1;32mNone[1;m[0;32m][0;m 'msg2'
>>> msg4 = RawMessage(b"msg4", date=145548.0)
>>> l.add(msg4)
>>> msg5 = RawMessage(b"msg5", date=14.0)
>>> msg6 = RawMessage(b"msg6", date=1745645548.0)
>>> l.addAll([msg5, msg6])
>>> print(list(l.values())[5])
[0;32m[1745645548.0 [0;m[1;32mNone[1;m[0;32m->[0;m[1;32mNone[1;m[0;32m][0;m 'msg6'
>>> print(len(l))
6
"""
def __init__(self, membersTypes, elements=None):
self.membersTypes = membersTypes
self.__treePriorities = AVLTree()
self.__mapMessages = dict()
if elements is not None and len(elements) > 0:
self._extend(elements)
def add(self, element):
"""Insert in the proper place the specified element.
:type: any object that comply with the typed of the current list and inherits from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
:raises: a TypeError if element is None or if its type doesn't comply with
the definition of the list.
"""
if element is None:
raise TypeError("Element cannot be None")
self._extend([element])
def addAll(self, elements):
"""Insert in their proper place all the specified element.
:type: a list of any object that comply with the typed of the current list and inherits from :class:`netzob.Common.Utils.SortableObject.SortableObject`.
:raises: a TypeError if element is None or if its type doesn't comply with
the definition of the list.
"""
if elements is None:
raise TypeError("Element cannot be None")
self._extend(elements)
def values(self):
"""Return a list sorted with the values of the current SortedTypedList.
:warning: modifying this list has no effect on the SortedTypedList.
:rtype: :mod:list
"""
l = []
for x in list(self.__treePriorities.keys()):
l.extend(self.__mapMessages[x])
return l
def clear(self):
"""remove all items from the list.
It's a O(n) operation"""
self.__treePriorities.clear()
def _extend(self, elements):
"""Add all the elements in the current list.
:parameter elements: a list of :class:`netzob.Common.Utils.SortableObject.SortableObject` to insert.
:raises: TypeError if something is wrong with the given elements
"""
for e in elements:
self._check(e)
d = dict()
for e in elements:
d[e.priority()] = None
if e.priority() in self.__mapMessages:
self.__mapMessages[e.priority()].append(e)
else:
self.__mapMessages[e.priority()] = [e]
self.__treePriorities.update(d)
def _check(self, v):
if not isinstance(v, self.membersTypes):
raise TypeError(
"Invalid type for argument, expecting: {0}, received : {1}".
format(self.membersTypes, v.__class__.__name__))
if not isinstance(v, SortableObject):
raise TypeError(
"Objects inserted in a SortedTypedList must inherits from SortableObject class"
)
def __len__(self):
"""Returns the number of elements in the sorted list which takes
O(1) operation :)"""
return len(self.__treePriorities)
def __str__(self):
return ', \n'.join([str(v) for v in list(self.values())])
def __repr__(self):
return repr(str(self))
def __iter__(self):
"""SortedTypedList is an iterable over its values (and not its keys)."""
return list(self.__treePriorities.values()).__iter__()
class _SweepState(object):
def __init__(self, segments, pos_x):
self.segments = segments
self.pos_x = pos_x
self.tree = AVLTree()
def insert_and_get_new_neighbours(self, new_segment_id):
new_neighbours = []
self.tree.insert(new_segment_id, new_segment_id)
try:
new_neighbours.append((self.tree.prev_item(new_segment_id)[0], new_segment_id))
except KeyError:
pass
try:
new_neighbours.append((self.tree.succ_item(new_segment_id)[0], new_segment_id))
except KeyError:
pass
return new_neighbours
def erase_and_get_new_neighbours(self, segment_id):
new_neighbours = []
before = None
after = None
try:
before = self.tree.prev_item(segment_id)[0]
except KeyError:
pass
try:
after = self.tree.succ_item(segment_id)[0]
except KeyError:
pass
if before and after:
new_neighbours.append((before, after))
self.tree.remove(segment_id)
return new_neighbours
def swap_and_get_new_neighbours(self, segment1_id, segment2_id):
pos_x = self.pos_x
self.pos_x = pos_x - epsilon
new_neighbours1 = self.erase_and_get_new_neighbours(segment1_id)
self.pos_x = pos_x + epsilon
new_neighbours2 = self.insert_and_get_new_neighbours(segment1_id)
self.pos_x = pos_x
all_neighbours = new_neighbours1 + new_neighbours2
new_neighbours = []
for neighbours in all_neighbours:
if neighbours != (segment1_id, segment2_id) and neighbours != (segment2_id, segment1_id):
new_neighbours.append(neighbours)
return new_neighbours
def _get_cross_y_with_sweep_for_segment(self, segment):
segment_dx = segment.x2 - segment.x1
sweep_dx = self.pos_x - segment.x1
segment_dy = segment.y2 - segment.y1
sweep_dy = segment_dy * (sweep_dx / segment_dx)
return segment.y1 + sweep_dy
def get_cross_y_with_sweep_for_segment_id(self, segment_id):
return self._get_cross_y_with_sweep_for_segment(self.segments[segment_id.idx])
def get_segments_in_state(self):
return self.tree.values()
def sweep(points):
segments = []
a = points[-1]
for b in points:
segments.append(Segment(a, b))
a = b
n = len(segments)
split_points = [[] for _ in range(n)]
all_intersections = [Point(0, 0)]
for i in range(n):
seg_ab = segments[i]
for j in range(i+1, n):
seg_cd = segments[j]
intersections = intersect(seg_ab, seg_cd)
"""
for intersection in intersections:
closest = min(p in all_intersection, key=p.dist2(intersection))
if intersection.dist2(closest) < 1e-10:
print("Merge")
"""
split_points[i].extend(intersections)
split_points[j].extend(intersections)
segments = split_things(segments, split_points, split_segment)
random.seed(1)
for segment in segments:
draw_line(segment[0], segment[1], random_hex_color())
event_points = []
for segment in segments:
a, b = segment
if a > b:
a, b = b, a
event_points.append((a, 0, segment))
event_points.append((b, 1, segment))
event_points.sort()
n = len(segments)
for i in range(n):
ab = segments[i]
for j in range(i+1, n):
cd = segments[j]
if ab == cd:
print("Duplicate segment", i, j)
print(ab)
print(cd)
print(ab.slope())
print(cd.slope())
print(ab.current_y())
print(cd.current_y())
import sys
sys.exit(0)
current_curves = AVLTree()
for p, event_type, segment in event_points:
global current_x
current_x = p[0]
if event_type == 0:
current_curves.insert(segment, None)
if event_type == 1:
current_curves.remove(segment)
class Chromosome(object):
def __init__(self, name, genome):
self.name = name
self.genome = genome
self.sb_name_mapping = {}
self.start_telomer = StartTelomereSyntenyBlock(name)
self.end_telomer = EndTelomereSyntenyBlock(name)
self.blocks = AVLTree()
self.add_sb(self.start_telomer)
self.add_sb(self.end_telomer)
# maps sb into a dict of resolutions, it has mapping in
# in each particular resolution, mapping is represented as a list,
# this particular block is mapped to
self.mapping = defaultdict(lambda: defaultdict(list))
def add_sb(self, sb):
if not isinstance(sb, SyntenyBlock):
sb = self._create_sb(sb)
self.genome.location_info[sb] = self
self.sb_name_mapping[sb] = sb
self.blocks.insert(key=(sb.start, sb.end), value=sb)
def _create_sb(self, info):
"""
example:
sb = SyntenyBlock(name=info.name,
start=info.start,
end=info.end,
strand=info.strand,
chromosome=self)
"""
sb = SyntenyBlock(name=info[0], start=info[1], end=info[2], strand=info[3], chromosome=self)
return sb
def add_mapping(self, sb, res_to_map, sbs):
for synteny_block in sbs:
if synteny_block not in self.mapping[sb][res_to_map]:
self.mapping[sb][res_to_map].append(synteny_block)
def get_right_outer_most_mapping(self, sb, mapping):
if sb in self.mapping and mapping in self.mapping[sb]:
return self.mapping[sb][mapping][-1]
return self.start_telomer
def get_left_outer_most_mapping(self, sb, mapping):
if sb in self.mapping and mapping in self.mapping[sb]:
return self.mapping[sb][mapping][0]
return self.end_telomer
def get_sbs_between(self, sb_start, sb_end):
# slice from 1: because, value_slice returns all values, which keys are greater or equals that minimum,
# but less than maximum. Because of that "or equal" we have to slice
result = list(self.blocks.value_slice((sb_start.start, sb_start.end), (sb_end.start, sb_end.end)))
return result[1:] if len(result) > 0 else result
def get_two_closest_mappings_and_relative_position(self, sb, res_to_map):
l_position, r_position = 0, 0
if res_to_map in self.mapping[sb]:
return sb, sb, l_position, r_position
for value in self.blocks.value_slice(start_key=(sb.start, sb.end)):
r_position += 1
if res_to_map in self.mapping[value]:
right_closest_mapping = value
break
else:
right_closest_mapping = self.end_telomer
for value in self.blocks.value_slice(end_key=(sb.start, sb.end), reverse=True):
l_position += 1
if res_to_map in self.mapping[value]:
left_closest_mapping = value
break
else:
left_closest_mapping = self.start_telomer
return left_closest_mapping, right_closest_mapping, l_position, r_position
def __hash__(self):
return self.name.__hash__()
def __str__(self):
return str(self.blocks)
