def test_point_types(self):
emptyTree = kdtree.create(dimensions=3)
point1 = (2, 3, 4)
point2 = [4, 5, 6]
Point = collections.namedtuple('Point', 'x y z')
point3 = Point(5, 3, 2)
tree = kdtree.create([point1, point2, point3])
res, dist = tree.search_nn( (1, 2, 3) )
self.assertEqual(res, kdtree.KDNode( (2, 3, 4) ))
def test_invalid_child(self):
""" Children on wrong subtree invalidate Tree """
child = kdtree.KDNode( (3, 2) )
child.axis = 2
tree = kdtree.create([(2, 3)])
tree.left=child
self.assertFalse(tree.is_valid())
tree = kdtree.create([(4, 1)])
tree.right=child
self.assertFalse(tree.is_valid())
def test_search_nn_dist3(self):
""" Test case from #36 """
pointslst = [
(0.25, 0.25, 1.600000023841858),
(0.75, 0.25, 1.600000023841858),
(1.25, 0.25, 1.600000023841858),
(1.75, 0.25, 1.600000023841858),
(2.25, 0.25, 1.600000023841858),
(2.75, 0.25, 1.600000023841858),
]
tree = kdtree.create(pointslst)
point = (0.42621034383773804, 0.18793821334838867, 1.44510018825531)
points = tree.inorder()
points = sorted(points, key=lambda p: p.dist(point))
for p in points:
dist = p.dist(point)
nn = tree.search_nn_dist(point, dist)
for pn in points:
if pn in nn:
msg = '{} in {} but {} < {}'.format(
pn, nn, pn.dist(point), dist)
self.assertTrue(pn.dist(point) < dist, msg)
else:
msg = '{} not in {} but {} >= {}'.format(
pn, nn, pn.dist(point), dist)
self.assertTrue(pn.dist(point) >= dist, msg)
def __init__(self,p,s,t,seed = None,smooth=True,motion=30,max_time = 400, m = 8000):
#s is the starting position
#t is the goal position
self.s = Node((s[0],s[1],0))
self.t = t
self.polygon = p
self.smooth = smooth
self.motion = motion
self.max_time = max_time
self.m = m
#data structure
# self.tree = Tree()
self.tree = kdtree.create(dimensions=3)
self.tree.add(self.s)
self.outside = multiLevelPoly.bounds
if not seed is None:
random.seed(seed)
else:
self.seed = random.randint(0, sys.maxint)
random.seed(self.seed)
self.pathsCollection = []
self.timeDistribution = []
def test_search_nn3(self):
points = [(0, 25, 73), (1, 91, 85), (1, 47, 12), (2, 90, 20),
(2, 66, 79), (2, 46, 27), (4, 48, 99), (5, 73, 64), (7, 42, 70),
(7, 34, 60), (8, 86, 80), (10, 27, 14), (15, 64, 39), (17, 74, 24),
(18, 58, 12), (18, 58, 5), (19, 14, 2), (20, 88, 11), (20, 28, 58),
(20, 79, 48), (21, 32, 8), (21, 46, 41), (22, 6, 4), (22, 42, 68),
(22, 62, 42), (24, 70, 96), (27, 77, 57), (27, 47, 39), (28, 61, 19),
(30, 28, 22), (34, 13, 85), (34, 39, 96), (34, 90, 32), (39, 7, 45),
(40, 61, 53), (40, 69, 50), (41, 45, 16), (41, 15, 44), (42, 40, 19),
(45, 6, 68), (46, 79, 91), (47, 91, 86), (47, 50, 24), (48, 57, 64),
(49, 21, 72), (49, 87, 21), (49, 41, 62), (54, 94, 32), (56, 14, 54),
(56, 93, 2), (58, 34, 44), (58, 27, 42), (59, 62, 80), (60, 69, 69),
(61, 67, 35), (62, 31, 50), (63, 9, 93), (63, 46, 95), (64, 31, 2),
(64, 2, 36), (65, 23, 96), (66, 94, 69), (67, 98, 10), (67, 40, 88),
(68, 4, 15), (68, 1, 6), (68, 88, 72), (70, 24, 53), (70, 31, 87),
(71, 95, 26), (74, 80, 34), (75, 59, 99), (75, 15, 25), (76, 90, 99),
(77, 75, 19), (77, 68, 26), (80, 19, 98), (82, 90, 50), (82, 87, 37),
(84, 88, 59), (85, 76, 61), (85, 89, 20), (85, 64, 64), (86, 55, 92),
(86, 15, 69), (87, 48, 46), (87, 67, 47), (89, 81, 65), (89, 87, 39),
(89, 87, 3), (91, 65, 87), (94, 37, 74), (94, 20, 92), (95, 95, 49),
(96, 15, 80), (96, 27, 39), (97, 87, 32), (97, 43, 7), (98, 78, 10),
(99, 64, 55)]
tree = kdtree.create(points)
point1 = (66, 54, 29)
nn, dist = tree.search_nn(point1)
best, best_dist = self.find_best(tree, point1)
self.assertEqual(best_dist, dist)
def __init__(self, train_data=None, train_label=None, dimensions=None,
axis=0, sel_axis=None):
"""
Creates a new KNN model contains a kdtree build by the point_list.
train_data is the list of (point, label) tuples, which is sample.
We use point to build kdtree, and use label to make the decision
when classify new data.
All points in the point_list must be of the same dimensionality.
dimensions is the dimension of points in pointlist.
If both a point_list and dimensions are given, the numbers must agree.
axis is the axis on which the root-node should split.
sel_axis is a function, sel_axis(axis) is used when creating subnodes
of a node. It receives the axis of the parent node and returns the axis
of the child node.
"""
# As train_data is a list of samples, we use dict() to change data
# structure of samples.
self.train_data = train_data
self.train_label = train_label
self.labels = set(self.train_label)
self.class_prb = self._calc_train_class_prb(self.train_label)
self.kdtree = kdtree.create(
copy.deepcopy(train_data), dimensions, axis, sel_axis)
def run_kd_tree():
t = kdtree.create(dimensions=2)
coordinates = []
currentCoord = (0.5, 0.5)
totalDistance = 0.0
t.add(currentCoord)
fIn = open(sys.argv[1])
N = int(fIn.readline())
for line in fIn.readlines():
coordInput = line.split(" ")
x = float(coordInput[0])
y = float(coordInput[1])
t.add((x, y))
start = timeit.default_timer()
# Using k-d tree
for i in range(N+1):
new = t.search_knn(currentCoord, 2)[-1] # returns closest point to currentCoord
t = t.remove(currentCoord) # mark as visited by removing
totalDistance += float(new[1])
currentCoord = eval(str(new[0])) # convert into tuple
end = timeit.default_timer()
fIn.close()
print("\nTotal distance traveled:", end='')
print(repr(totalDistance).rjust(25))
print("Total time taken:", end='')
print(repr(end-start).rjust(32))
def test_level_node_unbalanced(self):
tree2=kdtree.create([ (1, 2), (2, 3) ])
node=tree2.search_nn((1.1,2.1))
node[0].add((1,1))
self.assertEqual(tree2.level(tree2), 0)
self.assertEqual(node[0].level(tree2), 1)
self.assertEqual(node[0].left.level(tree2), 2)
def build_tree(data): tree = kdtree.create(dimensions=3) for item in data: curr = item node = [curr['x'],curr['y'],curr['z']] tree.add(node) tree = tree.rebalance() return tree
def create_fast_climb_kdtree(X, kernel, param):
d = len(X[0])
tree = kdtree.create(X)
def climb(x):
pass
return climb
def test_search_nn(self, nodes=100):
points = list(islice(random_points(), 0, nodes))
tree = kdtree.create(points)
point1 = random_point()
nn, dist = tree.search_nn(point1)
best, best_dist = self.find_best(tree, point1)
self.assertEqual(best_dist, dist, msg=', '.join(repr(p) for p in points) + ' / ' + repr(point1))
def reset(self):
self.tree = kdtree.create(dimensions=3)
self.tree.add(self.s)
self.seed = random.randint(0, sys.maxint)
random.seed(self.seed)
self.tStart = time.clock()
def test_payload(self, nodes=100, dimensions=3):
points = list(islice(random_points(dimensions=dimensions), 0, nodes))
tree = kdtree.create(dimensions=dimensions)
for i, p in enumerate(points):
tree.add(p).payload = i
for i, p in enumerate(points):
self.assertEqual(i, tree.search_nn(p)[0].payload)
def start(self, source, goal, nodes):
self.source = source
self.goal = goal
pygame.init()
screen = pygame.display.set_mode(WINSIZE)
pygame.display.set_caption('RRT star using KdTrees')
white = 100, 100, 100
black = 20, 20, 40
bright = 255, 255, 255
screen.fill(black)
RRTree = node(source, [], None, True) #actual RRTree
Points = kdTree(None, None, 0, source, RRTree) #for storing generated points to increase the search complexity
NearestNeighbourTree = kdtree.create([source]) # for storing points same as Points, using it for finding points in a range
current = source
Pointmap = [[0 for i in range (YDIM)] for i in range(XDIM)]
Pointmap[source[0]][source[1]] = 1
count = 0
while not self.check(current, goal):
rand = self.generatePoints(Pointmap)
current = self.addConnections(Points, Pointmap, rand, screen, source)
count = count + 1
#current = self.addConnection1(Points, rand, screen)
pygame.display.update()
for e in pygame.event.get():
if e.type == QUIT or (e.type == KEYUP and e.key == K_ESCAPE):
sys.exit("Leaving .")
ret = Points.search(current, 100000000000000000000, None, None, None, None, None)
nde = ret[2]
self.goalNode = nde
path = []
while nde.parent != None:
print nde.point, nde.cost
path.append(nde.point)
pygame.draw.line(screen, bright, nde.point, nde.parent.point)
pygame.display.update()
nde = nde.parent
if nde.parent == nde:
break
time.sleep(0.05)
for e in pygame.event.get():
if e.type == QUIT or (e.type == KEYUP and e.key == K_ESCAPE):
sys.exit("Leaving.")
print 'count', count
self.path = path
for i in range(10000-count):
rand = self.generatePoints(Pointmap)
current = self.addConnections(Points, Pointmap, rand, screen, source)
pygame.display.update()
for e in pygame.event.get():
if e.type == QUIT or (e.type == KEYUP and e.key == K_ESCAPE):
sys.exit("Leaving.")
def simplify_path(path):
#lines = list(extract_lines(path))
lines = to_steps(extract_lines(path))
result = shapely.ops.linemerge(lines)
print "building graph"
graph = graph_lines(result)
print "building kdtree"
tree = kdtree.create(graph.keys())
return build_path_commands(tree, graph)
def test_search_nn2(self):
points = [(1,2,3),(5,1,2),(9,3,4),(3,9,1),(4,8,3),(9,1,1),(5,0,0),
(1,1,1),(7,2,2),(5,9,1),(1,1,9),(9,8,7),(2,3,4),(4,5,4.01)]
tree = kdtree.create(points)
point1 = (2,5,6)
nn, dist = tree.search_nn(point1)
best, best_dist = self.find_best(tree, point1)
self.assertEqual(best_dist, dist)
def __init__(self, svg_file, pins):
GObject.GObject.__init__(self)
self.svg_handle = Rsvg.Handle.new_from_file(svg_file)
self.pins = kdtree.create(dimensions=2)
self.connections = []
self.quick_union = None
self.add_pins(pins)
def __init__(self, world):
# kd trees have a fast, binary search-like lookup for items at or near a given point. This should be a better
# way to track sparsely distributed, arbitrarily placed mobile objects like Entities compared to scanning
# entire near-empty grids or keeping a short list but comparing a known Entity against literally all others
# just to find out who's in range of whom (like when drawing the viewport).
self.entities = kdtree.create(dimensions=2)
self.world = world
EventHandler.subscribe(Subscription(
EVENTS.UI_EVENT, self.on_entity_moved,
priority=-1, is_permanent=True,
))
def test_search_nn_dist(self):
""" tests search_nn_dist() according to bug #8 """
points = [(x,y) for x in range(10) for y in range(10)]
tree = kdtree.create(points)
nn = tree.search_nn_dist((5,5), 2.5)
self.assertEquals(len(nn), 4)
self.assertTrue( kdtree.KDNode(data=(6,6)) in nn)
self.assertTrue( (5,5) in nn)
self.assertTrue( (5,6) in nn)
self.assertTrue( (6,5) in nn)
def __init__(self, gen, verbose=False, maxsize=None):
tree = kdtree.create(dimensions=Mobius.REAL_DIMS)
self.tree = tree
self.size = 0
for op in gen:
tree.add(op)
self.size += 1
if I not in gen:
tree.add(I)
self.size += 1
self.gen = list(gen)
if maxsize is not None:
self.generate(verbose, maxsize)
def _get_palette(path_to_palette):
palette_file = open(path_to_palette, newline="")
palette_reader = csv.reader(palette_file,
dialect="excel",
delimiter=";",
quotechar='"')
blocks = [
_Block(tuple([int(i) for i in block[1:4]]), block[0], block[4] or None)
for block in palette_reader
]
palette = kdtree.create(blocks)
palette_file.close()
return palette
def __init__(self,
occupancy_list,
cell_size,
car_geometry,
path_collsion_interval,
accelerated=True):
self.occupancy_list = occupancy_list
self.cell_size = float(cell_size)
self.car_geometry = car_geometry
self.path_collsion_interval = float(path_collsion_interval)
obstacles = [obsnode.ObstacleNode(config) for config in occupancy_list]
self.kd_tree = kd.create(point_list=obstacles, dimensions=2)
self.accerated = accelerated
def check_dataset_for_duplicates(profile, dataset, print_all=False):
# First checking for duplicate ids and collecting tags with varying values
ids = set()
tags = {}
found_duplicate_ids = False
for d in dataset:
if d.id in ids:
found_duplicate_ids = True
logging.error('Duplicate id {} in the dataset'.format(d.id))
ids.add(d.id)
for k, v in d.tags.items():
if k not in tags:
tags[k] = v
elif tags[k] != '---' and tags[k] != v:
tags[k] = '---'
# And then for near-duplicate points with similar tags
uncond_distance = profile.get('duplicate_distance', 1)
diff_tags = [k for k in tags if tags[k] == '---']
kd = kdtree.create(list(dataset))
duplicates = set()
group = 0
for d in dataset:
if d.id in duplicates:
continue
group += 1
dups = kd.search_knn(d, 2) # The first one will be equal to d
if len(dups) < 2 or dups[1][0].data.distance(d) > profile.max_distance:
continue
for alt, _ in kd.search_knn(d, 20):
dist = alt.data.distance(d)
if alt.data.id != d.id and dist <= profile.max_distance:
tags_differ = 0
if dist > uncond_distance:
for k in diff_tags:
if alt.data.tags.get(k) != d.tags.get(k):
tags_differ += 1
if tags_differ <= len(diff_tags) / 3:
duplicates.add(alt.data.id)
d.exclusive_group = group
alt.data.exclusive_group = group
if print_all or len(duplicates) <= 5:
is_duplicate = tags_differ <= 1
logging.error('Dataset points %s: %s and %s',
'duplicate each other' if is_duplicate else 'are too similar',
d.id, alt.data.id)
if duplicates:
logging.error('Found %s duplicates in the dataset', len(duplicates))
if found_duplicate_ids:
raise KeyError('Cannot continue with duplicate ids')
def do_random_add(self, num_points=100):
points = list(set(islice(random_points(), 0, num_points)))
tree = kdtree.create(dimensions=len(points[0]))
for n, point1 in enumerate(points, 1):
tree.add(point1)
self.assertTrue(tree.is_valid())
self.assertTrue(point1 in [node.data for node in tree.inorder()])
nodes_in_tree = len(list(tree.inorder()))
self.assertEqual(nodes_in_tree, n)
def do_random_add(self, num_points=100):
points = list(set(islice(random_points(), 0, num_points)))
tree = kdtree.create(dimensions=len(points[0]))
for n, point in enumerate(points, 1):
tree.add(point)
self.assertTrue(tree.is_valid())
self.assertTrue(point in [node.data for node in tree.inorder()])
nodes_in_tree = len(list(tree.inorder()))
self.assertEqual(nodes_in_tree, n)
def get_kd_tree_result(self, standard_gps, rel_coords):
'''
@summary: get kd tree search results
'''
kd_results = ""
if len(rel_coords) >= SEARCH_NODES:
click_coord = utils.get_rel_coord(standard_gps,
self.add_coords_list[0])
kd_tree = kdtree.create(rel_coords, 2, 0, 0)
kd_results = kd_tree.search_knn(click_coord, SEARCH_NODES)
else:
print "no enough coords for kd tree search, min = " + str(
SEARCH_NODES)
return kd_results
def progressive_peak_find(h, distinctness=1.0):
array = sorted(np.ndenumerate(h), key=lambda a: a[1], reverse=True)
peaks = dict()
peaks[array[0][0]] = array[0][1]
visited = kdtree.create([array[0][0]])
for idx, value in array[1:]:
nearest_visited, d = visited.search_nn(idx)
d = d / h.shape[0]
if d > distinctness:
peaks[idx] = value
visited.add(idx)
ordered_peaks = sorted(peaks.items(), key=lambda p: p[1], reverse=True)
coords, values = zip(*ordered_peaks)
return coords, values
def initializeState(appState):
app = appState.app
app.GEODB = geoip2.database.Reader(app.config['GEODB_PATH'])
def makePoint(value):
return PointWithInfo(
value['_real_position']
if '_real_position' in value else value['position'], {
'name': value['name'],
'host': urllib.parse.urlparse(value['site']).hostname
})
hubJson = json.load(open(app.config['HUBS_PATH'], encoding="utf-8"))
hubList = [
value for value in hubJson.values() if value['fch-enabled'] != False
]
app.HUB_INDEX = kdtree.create(
[makePoint(value) for value in hubList if value['ndn-up'] == True])
app.WSS_INDEX = kdtree.create([
makePoint(value) for value in hubList
if value['ndn-up'] == True and value['ws-tls'] == True
])
def __init__(self, dronesSet: DronesSet, missionType: MissionType,
initialDronePos: dict, offsetDronePos: dict,
sendMessageCallable: Callable[[Message], None]):
"""Initialize the mission handler. Reject the mission if the droneSet
is empty. Gives random colors to the drones ins the droneSet. Save
the newly created mission object and saves it in the database.
@param dronesSet: the set of drones participating in the mission.
@param missionType: The type of the mission: real or fake. Fake is
for demo purposes only. @param sendMessageCallable: the function to
call to send mission pulses.
"""
self.RANGE_SCALE: float = (
missionType == 'argos') * self.ARGOS_SCALE + (
missionType == 'crazyradio') * self.CRAZYRADIO_SCALE
self.maxRange = (
(missionType == 'argos') * self.ARGOS_MAX_RANGE +
(missionType == 'crazyradio') * self.CRAZYRADIO_MAX_RANGE)
drones: List[Drone] = list(dronesSet.getDrones().values())
if len(drones) == 0:
logging.info("Mission rejected: no drones")
status: MissionStatus = 'rejected'
sendMessageCallable(
Message(type='missionPulse', data={'status': status}))
return
self.initialDronePos = initialDronePos
self.offsetDronePos = offsetDronePos
self.dronesSet = dronesSet
self.sendMessageCallable = sendMessageCallable
missionDrones: MissionDrones = {
drone['name']: (CSS_PREDEFINED_COLORS[drone['name'].__hash__() %
len(CSS_PREDEFINED_COLORS)])
for drone in drones
}
timestamp = getTimestamp()
self.mission = Mission(
id=f'Mission - {timestamp}',
timestamp=timestamp,
type=missionType,
status='inProgress',
drones=missionDrones,
dronesPositions={drone['name']: []
for drone in drones},
dronesPaths={drone['name']: []
for drone in drones},
shapes=[],
points=[])
# DatabaseService.saveMission(self.mission['id'], self.mission)
sendMessageCallable(Message(type='mission', data=self.mission))
self.kdtree = kdtree.create(dimensions=2)
def run(self):
debug('[run]')
if self.with_color:
print('Counting colors ...')
color = self.rescale(self.img, self.pseudoDim)
collapsed = np.sum(color, axis=2) / 3
fill = np.argwhere(collapsed < 230) # color 2-d indices
fill = np.swapaxes(fill, 0, 1) # swap to index into color
RGB = color[fill[0], fill[1], :]
k_means = KMeans(n_clusters=self.num_colors).fit(RGB)
colors = k_means.cluster_centers_
labels = k_means.labels_
fill = np.swapaxes(fill, 0,
1).tolist() # swap back to make dictionary
label_2_index = defaultdict(list)
for i, j in zip(labels, fill):
label_2_index[i].append(j)
print('Number of colors:', len(colors))
for (i, color) in enumerate(colors):
B, G, R = map(int, color)
print(
f'Change pen to {CY}THICK{CX} (big), color to {CY}RGB{CX} values: R: {CR}{R}{CX} G: {CG}{G}{CX}, B: {CB}{B}{CX} (hex: #{CR}{R:02X}{CG}{G:02X}{CB}{B:02X}{CX})'
)
input(f"\nPress {CG}ENTER{CX} once ready")
print('')
points = label_2_index[i]
index_tuples = map(tuple, points)
self.hashSet = set(index_tuples)
self.KDTree = create(points)
self.commands = []
self.curr_pos = (0, 0)
point = self.translate(self.curr_pos)
self.commands.append(point)
self.commands.append("UP")
self.createPath()
input(f'\n{CR}Ready!{CX} Press {CG}ENTER{CX} to draw')
print(f'\n{CY}5 seconds until drawing begins...{CX}\n')
time.sleep(5)
self.execute(self.commands)
if self.outline_again:
self.drawOutline()
def splitClusters(self):
for k in range(len(self.Clusters[:, 0])): #For every cluster
if self.Clusters[
k,
1] >= 2 * self.N: # and self.Clusters[k,2]>2*self.r: #If # of data that cluster has is greater than N
X = self.buffered_data[self.buffered_data[:, 1] ==
self.Clusters[k,
0], 3:] #data cluster k
tree = kdtree.create(
X.tolist()) #construct kdtree with data of cluster k
for l in range(len(X[:, 0])): # for each data of cluster k
points = tree.search_nn_dist(
X[l, :],
self.r) # find number of data in radius r for data l
if len(
points
) >= self.N: # if # of data in area is greater than N
center = self.calculate_cluster_center(
np.array(points)
) # calculate centroid of candidate cluster
indices = npi.indices(
X, points, missing='ignore'
) #find data in the all data of cluster k
points2 = np.delete(
X, indices, 0) # find remaining data of cluster k
if len(points2) >= self.N:
center2 = self.calculate_cluster_center(
np.array(points2))
dis = euclidean_distances([center], [center2])
r1 = self.calculateRadius(points, center)
r2 = self.calculateRadius(points2, center2)
if float(dis) > r1 + r2 + 0.5 * self.r:
new_cluster_label = self.Clusters.shape[0] + 1
self.Clusters = np.vstack([
self.Clusters,
np.hstack([
new_cluster_label,
len(points), 1, self.r,
np.mean(np.std(points, axis=0)), center
])
])
indices = np.isin(self.buffered_data[:, 3:],
points)[:, 0]
self.buffered_data[indices == True,
1] = new_cluster_label
self.buffered_data[indices == True, 2] = 1
print("Cluster #%d is split." %
(self.Clusters[k, 0]))
break
def solve(vehicles_amount, per_ride_bonus, num_steps, ride_list):
global rides
priority_queue = VehiclePriority()
rides = kdtree.create(ride_list, dimensions=3)
vehicles = [Vehicle(vehicle_id) for vehicle_id in range(vehicles_amount)]
for vehicle in vehicles:
priority_queue.add_vehicle(vehicle)
while not priority_queue.empty():
vehicle = priority_queue.get_vehicle()
did_find_ride = vehicle.calculate_best_route(num_steps)
if did_find_ride:
priority_queue.add_vehicle(vehicle)
return vehicles
def nearest(point, Otherpoints):
ansr = []
# print(ansr)
# input()
points = list(set([point])) + Otherpoints
ManhattanDistance = lambda a, b: sum(
abs(a[axis] - b[axis]) for axis in range(len(a)))
if Otherpoints != []:
root = kdtree.create(points, dimensions=2)
ans = root.search_knn(point=points[0], k=3, dist=ManhattanDistance)
i = 0
for r in ans:
ansr.append(ans[i][0].data)
i += 1
return ansr[1:]
def load_places_tree(self):
class PlacePoint:
def __init__(self, lon, lat, country, region):
self.coord = (lon, lat)
self.country = country
self.region = region
def __len__(self):
return len(self.coord)
def __getitem__(self, i):
return self.coord[i]
def unpack_coord(data):
if data[-1] > 0x7f:
data += b'\xFF'
else:
data += b'\0'
return struct.unpack('<l', data)[0] / 10000
filename = os.path.join(os.getcwd(), os.path.dirname(__file__),
'places.bin')
if not os.path.exists(filename):
return None
places = []
with open(filename, 'rb') as f:
countries = []
cnt = struct.unpack('B', f.read(1))[0]
for i in range(cnt):
countries.append(
struct.unpack('2s', f.read(2))[0].decode('ascii'))
regions = []
cnt = struct.unpack('<h', f.read(2))[0]
for i in range(cnt):
l = struct.unpack('B', f.read(1))[0]
regions.append(f.read(l).decode('ascii'))
dlon = f.read(3)
while len(dlon) == 3:
dlat = f.read(3)
country = struct.unpack('B', f.read(1))[0]
region = struct.unpack('<h', f.read(2))[0]
places.append(
PlacePoint(unpack_coord(dlon), unpack_coord(dlat),
countries[country], regions[region]))
dlon = f.read(3)
if not places:
return None
return kdtree.create(places)
def simplify_path(path):
lines = svg.path.parse_path(path)
coords = [lines[0].start]
for line in lines:
if type(line) != svg.path.Line:
raise NameError('The SVG file contains a path with crap: {}.'.format(type(line)))
coords.append(line.end)
coords = [(c.real, c.imag) for c in coords]
lines = to_steps(coords)
lines = [list(lines)]
result = shapely.ops.linemerge(lines)
print("building graph")
graph = graph_lines(result)
print("building kdtree")
tree = kdtree.create(list(graph.keys()))
return build_path_commands(tree, graph)
def __init__(self):
self.name = []
self.data = []
f = open('data/caffenet4096.txt', 'r')
for line in f.readlines():
l = line.split()
n = l.pop(0)
l = map(float, l)
self.data.append(tuple(l))
self.name.append(n)
f.close()
self.tree = kdtree.create(list(self.data))
def test_search_nn_dist2(self):
""" Test case from #36 """
points = [[0.25, 0.25, 1.600000023841858], [0.75, 0.25, 1.600000023841858], [1.25, 0.25, 1.600000023841858],
[1.75, 0.25, 1.600000023841858], [2.25, 0.25, 1.600000023841858], [2.75, 0.25, 1.600000023841858]]
expected = [0.25, 0.25, 1.600000023841858]
tree = kdtree.create(points)
rmax = 1.0
search_p = [0.42621034383773804, 0.18793821334838867, 1.44510018825531]
results = tree.search_nn_dist(search_p, rmax)
found = False
for result in results:
if result == expected:
found = True
break
self.assertTrue(found)
def generate_outline_commands(self):
image = process_image(
self.image,
size=(self.canvas_width, self.canvas_height),
blur=self.blur
)
image = np.swapaxes(image, 0, 1)
black_points_indices = np.argwhere(image == 0).tolist()
index_tuples = map(tuple, black_points_indices)
self.hash_set = set(index_tuples)
self.KD_tree = kdtree.create(black_points_indices)
self._create_path()
def task_3(foursqr, cities):
a = time.time()
# Create a dictionary where the key is a
# tuple 3-d cartesian coordinates, and the value is a row
cartesian_coords_cities_dict = dict(
cities.map(lambda row: (tuple(latlong_to_cartesian(row.lat, row.lon)),
row)).collect())
# Create a k-d tree (3-d in this case) for searching
tree = kdtree.create(cartesian_coords_cities_dict.keys())
# For each row in foursqr
# Query the k-d tree for a nearest neighbour
# Return neighbour and row
cart_user_city = foursqr.map(lambda row: (cartesian_coords_cities_dict[
kdtree_query(tree, row.lat, row.lon)], row))
print("KDTree: ", time.time() - a)
return cart_user_city
def walk(self, paint=False):
"""does one randomwalk with n_steps. saves the result to the
classes list and returns it
returns [distance, array([x,y,z])]
"""
X = zeros(self.dim, dtype=float)
tree = kdtree.create([X])
for i in range(self.n_steps):
successful = False
while True:
phi = rnd.random()*2*pi
psi = rnd.random()*2*pi # for 2d set psi to 0
dX = array([sin(phi)*cos(psi),
cos(phi),
sin(phi)*sin(psi)]) * 2
X_new = X+dX
#print dX, X_new
#check if this new point is within the sphere of other
if len(tree.search_nn_dist(X_new, 2*self.steplength)) == 0:
X = X_new
tree.add(X)
tree = tree.rebalance()
#print " accepted"
break
self.result_dist2.append(sum(X**2))
self.result_element.append(X)
if paint:
"""This only makes sense if solving the 2d prob..."""
circles = []
for i, p in enumerate(list(kdtree.level_order(tree))):
circles.append(Circle(p.data, self.steplength))
fig=pylab.figure()
ax=fig.add_subplot(111)
colors = 100*pylab.rand(len(circles))
p = PatchCollection(circles, cmap=matplotlib.cm.jet, alpha=0.4)
p.set_array(pylab.array(colors))
ax.add_collection(p)
pylab.colorbar(p)
pylab.show()
return sum(X**2), X
def rball_check(x0, y0, x1, y1):
if not balls:
return rball()
import kdtree
pts = []
for ball in balls:
x, _, y = ball.getPosition()
pts.append((x, y))
tree = kdtree.create(pts)
while 1:
x = rnd(x0, x1)
y = rnd(y0, y1)
nearest = tree.search_nn_dist((x, y), (1.9 * radius)**2)
if not nearest:
break
#print(".", end="")
mkball(x, y)
def test_search_nn_dist(self):
""" tests search_nn_dist() according to bug #8 """
points = [(x, y) for x in range(10) for y in range(10)]
tree = kdtree.create(points)
nn = tree.search_nn_dist((5, 5), 2.5)
self.assertEqual(len(nn), 9)
self.assertTrue((4, 4) in nn)
self.assertTrue((4, 5) in nn)
self.assertTrue((4, 6) in nn)
self.assertTrue((5, 4) in nn)
self.assertTrue((6, 4) in nn)
self.assertTrue((6, 6) in nn)
self.assertTrue((5, 5) in nn)
self.assertTrue((5, 6) in nn)
self.assertTrue((6, 5) in nn)
def __init__(self, precision=1e-7, point_match_threshold=None):
self._precision = precision
if point_match_threshold is None:
self._point_match_threshold = precision
else:
assert (point_match_threshold >= precision)
self._point_match_threshold = point_match_threshold
self.objects = {}
# TODO: allow for more than 2 dimensions
self._kdtree = kdtree.create(dimensions=2)
self._junctions = set()
self._neighbors = {}
self._reset()
def main():
num_pts = 10000
#num_pts_to_add = 1000
#total_pts_to_add = num_pts + num_pts_to_add
n = 7
r = 0.9
H = np.zeros(
num_pts,
dtype=[
('x', float, n), # the point
('f', float), # its value
('pt_id', int), # its index
('best_dist', float) # dist to better nb
])
np.random.seed(1)
H['x'] = np.random.uniform(0, 1, (num_pts, n))
H['f'] = np.apply_along_axis(price01, 1, H['x'])
H['pt_id'] = np.arange(0, num_pts)
H['best_dist'] = np.inf
A = np.sort(H, order='f') #sorted according to function value
a = np.array(A['x']).tolist() #pre-processing for kdtree constructor
tree = kdtree.create(a[0:1], n)
#H = np.append(H,np.zeros(num_pts_to_add,dtype=H.dtype))
'''
for i in np.arange(len(H)-num_pts_to_add,len(H)):
H['x'][i] = np.random.uniform(0,1,n)
H['pt_id'][i] = i
H['best_dist'] = np.inf
H['f'] = np.apply_along_axis(price01,1,H['x'])
'''
#start = time.time()
#T = spatial.cKDTree(H['x'])
#end = time.time() - start
start = time.time()
#print("Time to initialize tree with {0} pts is {1}".format(total_pts_to_add,end))
for i in range(1, num_pts):
vect = tree.search_nn(a[i])[0].data
dist = np.linalg.norm(A['x'][i] - vect)
if dist < r:
H['best_dist'][A['pt_id'][i]] = dist
tree.add(a[i])
end = time.time() - start
print('(dim = {0})(Time to initialize {1} pts, r = {3}, is {2})'.format(
n, num_pts, end, r))
def test_search_nn(self, nodes=100):
points = list(islice(random_points(), 0, nodes))
tree = kdtree.create(points)
point = random_point()
nn = tree.search_nn(point)
best = None
best_dist = None
for p in tree.inorder():
dist = p.dist(point)
if best is None or dist < best_dist:
best = p
best_dist = dist
self.assertEqual(best_dist, best.dist(point))
def test_search_nn(self, nodes=100):
points = list(islice(random_points(), 0, nodes))
tree = kdtree.create(points)
point = random_point()
nn = tree.search_nn(point)
best = None
best_dist = None
for p in tree.inorder():
dist = p.dist(point)
if best is None or dist < best_dist:
best = p
best_dist = dist
self.assertEqual(best_dist, best.dist(point))
def rrtree(lat, threshold):
if lat is None or len(lat) == 0:
return []
tree = kdtree.create(dimensions=2)
distance_threshold = threshold # 8^2
for i,pt in enumerate(lat):
t_pt = (float(pt[0]), float(pt[1]))
search_result = tree.search_nn(t_pt, dist=euclid)
if search_result is None:
tree.add(t_pt)
else:
node, dist = search_result[0], search_result[1]
if dist >= distance_threshold:
tree.add(t_pt)
filtered_points = [(int(pt.data[0]), int(pt.data[1])) for pt in kdtree.level_order(tree)]
return filtered_points
def convert_bin_to_kdtree(tree_node_list):
id_center_dict = {}
center_list = []
edge_set = set()
for node in tree_node_list:
if node.is_virtual():
continue
if tuple(node._pos) not in id_center_dict.keys():
id_center_dict[tuple(node._pos)] = []
id_center_dict[tuple(node._pos)].append(node)
edge_set.add(tuple([node, node.parent]))
edge_set.add(tuple([node.parent, node]))
center_list.append(node._pos)
my_kdtree = kdtree.create(center_list)
return my_kdtree, id_center_dict, edge_set
def __init__(self):
# Create an empty tree of 2 dimensions for lat,long
self.tree = kdtree.create(dimensions=2)
stations_file = open(ut.get_path("nodes_with_latlng_updated.txt"), "r")
# Read each station into the kdtree
for line in stations_file:
station_details = line.strip().split("\t")
lat_lng = station_details[1].split(",")
lat = float(lat_lng[0])
lng = float(lat_lng[1])
coords = (lat, lng)
name = station_details[0]
self.tree.add(metro_parts.Station(
name, coords)) # Station is a class in metro_parts
self.tree = self.tree.rebalance()
print("Created the", self.tree.dimensions, "-d tree")
def test_remove_duplicates(self):
""" creates a tree with only duplicate points, and removes them all """
points = [(1,1)] * 100
tree = kdtree.create(points)
self.assertTrue(tree.is_valid())
random.shuffle(points)
while points:
point1 = points.pop(0)
tree = tree.remove(point1)
# Check if the Tree is valid after the removal
self.assertTrue(tree.is_valid())
# Check if the removal reduced the number of nodes by 1 (not more, not less)
remaining_points = len(points)
nodes_in_tree = len(list(tree.inorder()))
self.assertEqual(nodes_in_tree, remaining_points)
def test_search_nn_dist2(self):
""" Test case from #36 """
points = [[0.25, 0.25, 1.600000023841858],
[0.75, 0.25, 1.600000023841858],
[1.25, 0.25, 1.600000023841858],
[1.75, 0.25, 1.600000023841858],
[2.25, 0.25, 1.600000023841858],
[2.75, 0.25, 1.600000023841858]]
expected = [0.25, 0.25, 1.600000023841858]
tree = kdtree.create(points)
rmax = 1.0
search_p = [0.42621034383773804, 0.18793821334838867, 1.44510018825531]
results = tree.search_nn_dist(search_p, rmax)
found = False
for result in results:
if result == expected:
found = True
break
self.assertTrue(found)
def test_remove_duplicates(self):
""" creates a tree with only duplicate points, and removes them all """
points = [(1, 1)] * 100
tree = kdtree.create(points)
self.assertTrue(tree.is_valid())
random.shuffle(points)
while points:
point = points.pop(0)
tree = tree.remove(point)
# Check if the Tree is valid after the removal
self.assertTrue(tree.is_valid())
# Check if the removal reduced the number of nodes by 1 (not more, not less)
remaining_points = len(points)
nodes_in_tree = len(list(tree.inorder()))
self.assertEqual(nodes_in_tree, remaining_points)
def NewClusterAppear(self):
X = self.buffered_data[self.buffered_data[:, 1] == 0,
3:] #data that do not belong to any cluster
tree = kdtree.create(X.tolist()) #construct kdtree
for i in range(len(X)): # for each data of tree do reangeserach
points = tree.search_nn_dist(X[i, :], self.r)
center = self.calculate_cluster_center(np.array(points))
if len(points) >= self.N:
if (self.Clusters.size == 0):
self.Clusters = np.vstack([
self.Clusters,
np.hstack([
1,
len(points), 1, self.r,
np.mean(np.std(points, axis=0)), center
])
])
indices = np.isin(self.buffered_data[:, 3:], points)[:, 0]
self.buffered_data[indices == True, 1] = 1
print("Cluster #1 is defined.")
# print(points)
else:
flag = self.is_far_enough_to_all_clusters(center)
if (flag):
# print(points)
# print(np.mean(np.std(points,axis=0)))
new_cluster_label = self.Clusters.shape[0] + 1
self.Clusters = np.vstack([
self.Clusters,
np.hstack([
new_cluster_label,
len(points), 1, self.r,
np.mean(np.std(points, axis=0)), center
])
])
indices = np.isin(self.buffered_data[:, 3:], points)[:,
0]
self.buffered_data[indices == True,
1] = new_cluster_label
self.buffered_data[indices == True, 2] = 1
print("Cluster #%d is defined." % len(self.Clusters))
def get_recommendation(self, ingredients, k): """ Returns a list of k recommended recipes given some ingredients""" # Creates the KDTree of ingredients: # For each recipe we create an array that indicates # the quantity of each known ingredient on that recipe # --> each array becomes a point in the tree points = self.create_points() tree = kdtree.create(points) # Create a point according to selected ingredients point = self._create_point(ingredients, query=True) # Get matching recipes result = self.knn(tree, point, k) #tree.search_knn(point, k) best_recipes = [] for point in result: for r in self.recipes_dic[point]: best_recipes.append(r) return self._remove_duplicates(best_recipes)
def _next_further_point(self, points, centroids):
""" Returns furthest point from centroids """
tree = kdtree.create(centroids)
#dic = {}
further_point = (points[0], 0) # To always have the further point
for point in points:
# Get distance to all centroids
#dic[point] = 0
dist = 0
for node in list(tree.inorder()):
#dic[point] += node.dist(point)
dist += node.dist(point)
# Check if we found a better point
if further_point[1] < dist:
further_point = (point, dist)
return further_point[0]
def nn(dots, start=0):
assert start < dots.shape[0]
not_visited = dots.tolist()
current = not_visited[start]
del not_visited[start]
ordered_dots = [current]
tree = kdtree.create(not_visited)
while tree.data is not None:
nearest_info = tree.search_nn(current)
nearest = nearest_info[0].data
ordered_dots.append(nearest)
current = nearest
tree = tree.remove(current)
return np.vstack(ordered_dots)
def orderpaths(paths):
ret = []
# strip the first element (see above)
pathpoints = [map(lambda p: p[1:], path) for path in paths]
starts = [path[0] for path in pathpoints]
ends = [path[-1] for path in pathpoints]
import kdtree
tree = kdtree.create(starts + ends)
lpi = 0
ret = [paths[lpi]]
# laststart = starts.pop(lpi)
tree.remove(starts[lpi])
tree.remove(ends[lpi])
wasstart = True
for i in range(len(paths)-1):
v = tree.search_nn(ends[lpi] if wasstart else starts[lpi])
if not v:
continue
try:
lpi = starts.index(v.data)
wasstart = True
except ValueError:
lpi = ends.index(v.data)
wasstart = False
tree = tree.remove(starts[lpi])
tree = tree.remove(ends[lpi])
# i have a feeling that the order of the path should be reversed
# if wasstart is False...
ret.append(paths[lpi])
return ret
def do_random_remove(self):
""" Creates a random tree, removes all points in random order """
points = list(set(islice(random_points(), 0, 20)))
tree = kdtree.create(points)
self.assertTrue(tree.is_valid())
random.shuffle(points)
while points:
point1 = points.pop(0)
tree = tree.remove(point1)
# Check if the Tree is valid after the removal
self.assertTrue(tree.is_valid())
# Check if the point1 has actually been removed
self.assertTrue(point1 not in [n.data for n in tree.inorder()])
# Check if the removal reduced the number of nodes by 1 (not more, not less)
remaining_points = len(points)
nodes_in_tree = len(list(tree.inorder()))
self.assertEqual(nodes_in_tree, remaining_points)
def test_search_knn(self):
points = [(50, 20), (51, 19), (1, 80)]
tree = kdtree.create(points)
point1 = (48, 18)
all_dist = []
for p in tree.inorder():
dist = p.dist(point1)
all_dist.append([p, dist])
all_dist = sorted(all_dist, key = lambda n:n[1])
result = tree.search_knn(point1, 1)
self.assertEqual(result[0][1], all_dist[0][1])
result = tree.search_knn(point1, 2)
self.assertEqual(result[0][1], all_dist[0][1])
self.assertEqual(result[1][1], all_dist[1][1])
result = tree.search_knn(point1, 3)
self.assertEqual(result[0][1], all_dist[0][1])
self.assertEqual(result[1][1], all_dist[1][1])
self.assertEqual(result[2][1], all_dist[2][1])
def example_kdtree():
# An example of how to use kdtree
print "*" * 60
print "*" * 15, "An Example of kdtree's Usage", "*" * 15
print "*" * 60
point = [(2, 3), (5, 4), (9, 6), (4, 7), (8, 1), (7, 2), (8, 8)]
point1 = []
for i in point:
point1.append({1: i[0], 2: i[1]})
print "point list"
print point1
# Create a kdtree
root = kdtree.create(point1, dimensions=2)
# Visualize the kdtree
print "visualize the kd-tree: "
kdtree.visualize(root)
# Search for k-nearsest neighbor by given p-Minkowski distance
f = ds.EuclideanDistance
ans = root.search_knn(point={1: 7, 2: 3}, k=10, dist=f)
print "The 3 nearest nodes to point (7, 3) are:"
print ans
print "The nearest node to the point is:"
print ans[0][0].data
