def generate_rtree_from_entities(self):
"""Create an rtree with all entities with bounding rectangles."""
self.bounding_rects = {
id_: e.bounding_rect
for id_, e in self.entities.items() if hasattr(e, 'bounding_rect')
}
self.rtree = Rtree(
(id_, rect, None) for id_, rect in self.bounding_rects.items())
def createRtree(data):
"""
Creates an R-Tree from the given data
"""
tree = Rtree()
for index, pair in enumerate(data):
tree.insert(index, (pair[3], pair[4]), obj=pair)
return tree
def reset(self, name: Optional[str] = None):
"""Reset index and set name."""
self.name = name
self.id_count = count()
self.entities = {}
self.bounding_rects = {}
self.rtree = Rtree()
self.path_map = PathMap()
self.register_updates = False
self.simulation = Simulation()
self.stats = defaultdict(dict)
self._updates = set()
def _build_conflict_points(node: Node, curves: Dict[LaneConnection, Curve]):
"""Create conflict points from lane connection curves.
Fill conflict points in given Curve objects. Merges points that are less
than MERGE_RADIUS apart and add points that are within NEIGHBOR_RADIUS as
neighbors.
"""
id_generator = count()
points: Dict[int, ConflictPoint] = {}
diverge = defaultdict(set)
merge = defaultdict(set)
for (lanes1, curve1), (lanes2, curve2) in combinations(curves.items(), 2):
if lanes1[0] == lanes2[0]:
diverge[lanes1[0]].update((curve1, curve2))
if lanes1[1] == lanes2[1]:
merge[lanes1[1]].update((curve1, curve2))
_add_crossing_conflict_points(id_generator, points, curve1, curve2)
_add_diverge_merge_conflict_points(id_generator, points, diverge, merge)
if len(points) > 1:
rtree = Rtree((id_, p.point.bounding_rect, None)
for id_, p in points.items())
_merge_points(points, rtree)
_fill_neighbors(points, rtree)
for point in points.values():
point.create_lock_order()
# This makes the conflict point positions relative to the node.
point.point = point.point - node.position
for curve in curves.values():
curve.remove_redundant_conflict_point()
# Remove from neighbors points that aren't in any curve
points = {p for _, p in chain.from_iterable(c.conflict_points
for c in curves.values())}
for point in points:
for neighbor in list(point.neighbors):
if neighbor not in points:
point.neighbors.remove(neighbor)
def main(query, train, query_num, qgram_size):
logger = logging.getLogger('search_rtree')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('./log/%s' % query)
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
logger.info(
'------------------------- Calculate common Q-grams for query trajectories -------------------------'
)
qgram_tag = 'q_%d' % qgram_size
query_path = './data/processed/%s.txt' % query
rtree_path = './data/interim/%s/my_rtree_%s' % (train, qgram_tag)
logger.info('Query trajectory path: %s' % query_path)
logger.info('Rtree path: %s' % rtree_path)
query_data = load_trajectory(query_path, n=query_num)
logger.info('Load %d query trajectories' % query_num)
qry_qgram, qry_id_list = build_qgram(query_data, qgram_size)
qry_id_dict = build_id_to_key(
qry_id_list) # key: query_id, value: query_key
data_index = Rtree(rtree_path)
conf = SparkConf().setAppName("PythonWordCount").setMaster("local")
sc = SparkContext(conf=conf)
all_data = []
for qry_id, qry_qgrams in qry_qgram.items():
qry_key = qry_id_dict[qry_id]
data = []
for qry_qgram in qry_qgrams:
matches = [
hit.object
for hit in data_index.intersection(qry_qgram, objects=True)
]
matches = set(matches)
data.append(list(matches))
flat_data = [item for sublist in data for item in sublist]
# print(flat_data)
dist_data = sc.parallelize(flat_data)
map_data = dist_data.map(lambda x: (x, 1))
reduce_data = map_data.reduceByKey(lambda a, b: a + b).sortBy(
lambda x: x[1], ascending=False).collect()
# print(reduce_data)
all_data.append([qry_key, reduce_data])
if not os.path.exists('./data/interim/%s' % query):
os.mkdir('./data/interim/%s' % query)
if not os.path.exists('./data/interim/%s/%s' % (query, train)):
os.mkdir('./data/interim/%s/%s' % (query, train))
candidate_traj_path = './data/interim/%s/%s/candidate_trajectory_%s.txt' % (
query, train, qgram_tag)
save_pickle(all_data, candidate_traj_path)
logger.info('Output candidate_trajectory: %s' % candidate_traj_path)
query_id_dict_path = './data/interim/%s/%s/query_id_dict_%s.txt' % (
query, train, qgram_tag)
logger.info('Output query_id_dict: %s' % query_id_dict_path)
save_pickle(qry_id_dict, query_id_dict_path)
gc.collect()
RTreePage.objects(name=self.name, page=page).delete(safe=True)
except:
returnError.contents.value = self.InvalidPageError
hasData = property(
lambda self: RTreePage.objects(name=self.name).first() is not None)
if __name__ == '__main__':
settings = Property()
settings.writethrough = True
settings.buffering_capacity = 1
storage = MongoStorage('test')
storage.clear()
r = Rtree(storage, properties=settings)
r.add(123, (0, 0, 1, 1))
print "test 1 should be true"
item = list(r.nearest((0, 0), 1, objects=True))[0]
print item.id
print r.valid()
print "test 2 should be true"
r.delete(123, (0, 0, 1, 1))
print r.valid()
print "test 3 should be true"
r.clearBuffer()
print r.valid()
def main(train, qgram_size):
logger = logging.getLogger('build_rtree')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('./log/%s' % train)
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
logger.info('---------------------------- Build R-tree ----------------------------')
qgram_tag = 'q_%d' % qgram_size
train_path = './data/processed/%s.txt' % train
data = load_trajectory(train_path)
logger.info('Load train trajectory: %s' % train_path)
trajectory, id_list = build_qgram(data, qgram_size)
id_to_key_dict = build_id_to_key(id_list)
# order_key_dict = build_order_dict(id_list)
#save orderId-key mapping
#key: trajectory id in string, value: encoded key
rtree_id_dict_path = './data/interim/%s/rtree_id_dict_%s.txt' % (train, qgram_tag)
save_pickle(id_to_key_dict, rtree_id_dict_path)
logger.info('Output rtree_id_dict: %s' % rtree_id_dict_path)
#key: key, value: trajectory id in string
# filename = '../data/processed/order_key_dict.txt'
# outfile = open(filename,'wb')
# pickle.dump(order_key_dict,outfile)
# outfile.close()
# R-tree constructor
# parameter: 'data_full' is the filename of R-tree storage
# 2 files are created: data_full.dat, data_full.idx
# return: r-tree index
rtree_path = './data/interim/%s/my_rtree_%s' % (train, qgram_tag)
data_idx = Rtree(rtree_path)
logger.info('Output R-tree: %s' % rtree_path)
# put all trajectories into r-tree in the form of bounding box
node_id = 0
start_time = time.time()
for key, qgrams in trajectory.items():
for qgram in qgrams:
# parameters:
# 1. node id
# 2. bounding box(point): (x,y,x,y)
# 3. data inside each node: trajectory's key from order_dict
x = np.around(qgram[0], decimals=5)
y = np.around(qgram[1], decimals=5)
data_idx.insert(node_id, (x, y, x, y), obj=(id_to_key_dict[key]))
node_id += 1
del data_idx
end_time = time.time()
logger.info("exec time: "+str(end_time-start_time))
logger.info('Finished building R-tree')
def searching(feature_vectors_database, feature_vectors_retrieval,
similarity_metric, image_paths, retrieval_number, file,
list_of_parameters, feature_extraction_method, path_database):
'''
feature_vectors: atriutos calculados
labels: label de cada classe
similarity_metric: qual medida utilizar
recuperados as k imagens com menor distancia. Se k = 0, entao o valor eh
setado como sendo o tamanho da classe da imagem
'''
#name to save the pickle file
parameters_name = ""
for parameter in list_of_parameters:
parameters_name = parameters_name + "_" + parameter
file = path_database + "features/sortingRTree" + "_" + feature_extraction_method + parameters_name + '_' + similarity_metric
feature_vectors_retrieval = preprocessing.scale(feature_vectors_retrieval)
if not (os.path.isfile(file + '.dat')):
#normalize signatures
feature_vectors_database = preprocessing.scale(
feature_vectors_database)
# Create a N-Dimensional index
p = index.Property()
p.dimension = feature_vectors_database.shape[1]
idx = index.Index(file, properties=p)
# Create the tree
for i, vector in enumerate(feature_vectors_database):
idx.add(i, vector.tolist())
#save_format = idx.dumps(idx)
#with open(file, 'wb') as handle:
#pickle.dump(save_format, handle)
else:
# Create a N-Dimensional index
p = index.Property()
p.dimension = feature_vectors_database.shape[1]
idx = Rtree(file, properties=p)
#with open(file, 'rb') as handle:
# idx = pickle.load(handle)
# Find closests pair for the first N points
########### debug this part ###########
small_distances = []
for id1, query in enumerate(feature_vectors_retrieval):
nearest = list(idx.nearest(query.tolist(), retrieval_number))
small_distances.append(nearest)
result = []
for cont1, i in enumerate(small_distances):
aux = []
for j in i:
aux.append(image_paths[j])
result.append(aux)
return result