def defineQuery(epsilon, approx=True, distributed=False, margin=0):
element_list = [
Element(1, 340.125920709671, 3.35842462611196, 15.88071, 0, 14.94726, 0, 14.25543, 0, 13.81744, 0, 13.52653,
0, 0, False),
Element(2, 340.125919636357, 3.35841548819963, 15.88897, 0, 14.96084, 0, 14.27048, 0, 13.83855, 0, 13.55785,
0, 0, False),
Element(3, 340.125941094769, 3.35841455436284, 15.84727, 0, 14.95589, 0, 14.26131, 0, 13.8294, 0, 13.52057,
0, 0, False),
Element(4, 340.125942982763, 3.3584407242725, 15.98358, 0, 14.94892, 0, 14.26593, 0, 13.8352, 0, 13.5495,
0, 0,False)]
anchorElement = Query.getCentroidFromElementList(element_list)
#print("Query Anchor: %s" % anchorElement.getId())
#print("Query Epsilon: %s"% epsilon)
qSize = len(element_list)
minDist = 1000
maxDist = 0
minDistElement = None
distM = [[0 for n in range(qSize + 1)] for m in range(qSize + 1)]
used = []
for element in element_list:
used.append(element)
if element != anchorElement:
element.distToCentroid = EucDist(anchorElement.Ra, element.Ra, anchorElement.Dec, element.Dec)
if element.distToCentroid < minDist:
minDistElement = element
minDist = element.distToCentroid
element_list_comp = list(set(element_list) - set(used))
for elementj in element_list_comp:
dist = EucDist(element.Ra, elementj.Ra, element.Dec, elementj.Dec)
distM[element.getId()][elementj.getId()] = dist
distM[elementj.getId()][element.getId()] = dist
if dist > maxDist:
maxDist = dist
# print("Distance to query centroid: queryid %s dist %s" % (element.getId(), element.distToCentroid))
#print ("*************** Matrix **************")
#for i in range(qSize + 1):
# for j in range(qSize + 1):
# print ("Posicao i: %s j: %s Val: %s" % (i,j,distM[i][j]))
#print("*************** End Matrix ***************")
# Compute pairwise distances
# #order the query points
element_list.sort(key=lambda elements: elements.distToCentroid)
# minimum distance is in the second element. First is zero
queryDefinition = Query(element_list, anchorElement, minDist, minDistElement, distM, epsilon, approx, distributed, margin, maxDist)
return queryDefinition
def checkScale(self, candidatesolution, margin=0, epsilon=0):
self.starsTuple = candidatesolution.getStars()
self.metadataTuple = candidatesolution.getMetadata()
self.margin = margin
k = 0
for i in range(len(self.starsTuple)):
rai = self.starsTuple[i].getRa()
deci = self.starsTuple[i].getDec()
if i == 0:
reli = self.anchorElement
else:
reli = self.metadataTuple[i]
for j in range(i + 1, len(self.starsTuple)):
raj = self.starsTuple[j].getRa()
decj = self.starsTuple[j].getDec()
relj = self.metadataTuple[j]
dist = EucDist(rai, raj, deci, decj)
if k == 0:
k = dist / self.distM[reli][relj]
else:
ratio = dist / self.distM[reli][relj]
if not (round(ratio) <= round(k) + epsilon):
if not (round(ratio) >= round(k) - epsilon):
return False
return True
def getNext(self):
epsilon = self.query.getEpsilon() * self.query.getMaxDistance()
listTuple = []
newListTuple = []
if self.leftRelation is None:
listTuple = self.leftJ.getNext()
if listTuple == False:
return listTuple
elif listTuple == "end":
return listTuple
else:
if self.starIndex == self.sizeLeftleave:
listTuple = "end"
return listTuple
else:
listTuple.append(self.stars[self.starIndex])
self.starIndex += 1
for l in range(len(listTuple)):
genTuple = listTuple[l]
tuple = Tuple()
if self.leftJ is None:
tuple.addStar(genTuple[0], self.leftRelation.getRelationId())
tuple.addStar(genTuple[1], self.leftRelation.getRelationId())
else:
tuple.addTuple(genTuple)
anchor = tuple.getStar(0)
idAnchor = anchor.getId()
distM = self.query.getDistanceMatrix()
for t in self.rightR.getStars():
match = True
rightId = t[0].getId()
if idAnchor == rightId:
for i in range(
len(tuple.getStars()) - 1
): # Checks pairwise distances with all elements in tuple
dist = EucDist(
tuple.getStar(i + 1).getRa(), t[1].getRa(),
tuple.getStar(i + 1).getDec(), t[1].getDec())
distanceMatrix = distM[self.rightR.getRelationId()][
tuple.getMetadataPos(i + 1)]
if (distanceMatrix -
epsilon) <= dist <= (distanceMatrix + epsilon):
s = tuple.getStar(i + 1)
if s.getId() == t[1].getId():
match = False
else:
match = False
break
if match:
newTuple = Tuple()
newTuple.addTuple(tuple)
newTuple.addStar(t[1], self.rightR.getRelationId())
newListTuple.append(newTuple)
return newListTuple
def build_tree(text):
query = query_broadcast.value
root = None
geometric_centroid_ra = geometric_centroid_dec = None
centroid = None
cent_min_dist = float("inf")
voxel = None
for lines in text:
for line in lines[1].split("\n"):
split = line.split(",")
if len(split) == 4:
min_ra, max_ra, min_dec, max_dec = split
voxel = Voxel(float(min_ra), float(max_ra), float(min_dec),
float(max_dec))
geometric_centroid_ra, geometric_centroid_dec = voxel.getVoxelCentroid(
)
root = Node(voxel)
elif line:
border = False if split[13].lower() == "false" else True
'''star = Element(int(split[0]), float(split[1]), float(split[2]), float(split[3]), float(split[4]),
float(split[5]), float(split[6]), float(split[7]), float(split[8]),
float(split[9]), float(split[10]), float(split[11]), float(split[12]), 0, border)'''
star = Element(int(split[0]), float(split[1]), float(split[2]),
float(split[3]), 0, border)
root.addElement(star)
if star.border is False:
dist = EucDist(star.getRa(), geometric_centroid_ra,
star.getDec(), geometric_centroid_dec)
if dist < cent_min_dist:
centroid = star
cent_min_dist = dist
root.setSize(len(root.getElements()))
root.addCentroid(centroid)
level = compute_level(voxel.getSideSize(), voxel.getHeightSize(),
query.getMaxDistance())
tree = QuadTree(root, level)
print("\n**** Data Descriptions *****")
print("Sky Voxel: %s,%s,%s,%s" %
(voxel.x_left, voxel.x_right, voxel.y_left, voxel.y_right))
print("Sky Diagonal: %s" % voxel.getDiagonal())
print("Tree Level: %s" % level)
print("Tree Elements: %s" % root.size)
print("Tree Leaf nodes: %s" % len(tree.nodes))
print("**** End Data Descriptions *****\n")
return [tree]
def produce_candidates_color(tree):
query = query_broadcast.value
query_elements = query.getQuery()
query_anchor_id = query.getAnchor().getId()
query_max_dist = query.getMaxDistance()
epsilon = query.getEpsilon() * query_max_dist
query_matrix_distance = query.getDistanceMatrix()
root = tree.root
nodes = tree.nodes
relations = defaultdict(set)
for node in nodes:
anchors = node.getElements()
neighbors = tree.find_neighbors(node, root, query_max_dist + epsilon,
[])
neighbors_size = len(neighbors)
if neighbors_size > 0:
for anchor in anchors:
u_err = anchor.u_err
g_err = anchor.g_err
r_err = anchor.r_err
i_err = anchor.i_err
z_err = anchor.z_err
for neighbor in neighbors:
if neighbor.pointId != anchor.pointId:
if anchor.u - u_err <= neighbor.u <= anchor.u + u_err and \
anchor.g - g_err <= neighbor.g <= anchor.g + g_err and \
anchor.r - r_err <= neighbor.r <= anchor.r + r_err and \
anchor.i - i_err <= neighbor.i <= anchor.i + i_err and \
anchor.z - z_err <= neighbor.z <= anchor.z + z_err:
distance = EucDist(anchor.getRa(),
neighbor.getRa(),
anchor.getDec(),
neighbor.getDec())
for e in query_elements:
eid = e.getId()
if eid != query_anchor_id:
query_distance = query_matrix_distance[
query_anchor_id][eid]
if query_distance - epsilon <= distance <= query_distance + epsilon:
relations[anchor.pointId].add(
(anchor, neighbor, eid))
node.visited = True
return relations.values()
def find_neighbors(self, node_search, node, max_distance, neighbor_list):
if node.visited is False:
node_centroid = node.getCentroid()
node_diagonal = node.voxel.getDiagonal() / 2
dist = EucDist(node_search.Ra, node_centroid.Ra, node_search.Dec, node_centroid.Dec)
if dist <= max_distance + node_diagonal:
if len(node.children) > 0:
for child in node.children:
self.find_neighbors(node_search, child, max_distance, neighbor_list)
else:
if node.getCentroid().getId() != node_search.getId():
neighbor_list.extend(node.elements)
node.mark_visited()
return neighbor_list
def partial_match(self, node_j, distance, epsilon):
voxel_i = self.getVoxel()
size_i = voxel_i.getSideSize()
centroid_i = self.getCentroid()
centroid_j = node_j.getCentroid()
dist = EucDist(centroid_i.getRa(), centroid_j.Ra, centroid_i.getDec(), centroid_j.getDec())
if size_i < distance:
if distance + epsilon + (size_i * sqrt(2) >= dist >= distance - epsilon - (size_i * sqrt(2))):
match = True
else:
match = False
else:
if (distance + epsilon) >= dist >= (distance - epsilon):
match = True
else:
match = False
return match
def build_tree(filename, query):
start_time = time.time()
root = None
geometric_centroid_ra = geometric_centroid_dec = None
centroid = None
cent_min_dist = float("inf")
voxel = None
with open(filename) as f:
for line in f:
split = line.replace("\n", "").split(",")
if len(split) == 4:
min_ra, max_ra, min_dec, max_dec = split
voxel = Voxel(float(min_ra), float(max_ra), float(min_dec), float(max_dec))
geometric_centroid_ra, geometric_centroid_dec = voxel.getVoxelCentroid()
root = Node(voxel)
elif line:
border = False # if split[13].lower() == "false" else True
star = Element(int(split[0]), float(split[1]), float(split[2]), float(split[3]), float(split[4]),
float(split[5]), float(split[6]), float(split[7]), float(split[8]),
float(split[9]), float(split[10]), float(split[11]), float(split[12]), 0, border)
root.addElement(star)
if star.border is False:
dist = EucDist(star.getRa(), geometric_centroid_ra, star.getDec(), geometric_centroid_dec)
if dist < cent_min_dist:
centroid = star
cent_min_dist = dist
root.setSize(len(root.getElements()))
root.addCentroid(centroid)
level = compute_level(voxel.getSideSize(), voxel.getHeightSize(), query.getMaxDistance())
tree = QuadTree(root, level)
end_time = time.time() - start_time
print("BT - %s - %0.25f" % (filename, end_time))
return tree
def build_tree(text):
#query = query_broadcast.value
root = None
geometric_centroid_ra = geometric_centroid_dec = None
centroid = None
cent_min_dist = float("inf")
voxel = None
for i in range(1, len(text)): # skip first line
#for line in lines[1].split("\n"):
split = text[i].split(",")
if len(split) == 4:
min_ra, max_ra, min_dec, max_dec = split
voxel = Voxel(float(min_ra), float(max_ra), float(min_dec),
float(max_dec))
geometric_centroid_ra, geometric_centroid_dec = voxel.getVoxelCentroid(
)
root = Node(voxel)
elif text[i]:
star = Element(int(split[0]), float(split[1]), float(split[2]),
float(split[3]), 0, split[4])
root.addElement(star)
dist = EucDist(star.getRa(), geometric_centroid_ra, star.getDec(),
geometric_centroid_dec)
if dist < cent_min_dist:
centroid = star
cent_min_dist = dist
root.setSize(len(root.getElements()))
root.addCentroid(centroid)
level = compute_level(voxel.getSideSize(), voxel.getHeightSize(),
0.00416667)
tree = QuadTree(root, level)
return tree
def getCentroidFromElementList(element_list):
qSize = len(element_list)
avg_Ra = 0
avg_Dec = 0
for element in element_list:
avg_Ra += element.Ra
avg_Dec += element.Dec
avg_Ra /= qSize
avg_Dec /= qSize
# #Find the nearest query point as the centroid
min_dist = 100000
tmp_dist = 0
centroid = None
for element in element_list:
tmp_dist = EucDist(element.Ra, avg_Ra, element.Dec, avg_Dec)
if tmp_dist < min_dist:
min_dist = tmp_dist
centroid = element
# print ("centroid point is the q" + (str)(centroid_index))
return centroid
def split_node(self):
level = self.level + 1
voxel = self.getVoxel()
centroid_voxel = voxel.getVoxelCentroid()
voxel11 = Voxel(voxel.x_left, centroid_voxel[0], centroid_voxel[1], voxel.y_right)
quarter11 = Node(voxel11, None, self, level)
voxel12 = Voxel(centroid_voxel[0], voxel.x_right, centroid_voxel[1], voxel.y_right)
quarter12 = Node(voxel12, None, self, level)
voxel01 = Voxel(voxel.x_left, centroid_voxel[0], voxel.y_left, centroid_voxel[1])
quarter01 = Node(voxel01, None, self, level)
voxel02 = Voxel(centroid_voxel[0], voxel.x_right, voxel.y_left, centroid_voxel[1])
quarter02 = Node(voxel02, None, self, level)
star11_cent = star12_cent = star01_cent = star02_cent = float("inf")
star11_med_x = star11_med_y = star12_med_x = star12_med_y = star01_med_x = star01_med_y = star02_med_x = \
star02_med_y = 0
tot_star11 = tot_star12 = tot_star01 = tot_star02 = 0
border_tot_star11 = border_tot_star12 = border_tot_star01 = border_tot_star02 = 0
for _ in range(len(self.getElements())):
star = self.elements.popleft()
if star.getRa() < centroid_voxel[0]:
if star.getDec() < centroid_voxel[1]:
quarter01.addElement(star)
if star.border is False:
star01_med_x += star.getRa()
star01_med_y += star.getDec()
tot_star01 += 1
else:
border_tot_star01 += 1
else:
quarter11.addElement(star)
if star.border is False:
star11_med_x += star.getRa()
star11_med_y += star.getDec()
tot_star11 += 1
else:
border_tot_star11 += 1
else:
if star.getDec() > centroid_voxel[1]:
quarter12.addElement(star)
if star.border is False:
star12_med_x += star.getRa()
star12_med_y += star.getDec()
tot_star12 += 1
else:
border_tot_star12 += 1
else:
quarter02.addElement(star)
if star.border is False:
star02_med_x += star.getRa()
star02_med_y += star.getDec()
tot_star02 += 1
else:
border_tot_star02 += 1
node11_cent = node12_cent = node01_cent = node02_cent = None
nodes = []
if tot_star11 > 0:
star11_med_x = star11_med_x / tot_star11
star11_med_y = star11_med_y / tot_star11
for star in quarter11.getElements():
dist = EucDist(star.getRa(), star11_med_x, star.getDec(), star11_med_y)
if star.border is False and dist < star11_cent:
node11_cent = star
star11_cent = dist
quarter11.centroidStar = node11_cent
size = tot_star11 + border_tot_star11
quarter11.setSize(size)
self.addChildren(quarter11)
nodes.append(quarter11)
if tot_star12 > 0:
star12_med_x = star12_med_x / tot_star12
star12_med_y = star12_med_y / tot_star12
for star in quarter12.getElements():
dist = EucDist(star.getRa(), star12_med_x, star.getDec(), star12_med_y)
if star.border is False and dist < star12_cent:
node12_cent = star
star12_cent = dist
quarter12.centroidStar = node12_cent
size = tot_star12 + border_tot_star12
quarter12.setSize(size)
self.addChildren(quarter12)
nodes.append(quarter12)
if tot_star01 > 0:
star01_med_x = star01_med_x / tot_star01
star01_med_y = star01_med_y / tot_star01
for star in quarter01.getElements():
dist = EucDist(star.getRa(), star01_med_x, star.getDec(), star01_med_y)
if star.border is False and dist < star01_cent:
node01_cent = star
star01_cent = dist
quarter01.centroidStar = node01_cent
size = tot_star01 + border_tot_star01
quarter01.setSize(size)
self.addChildren(quarter01)
nodes.append(quarter01)
if tot_star02 > 0:
star02_med_x = star02_med_x / tot_star02
star02_med_y = star02_med_y / tot_star02
for star in quarter02.getElements():
dist = EucDist(star.getRa(), star02_med_x, star.getDec(), star02_med_y)
if star.border is False and dist < star02_cent:
node02_cent = star
star02_cent = dist
quarter02.centroidStar = node02_cent
size = tot_star02 + border_tot_star02
quarter02.setSize(size)
self.addChildren(quarter02)
nodes.append(quarter02)
return nodes
input_dim = len(X['P1'][0].split())
# Model variables
# Define the shared model
x = Sequential()
x.add(Dense(40, activation='relu'))
x.add(Dense(DR_dim, activation='relu'))
shared_model = x
# The visible layer
left_input = Input(shape=(input_dim, ), dtype='float32')
right_input = Input(shape=(input_dim, ), dtype='float32')
# Pack it all up into a Manhattan Distance model
malstm_distance = EucDist()(
[shared_model(left_input),
shared_model(right_input)])
model = Model(inputs=[left_input, right_input], outputs=[malstm_distance])
if gpus >= 2:
model = keras.utils.multi_gpu_model(model, gpus=gpus)
#model.compile(loss='mean_squared_error', optimizer='sgd', metrics=['mse'])
model.compile(loss='mean_squared_error', optimizer='sgd')
#model.compile(loss=custom_loss, optimizer='sgd')
#model.summary()
# Start trainings
training_start_time = time()
#malstm_trained = model.fit(X_train, Y_train,