def test_get_distance(self):
"""Test utils.get_distance"""
self.assertGreater(utils.get_distance(self.point1, self.point2), 0)
self.assertEqual(utils.get_distance(self.point1, self.point1), 0)
self.assertEqual(utils.get_distance(self.point2, self.point2), 0)
first = utils.get_distance(self.point1, self.point2)
second = utils.get_distance(self.point2, self.point1)
self.assertEqual(first, second)
def kmeans(samples, k, cutoff):
"""
kmeans函数
"""
# 随机选k个样本点作为初始聚类中心
init_samples = random.sample(samples, k)
# 创建k个聚类，聚类的中心分别为随机初始的样本点
clusters = [Cluster([sample])
for sample in init_samples] #得到K个簇，每个簇只有一个样本点，就是中心点
# 迭代循环直到聚类划分稳定
n_loop = 0
while True:
# 初始化一组空列表用于存储每个聚类内的样本点
# 开辟三个list ，每一个list是一个类
lists = [[] for _ in clusters] #[[], [], []] ,k个元素的list，
# 开始迭代
n_loop += 1
# 遍历样本集中的每个样本
for sample in samples:
# 计算样本点sample和第一个聚类中心的距离
smallest_distance = get_distance(sample, clusters[0].centroid)
# 初始化属于聚类 0
cluster_index = 0
# 计算和其他聚类中心的距离
for i in range(k - 1):
# 计算样本点sample和聚类中心的距离
distance = get_distance(sample, clusters[i + 1].centroid)
# 如果存在更小的距离，更新距离
if distance < smallest_distance:
smallest_distance = distance
cluster_index = i + 1
# 找到最近的聚类中心，更新所属聚类
lists[cluster_index].append(sample) #归类，即每个样本找到了最近的中心
# 初始化最大移动距离
biggest_shift = 0.0
# 计算本次迭代中，聚类中心移动的距离
for i in range(k):
shift = clusters[i].update(lists[i])
# 记录最大移动距离
biggest_shift = max(biggest_shift, shift)
# 如果聚类中心移动的距离小于收敛阈值，即：聚类稳定
if biggest_shift < cutoff:
print("第{}次迭代后，聚类稳定。".format(n_loop))
break
# 返回聚类结果
return clusters
def update_centroid_direction(self):
"""
Identify the direction in which the centroid has travelled since the
previous timestep
:return:
"""
# Only get a centroid direction if we have a previous centroid and
# haven't merged recently
if self.duration < datetime.timedelta(hours=0.25):
self.centroid_direction = np.nan
self.track_centroid_direction.append(self.centroid_direction)
# elif self.merged == True:
# if (self.dates_observed[-1] - self.merge_date) < \
# datetime.timedelta(hours=0.25):
# self.centroid_direction = np.nan
# self.track_centroid_direction.append(self.centroid_direction)
else:
self.centroid_direction = utilities.\
calculate_initial_compass_bearing((self.track_centroid_lat[-2],
self.track_centroid_lon[-2]),
(self.centroid_lat,
self.centroid_lon))
#print self.centroid_direction
self.track_centroid_direction.append(self.centroid_direction)
axis_offset = utilities.get_distance(self.centroid_direction,
self.primary_axis_direction)
def check_radius_collision(self, target_object, scan_radius=False):
target_loc = (target_object.x, target_object.y)
distance = utilities.get_distance((self.x, self.y), target_loc)
radius = self.radius if not scan_radius else scan_radius
total_radius = radius + target_object.radius
if distance <= total_radius:
return True
return False
def test(dataset, W, b, type, da=None, fvs=None, images_to_indices=None, verbose=False):
if verbose: print 'Getting %s Error...' % type
# tp = really +1, pred +1
# fp = really -1, pred +1
# fn = really +1, pred -1
# tn = really -1, pred -1
tp = fp = fn = tn = 0.
if type == 'Train':
same_data = dataset.gen_same_person_train_samples
diff_data = dataset.gen_diff_person_train_samples
elif type == 'Test':
same_data = dataset.gen_same_person_test_samples
diff_data = dataset.gen_diff_person_test_samples
labels_and_data = ((+1, same_data), (-1, diff_data))
for (label, data) in labels_and_data:
for (img1, img2) in data():
if images_to_indices is not None and img1 in images_to_indices:
fv1 = fvs[:, images_to_indices[img1]]
else:
img1 = dataset.get_fv_file_for_image(img1)
fv1 = utilities.hydrate_fv_from_file(img1)
if da is not None:
fv1 = da.get_hidden_values(fv1).eval()
if images_to_indices is not None and img2 in images_to_indices:
fv2 = fvs[:, images_to_indices[img2]]
else:
img2 = dataset.get_fv_file_for_image(img2)
fv2 = utilities.hydrate_fv_from_file(img2)
if da is not None:
fv2 = da.get_hidden_values(fv2).eval()
dist = utilities.get_distance(W, b, fv1, fv2)
if dist >= 0 and label == +1:
tp += 1
elif dist < 0 and label == +1:
fp += 1
elif dist <= 0 and label == -1:
tn += 1
else:
fn += 1
total = tp + fp + fn + tn
print 'Total: %s' % total
print 'Precision: %s' % (tp / (1e-9 + tp + fp))
print 'Recall: %s' % (tp / (1e-9 + tp + fn))
print 'TP: %s' % tp
print 'FP: %s' % fp
print 'FN: %s' % fn
print 'TN: %s' % tn
return tp, fp, fn, tn
def update(self, samples): #
"""
计算之前的聚类中心和更新后聚类中心的距离
"""
old_centroid = self.centroid
self.samples = samples
self.centroid = self.cal_centroid()
shift = get_distance(old_centroid, self.centroid)
return shift
def update_mean_axis_offset(self):
"""
Calculates the mean offset from parallel between the centroid
direction of the plume and its primary axis
parallel is either a difference closer to zero or closer to 180
perpendicular is a difference closer to 90 or to 270
How about including the standard deviation of the axis offset and
centroid direction? So if there's a really high one we don't bother?
:return:
"""
# Take the smaller of the difference between 180 and the offset and
# 0 and the offset
mean_centroid_direction = np.nanmean(self.track_centroid_direction)
mean_primary_axis_direction = np.nanmean(
self.track_primary_axis_direction)
# Get the direction from the total distance travelled by the plume
first_position_lat = self.track_centroid_lat[0]
first_position_lon = self.track_centroid_lon[0]
last_position_lat = self.track_centroid_lat[-1]
last_position_lon = self.track_centroid_lon[-1]
# Get the direction
final_direction = utilities.calculate_initial_compass_bearing(
(first_position_lat, first_position_lon),
(last_position_lat, last_position_lon))
std_centroid_direction = np.nanstd(self.track_centroid_direction)
std_primary_axis_direction = np.nanstd(
self.track_primary_axis_direction)
#print '\n'
#print std_centroid_direction
#print std_primary_axis_direction
mean_axis_difference = abs(
utilities.get_distance(final_direction,
mean_primary_axis_direction))
diff_180 = abs(mean_axis_difference - 180)
diff_0 = abs(mean_axis_difference)
self.mean_axis_offset = np.min([diff_180, diff_0])
if np.isnan(self.mean_axis_offset):
self.mean_axis_offset = None
def within_distance(self, point, distance=(1.609344) / 2.0):
"""
Returns True if the distance between `self` and `point` is ≤0.5 miles.
:param point: a python dictionary with keys lat and lon corresponding to a geo position
:type: dict
:param distance: Default value is (1.609344)/2.0, which is half a mile
:type: float
:return: True or False
:rtype: bool
"""
d = {'lat': self.latitude, 'lon': self.longitude}
self.distance = get_distance(d, point)
return self.distance <= distance
def create_plume_dataframe(manual_filename, plume_archive_filename):
"""
Creates a pandas dataframe from manually identified dust plumes,
assigning them attributes found by plumetracker
:param manual_filename:
:param plume_archive_filename:
:return:
"""
# Read in the csv
data = pd.read_csv('/ouce-home/students/hert4173/workspace/plumetracker/' +
manual_filename,
header=None,
names=['plume_ID', 'LLJ'])
# Read in the plume archive
plume_archive = shelve.open(plume_archive_filename)
conv_distances = []
axis_direction_offsets = []
durations = []
emission_speeds = []
distances_from_09 = []
max_extents = []
for index, row in data.iterrows():
plume = plume_archive[str(row['plume_ID'])]
conv_distances.append(plume.conv_distance)
#axis_direction_offsets.append(plume.mean_axis_offset)
durations.append(plume.duration.total_seconds())
emission_speeds.append(plume.track_speed_centroid[1])
greatest_extent_ind = np.where(
plume.track_area == np.max(plume.track_area))[0][0]
"""
if greatest_extent_ind > 0 and greatest_extent_ind < len(
plume.track_centroid_lat)-1:
# Extract centroid positions around the greatest extent time
first_position_lat = plume.track_centroid_lat[
greatest_extent_ind-1]
first_position_lon = plume.track_centroid_lon[
greatest_extent_ind-1]
last_position_lat = plume.track_centroid_lat[
greatest_extent_ind+1]
last_position_lon = plume.track_centroid_lon[
greatest_extent_ind+1]
"""
# Get the direction from the total distance travelled by the plume
first_position_lat = plume.track_centroid_lat[0]
first_position_lon = plume.track_centroid_lon[0]
last_position_lat = plume.track_centroid_lat[-1]
last_position_lon = plume.track_centroid_lon[-1]
# Get the direction
final_direction = utilities.calculate_initial_compass_bearing(
(first_position_lat, first_position_lon),
(last_position_lat, last_position_lon))
# Do the same as before for the offset
mean_primary_axis_direction = np.nanmean(
plume.track_primary_axis_direction)
#plt.close()
#plt.contourf(lons, lats, plume.track_plume_bool[
# greatest_extent_ind].toarray())
image = plume.track_plume_bool[greatest_extent_ind].toarray()
label_img = label(image)
regions = regionprops(label_img)
for props in regions:
orientation = props.orientation
orientation = math.degrees(orientation) + 90
u = 1 * np.sin(math.radians(orientation))
v = 1 * np.cos(math.radians(orientation))
u1 = 1 * np.sin(math.radians(final_direction))
v1 = 1 * np.cos(math.radians(final_direction))
#plt.imshow(plume.track_plume_bool[greatest_extent_ind].toarray(),
# origin='lower')
#plt.quiver(plume.track_centroid_lon[greatest_extent_ind],
# plume.track_centroid_lat[greatest_extent_ind], u, v,
# color='r')
#plt.quiver(plume.track_centroid_lon[greatest_extent_ind],
# plume.track_centroid_lat[greatest_extent_ind], u1, v1,
# color='g')
#plt.savefig('Orientation_check_plot_'+str(plume.plume_id)+'.png')
#plt.close()
mean_axis_difference = abs(
utilities.get_distance(final_direction, orientation))
#print 'Direction travelled', final_direction
#print 'Orientation of axis', orientation
diff_180 = abs(mean_axis_difference - 180)
diff_0 = abs(mean_axis_difference)
mean_axis_offset = np.min([diff_180, diff_0])
axis_direction_offsets.append(mean_axis_offset)
#if str(row['plume_ID']) == '375':
# print 'Orientation', orientation
# print 'Final direction', final_direction
#print mean_axis_offset
emission_time = plume.dates_observed[0]
# Take the smallest time to the nearest 0900UTC
nine = emission_time.replace(hour=9, minute=0, second=0)
nine_1 = nine + dt.timedelta(days=1)
distance_nine = abs(emission_time - nine)
distance_nine_1 = abs(nine_1 - emission_time)
distance_from_09 = np.min(
[distance_nine.total_seconds(),
distance_nine_1.total_seconds()])
distances_from_09.append(distance_from_09)
max_extents.append(np.max(plume.track_area))
data['conv_distance'] = conv_distances
data['axis_direction_offset'] = axis_direction_offsets
data['duration'] = durations
#data['emission_speed'] = emission_speeds
data['time_to_09'] = distances_from_09
data['max_extent'] = max_extents
# Remove the plume ID field
del data['plume_ID']
# Close shelf
plume_archive.close()
# For each identified ID, create a row in a pandas data frame and
# extract the equivalent attributes from the plume archive
# Save the data frame as a csv
return data
def test_distance(self):
point_a = (0, 0)
point_b = (4, 3)
distance = get_distance(point_a, point_b)
self.assertEqual(distance, 5, "Distance should be 5")