def all_speeds(folder):
depths = [0, 100, 200]
ships = up.unpickle_ships(folder)
for ship in ships:
ship.sound_speed = []
up.one_jar(folder, ship, False)
for depth in depths:
ships = up.unpickle_ships(folder)
new_ships = specific_depth(ships, depth)
up.store(new_ships, folder)
def get_data(folder):
ships = up.unpickle_ships(folder)
random.shuffle(ships)
train_data_size = 5 * (len(ships) / 6)
test_data_size = len(ships) / 6
train_spect = []
test_spect = []
train_speeds = []
test_speeds = []
speeds_categories = []
i = 0
while len(train_spect) < train_data_size:
train_spect.append(ships[i].spect)
i += 1
while len(test_spect) < test_data_size and i < len(ships):
test_spect.append(ships[i].spect)
i += 1
i = 0
while len(train_speeds) < train_data_size:
train_speeds.append(
ships[i].speed
) #for this run only cpa will refer to soundspeed profile
i += 1
while len(test_speeds) < test_data_size and i < len(ships):
test_speeds.append(ships[i].speed)
i += 1
return train_spect, test_spect, Y_train, Y_test
def generate(rootdir):
print('here')
ships = up.unpickle_ships(rootdir)
i = len(ships)-1
j = 0
while i > 0:
ship = ships[i]
try:
wavfilepath = os.path.join(ship.filepath , ship.id + '.wav') #the original wav file
print(wavfilepath)
#destination = destination_folder + ship.month +'\\' + ship.id + '.png' #the destination for the spectrogram
sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate
frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(1024), noverlap = 0, nfft = 1024, mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results
spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results
ship.spect = spectrogram + np.transpose(uppc) #add tf results to spect results
except:
j+=1
print("bad ship " + str(j))
print(sys.exc_info()[0])
up.one_jar(rootdir,ship,True)
pass
up.one_jar(rootdir,ship,False)
ships.pop(i)
print(i)
i-=1
def get_data(folder):
ships = up.unpickle_ships(folder)
random.shuffle(ships)
train_data_size = 5 * (len(ships) / 6)
test_data_size = len(ships) / 6
train_spect = []
test_spect = []
train_months = []
test_months = []
i = 0
while len(train_spect) < train_data_size:
train_spect.append(ships[i].spect)
i += 1
while len(test_spect) < test_data_size and i < len(ships):
test_spect.append(ships[i].spect)
i += 1
i = 0
while len(train_months) < train_data_size:
train_months.append(
ships[i].month
) #for this run only cpa will refer to soundspeed profile
i += 1
while len(test_months) < test_data_size and i < len(ships):
test_months.append(ships[i].month)
i += 1
Y_train = np_utils.to_categorical(train_months, 12)
Y_test = np_utils.to_categorical(test_months, 12)
return train_spect, test_spect, Y_train, Y_test, 12
def find_ship(folder):
bad_ships = []
ships = up.unpickle_ships(folder)
for ship in ships:
txtfile = ship.filepath + ship.id + '.txt' #the original txt file
converted_times,start,cpa_index,cpa_time,bad = convert_time(ship) #convert all times and find the file start time and cpa time
if(bad):
bad_ships.append(ship)
print(txtfile)
print(len(bad_ships))
print(len(ships))
print(len(bad_ships)/len(ships))
def normalize(folder):
folder = 'D:\PickledData\\'
ships = up.unpickle_ships(folder)
sizes = get_sizes(ships)
max_size = max(sizes)
for i in range(len(ships)):
pad_length = max_size - sizes[i]
ships[i].spect = np.pad(ships[i].spect, [(0, 0), (0, pad_length)],
mode='constant')
print(ships[i].spect.shape)
print(i)
up.one_jar(folder, ships[i], False)
def correlations():
ships = up.unpickle_ships(folder)
feats, means = collect(ships)
nans = np.logical_or(np.isnan(feats), np.isnan(means))
infs = np.logical_or(np.isinf(feats), np.isinf(means))
nans_infs = np.logical_or(nans, infs)
for i in range(len(nans_infs)):
if nans_infs[i]:
print(ships[i].id)
feats.pop(i)
means.pop(i)
corr = pearsonr(means, feats)
print("Correlation between datetime and mean spect: " + str(corr))
def get_data(folder):
ships = up.unpickle_ships(folder)
random.shuffle(ships)
train_data_size = 5 * (len(ships) / 6)
test_data_size = len(ships) / 6
train_spect = []
test_spect = []
train_speeds = []
test_speeds = []
speeds_categories = []
i = 0
while len(train_spect) < train_data_size:
train_spect.append(ships[i].spect)
i += 1
while len(test_spect) < test_data_size and i < len(ships):
test_spect.append(ships[i].spect)
i += 1
i = 0
while len(train_speeds) < train_data_size:
train_speeds.append(
ships[i].speed
) #for this run only cpa will refer to soundspeed profile
i += 1
while len(test_speeds) < test_data_size and i < len(ships):
test_speeds.append(ships[i].speed)
i += 1
min_speed = min(train_speeds)
max_speed = max(train_speeds)
even_speeds = np.arange(min_speed, max_speed, .25)
speed_bins_train = np.digitize(train_speeds, even_speeds)
speed_bins_test = np.digitize(test_speeds, even_speeds)
y_train = []
y_test = []
for a in range(len(train_speeds)):
y_train.append(speed_bins_train[a] - 1)
for b in range(len(test_speeds)):
y_test.append(speed_bins_test[b] - 1)
Y_train = np_utils.to_categorical(y_train, len(even_speeds))
Y_test = np_utils.to_categorical(y_test, len(even_speeds))
return train_spect, test_spect, Y_train, Y_test, len(even_speeds)
def match_ships(filepath):
ships = up.unpickle_ships("D:\PickledData\\")
samples = get_samples(filepath)
for ship in ships:
ship.temp = 0
if isinstance(ship.temp, int):
closest = timedelta(days=100)
i = 0
for sample in samples:
this = abs(sample.date - ship.cpa_datetime)
if this < closest:
closest = this
index = i
i += 1
ship.depth = samples[index].depths
print(ship.depth)
ship.temp = samples[index].temps
ship.sal = samples[index].sals
#print(ship.temp)
up.store(ships, "D:\PickledData\\")
def calc_speed(z, s, t):
a1 = 1448.96
a2 = 4.591
a3 = (-5.304 * (10**(-2)))
a4 = (2.374 * (10**(-4)))
a5 = 1.340
a6 = (1.630 * (10**(-2)))
a7 = (1.675 * (10**(-7)))
a8 = (-1.025 * (10**(-2)))
a9 = (-7.139 * (10**(-13)))
speed = a1 + a2 * t + a3 * (t**2) + a4 * (t**3) + a5 * (
s - 35) + a6 * z + a7 * (z**2) + a8 * t * (s - 35) + a9 * t * (z**3)
return speed
ships = up.unpickle_ships(folder)
for ship in ships:
deepest_temp = float(ship.temp[len(ship.temp) - 1])
print(deepest_temp)
deepest_sal = float(ship.sal[len(ship.sal) - 1])
print(deepest_sal)
deepest_depth = float(ship.depth[len(ship.depth) - 1])
print(deepest_depth)
ship.cpa = calc_speed(
deepest_depth, deepest_sal,
deepest_temp) #for this run only cpa refers to sound speed
up.store(ships, folder)
def generate(folder):
ships = up.unpickle_ships(folder)
bad_apples = []
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file
destination = destination_folder + ship.year_month + '\\' + ship.id + '.png' #the destination for the spectrogram
print(txtfilepath)
converted_times, start, cpa_index, cpa_time = convert_time(
ship
) #convert all times and find the file start time and cpa time
#print(start)
#print(converted_times)
#print(cpa_time)
#print(cpa_index)
after_start = ship.distance[
start:] #find all distances after file_time
# print(len(after_start))
# print(after_start)
closest_range = np.min(
np.abs(after_start)) # find closest point of approach (cpa)
cpa_index = after_start.index(closest_range)
# print(cpa_index)
pre_cpa = after_start[:cpa_index]
post_cpa = after_start[
cpa_index:] #find all distances after cpa time
# print(len(pre_cpa))
pre_times = converted_times[:cpa_index]
# print(len(pre_times))
post_times = converted_times[cpa_index:]
# print(post_times)
#print(pre_cpa)
#print(post_cpa)
approach_inter = interpolate.interp1d(pre_times,
pre_cpa,
axis=0,
fill_value="extrapolate")
depart_inter = interpolate.interp1d(post_times,
post_cpa,
axis=0,
fill_value="extrapolate")
sample_rate, samples = wavfile.read(
wavfilepath
) #get original wav file samples at the original sample rate
sound_length = len(samples) // sample_rate
#print(sound_length)
approach_times = np.arange(0, cpa_time)
depart_times = np.arange(cpa_time, sound_length)
frequencies, times, spectrogram = signal.spectrogram(
samples,
sample_rate,
window=np.hanning(10e3),
noverlap=0,
nfft=10e3,
mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,
frequencies) #get the transfer function results
spectrogram = 10 * np.log10(
spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(
uppc, np.size(spectrogram, 1), 1
) #copy tf results several times to make it same size as spect results
spectrogram = spectrogram + np.transpose(
uppc) #add tf results to spect results
range_step = .01 # step size of 1m
range_approach = (
(np.arange(pre_cpa[0], closest_range, -range_step))
) # make a vector of distances between first range and cpa
range_depart = (
np.arange(closest_range, post_cpa[len(post_cpa) - 1],
range_step)
) # make a vector of distances between cpa and last range
range_desired = np.append(range_approach,
range_depart) # stick them together
number_range_samples = len(
range_desired
) # total length is the number of samples we expect.
#print(spectrogram.shape)
spect_dis_approach = approach_inter(approach_times)
spect_dis_depart = depart_inter(depart_times)
approach_bins = np.digitize(spect_dis_approach, range_approach)
depart_bins = np.digitize(spect_dis_depart, range_depart)
approach_spect = range_spect(approach_bins, spectrogram)
depart_spect = range_spect(depart_bins, spectrogram)
#print(approach_spect.shape)
#print(depart_spect.shape)
#print(spectrogram)
#print(times)
#print(times.shape)
range_spectrogram = np.concatenate((approach_spect, depart_spect),
axis=1)
ship.spect = range_spectrogram
normal = True
#ranges = get_ranges(approach_bins,depart_bins,range_approach,range_depart)
#print(range_spectrogram)
#print(ranges)
#plt.yscale('log') #make y scale log to match the new decibel units
#axes = plt.gca() #get axes object
#axes.set_ylim([10,1000]) #set upper limit of data on axes to be 1000
# plt.pcolormesh(ranges,frequencies,range_spectrogram,vmin=60,vmax=110 ) #plot the data and add color
# plt.set_cmap('jet')
# plt.ylabel('Frequency [Hz]')
# plt.xlabel('Distance [km]')
# locs, ticks = plt.xticks() #get current time ticks
# new_ticks = get_ticks(ranges,locs)
# plt.xticks(locs,new_ticks)
# plt.colorbar()
#plt.xticks(locs, new_ticks) # Set locations and labels to the distance
#plt.savefig(destination) #save spectrogram at destination
#plt.imshow(spectrogram)
#plt.show() #show plot
#plt.close()
except:
print("bad ship")
up.one_jar(folder, ship, True)
normal = False
pass
# print("bad ships:")
# print(bad_apples)
# print(len(bad_apples))
if normal:
up.one_jar(folder, ship, False)
def add_data(folder):
ships = up.unpickle_ships(folder)
#add month or whatever info you wanna give it
ship.encoded.append(ship.month)
up.store(ships, folder)
def histograms():
ships = up.unpickle_ships(folder)
month_histogram(ships)
speed_histogram(ships)
def generate(folder):
ships = up.unpickle_ships(folder)
bad_apples = []
i = 0
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file
destination = destination_folder + ship.year_month +'\\' + ship.id + '.png' #the destination for the spectrogram
print(txtfilepath)
converted_times,start,cpa_index,cpa_time,normal = convert_time(ship) #convert all times and find the file start time and cpa time
print(normal)
after_start = ship.distance[start:] #find all distances after file_time
closest_range = np.min(np.abs(after_start)) # find closest point of approach (cpa)
cpa_index = after_start.index(closest_range)
if len(after_start) < 20:
normal = False
raise NameError('Too Few Distances')
if normal: #for now only do the ones that are normal
print('here 1')
pre_cpa = after_start[:cpa_index]
pre_times = converted_times[:cpa_index]
approach_inter = interpolate.interp1d(pre_times,pre_cpa, axis=0, fill_value="extrapolate")
post_cpa = after_start[cpa_index:] #find all distances after cpa time
post_times = converted_times[cpa_index:]
depart_inter = interpolate.interp1d(post_times,post_cpa, axis=0, fill_value="extrapolate")
print('here 2')
sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate
sound_length = len(samples)//sample_rate
frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(10e3), noverlap = 0, nfft = 10e3, mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results
spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results
spectrogram = spectrogram + np.transpose(uppc) #add tf results to spect results
range_step = .01 # step size of 1m
if normal:
approach_times = np.arange(0,cpa_time)
range_approach = ((np.arange(pre_cpa[0], closest_range, -range_step))) # make a vector of distances between first range and cpa
depart_times = np.arange(cpa_time,sound_length)
else:
depart_times = np.arange(,sound_length)
range_depart = (np.arange(closest_range, post_cpa[len(post_cpa)-1], range_step)) # make a vector of distances between cpa and last range
#range_desired = np.append(range_approach,range_depart)# stick them together
#number_range_samples = len(range_desired)# total length is the number of samples we expect.
if normal:
print('here 3')
spect_dis_approach = approach_inter(approach_times)
approach_bins = np.digitize(spect_dis_approach,range_approach)
approach_spect = range_spect(approach_bins,spectrogram)
print('here 4')
spect_dis_depart = depart_inter(depart_times)
depart_bins = np.digitize(spect_dis_depart,range_depart)
#depart_bins = depart_bins + last_bin
depart_spect = range_spect(depart_bins,spectrogram)
if normal:
print('here 5')
range_spectrogram = np.concatenate((approach_spect,depart_spect),axis=1)
else:
print('here 6')
range_spectrogram = depart_spect
ship.spect = range_spectrogram
normal = True
#plt.savefig(destination) #save spectrogram at destination
#plt.close()
except:
i+=1
print("bad ship" + str(i))
up.one_jar(folder,ship,True)
normal = False
pass
if normal:
up.one_jar(folder,ship,False)
thirty_sec = 30 * 1000
cpa_time = cpa_time * 1000
print(cpa_time)
if cpa_time > 0:
pre_cpa = og_wav[(cpa_time - thirty_sec):cpa_time]
post_cpa = og_wav[cpa_time:(cpa_time + thirty_sec)]
new_wav = (pre_cpa + post_cpa)
return new_wav, bad
else:
bad = True
return None, bad
ships = up.unpickle_ships(rootdir)
ships = ships[1500:]
i = 0
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
print(wavfilepath)
destination = destination_folder + ship.year_month + '\\' + ship.id + '.wav'
result_array, start_index, cpa_index, cpa_time, normal = convert_time(
ship)
cpa_wav, bad = extract_cpa(cpa_time, wavfilepath)
if bad:
i += 1
print('bad' + str(i))
