def all_speeds(folder): depths = [0, 100, 200] ships = up.unpickle_ships(folder) for ship in ships: ship.sound_speed = [] up.one_jar(folder, ship, False) for depth in depths: ships = up.unpickle_ships(folder) new_ships = specific_depth(ships, depth) up.store(new_ships, folder)
def get_data(folder): ships = up.unpickle_ships(folder) random.shuffle(ships) train_data_size = 5 * (len(ships) / 6) test_data_size = len(ships) / 6 train_spect = [] test_spect = [] train_speeds = [] test_speeds = [] speeds_categories = [] i = 0 while len(train_spect) < train_data_size: train_spect.append(ships[i].spect) i += 1 while len(test_spect) < test_data_size and i < len(ships): test_spect.append(ships[i].spect) i += 1 i = 0 while len(train_speeds) < train_data_size: train_speeds.append( ships[i].speed ) #for this run only cpa will refer to soundspeed profile i += 1 while len(test_speeds) < test_data_size and i < len(ships): test_speeds.append(ships[i].speed) i += 1 return train_spect, test_spect, Y_train, Y_test
def generate(rootdir): print('here') ships = up.unpickle_ships(rootdir) i = len(ships)-1 j = 0 while i > 0: ship = ships[i] try: wavfilepath = os.path.join(ship.filepath , ship.id + '.wav') #the original wav file print(wavfilepath) #destination = destination_folder + ship.month +'\\' + ship.id + '.png' #the destination for the spectrogram sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(1024), noverlap = 0, nfft = 1024, mode='psd') #generate spectrogram uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ? uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results ship.spect = spectrogram + np.transpose(uppc) #add tf results to spect results except: j+=1 print("bad ship " + str(j)) print(sys.exc_info()[0]) up.one_jar(rootdir,ship,True) pass up.one_jar(rootdir,ship,False) ships.pop(i) print(i) i-=1
def get_data(folder): ships = up.unpickle_ships(folder) random.shuffle(ships) train_data_size = 5 * (len(ships) / 6) test_data_size = len(ships) / 6 train_spect = [] test_spect = [] train_months = [] test_months = [] i = 0 while len(train_spect) < train_data_size: train_spect.append(ships[i].spect) i += 1 while len(test_spect) < test_data_size and i < len(ships): test_spect.append(ships[i].spect) i += 1 i = 0 while len(train_months) < train_data_size: train_months.append( ships[i].month ) #for this run only cpa will refer to soundspeed profile i += 1 while len(test_months) < test_data_size and i < len(ships): test_months.append(ships[i].month) i += 1 Y_train = np_utils.to_categorical(train_months, 12) Y_test = np_utils.to_categorical(test_months, 12) return train_spect, test_spect, Y_train, Y_test, 12
def find_ship(folder): bad_ships = [] ships = up.unpickle_ships(folder) for ship in ships: txtfile = ship.filepath + ship.id + '.txt' #the original txt file converted_times,start,cpa_index,cpa_time,bad = convert_time(ship) #convert all times and find the file start time and cpa time if(bad): bad_ships.append(ship) print(txtfile) print(len(bad_ships)) print(len(ships)) print(len(bad_ships)/len(ships))
def normalize(folder): folder = 'D:\PickledData\\' ships = up.unpickle_ships(folder) sizes = get_sizes(ships) max_size = max(sizes) for i in range(len(ships)): pad_length = max_size - sizes[i] ships[i].spect = np.pad(ships[i].spect, [(0, 0), (0, pad_length)], mode='constant') print(ships[i].spect.shape) print(i) up.one_jar(folder, ships[i], False)
def correlations(): ships = up.unpickle_ships(folder) feats, means = collect(ships) nans = np.logical_or(np.isnan(feats), np.isnan(means)) infs = np.logical_or(np.isinf(feats), np.isinf(means)) nans_infs = np.logical_or(nans, infs) for i in range(len(nans_infs)): if nans_infs[i]: print(ships[i].id) feats.pop(i) means.pop(i) corr = pearsonr(means, feats) print("Correlation between datetime and mean spect: " + str(corr))
def get_data(folder): ships = up.unpickle_ships(folder) random.shuffle(ships) train_data_size = 5 * (len(ships) / 6) test_data_size = len(ships) / 6 train_spect = [] test_spect = [] train_speeds = [] test_speeds = [] speeds_categories = [] i = 0 while len(train_spect) < train_data_size: train_spect.append(ships[i].spect) i += 1 while len(test_spect) < test_data_size and i < len(ships): test_spect.append(ships[i].spect) i += 1 i = 0 while len(train_speeds) < train_data_size: train_speeds.append( ships[i].speed ) #for this run only cpa will refer to soundspeed profile i += 1 while len(test_speeds) < test_data_size and i < len(ships): test_speeds.append(ships[i].speed) i += 1 min_speed = min(train_speeds) max_speed = max(train_speeds) even_speeds = np.arange(min_speed, max_speed, .25) speed_bins_train = np.digitize(train_speeds, even_speeds) speed_bins_test = np.digitize(test_speeds, even_speeds) y_train = [] y_test = [] for a in range(len(train_speeds)): y_train.append(speed_bins_train[a] - 1) for b in range(len(test_speeds)): y_test.append(speed_bins_test[b] - 1) Y_train = np_utils.to_categorical(y_train, len(even_speeds)) Y_test = np_utils.to_categorical(y_test, len(even_speeds)) return train_spect, test_spect, Y_train, Y_test, len(even_speeds)
def match_ships(filepath): ships = up.unpickle_ships("D:\PickledData\\") samples = get_samples(filepath) for ship in ships: ship.temp = 0 if isinstance(ship.temp, int): closest = timedelta(days=100) i = 0 for sample in samples: this = abs(sample.date - ship.cpa_datetime) if this < closest: closest = this index = i i += 1 ship.depth = samples[index].depths print(ship.depth) ship.temp = samples[index].temps ship.sal = samples[index].sals #print(ship.temp) up.store(ships, "D:\PickledData\\")
def calc_speed(z, s, t): a1 = 1448.96 a2 = 4.591 a3 = (-5.304 * (10**(-2))) a4 = (2.374 * (10**(-4))) a5 = 1.340 a6 = (1.630 * (10**(-2))) a7 = (1.675 * (10**(-7))) a8 = (-1.025 * (10**(-2))) a9 = (-7.139 * (10**(-13))) speed = a1 + a2 * t + a3 * (t**2) + a4 * (t**3) + a5 * ( s - 35) + a6 * z + a7 * (z**2) + a8 * t * (s - 35) + a9 * t * (z**3) return speed ships = up.unpickle_ships(folder) for ship in ships: deepest_temp = float(ship.temp[len(ship.temp) - 1]) print(deepest_temp) deepest_sal = float(ship.sal[len(ship.sal) - 1]) print(deepest_sal) deepest_depth = float(ship.depth[len(ship.depth) - 1]) print(deepest_depth) ship.cpa = calc_speed( deepest_depth, deepest_sal, deepest_temp) #for this run only cpa refers to sound speed up.store(ships, folder)
def generate(folder): ships = up.unpickle_ships(folder) bad_apples = [] for ship in ships: try: wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file destination = destination_folder + ship.year_month + '\\' + ship.id + '.png' #the destination for the spectrogram print(txtfilepath) converted_times, start, cpa_index, cpa_time = convert_time( ship ) #convert all times and find the file start time and cpa time #print(start) #print(converted_times) #print(cpa_time) #print(cpa_index) after_start = ship.distance[ start:] #find all distances after file_time # print(len(after_start)) # print(after_start) closest_range = np.min( np.abs(after_start)) # find closest point of approach (cpa) cpa_index = after_start.index(closest_range) # print(cpa_index) pre_cpa = after_start[:cpa_index] post_cpa = after_start[ cpa_index:] #find all distances after cpa time # print(len(pre_cpa)) pre_times = converted_times[:cpa_index] # print(len(pre_times)) post_times = converted_times[cpa_index:] # print(post_times) #print(pre_cpa) #print(post_cpa) approach_inter = interpolate.interp1d(pre_times, pre_cpa, axis=0, fill_value="extrapolate") depart_inter = interpolate.interp1d(post_times, post_cpa, axis=0, fill_value="extrapolate") sample_rate, samples = wavfile.read( wavfilepath ) #get original wav file samples at the original sample rate sound_length = len(samples) // sample_rate #print(sound_length) approach_times = np.arange(0, cpa_time) depart_times = np.arange(cpa_time, sound_length) frequencies, times, spectrogram = signal.spectrogram( samples, sample_rate, window=np.hanning(10e3), noverlap=0, nfft=10e3, mode='psd') #generate spectrogram uppc = tf.get_tf(ship.harp, frequencies) #get the transfer function results spectrogram = 10 * np.log10( spectrogram) #convert to/from decibels ? uppc = npmb.repmat( uppc, np.size(spectrogram, 1), 1 ) #copy tf results several times to make it same size as spect results spectrogram = spectrogram + np.transpose( uppc) #add tf results to spect results range_step = .01 # step size of 1m range_approach = ( (np.arange(pre_cpa[0], closest_range, -range_step)) ) # make a vector of distances between first range and cpa range_depart = ( np.arange(closest_range, post_cpa[len(post_cpa) - 1], range_step) ) # make a vector of distances between cpa and last range range_desired = np.append(range_approach, range_depart) # stick them together number_range_samples = len( range_desired ) # total length is the number of samples we expect. #print(spectrogram.shape) spect_dis_approach = approach_inter(approach_times) spect_dis_depart = depart_inter(depart_times) approach_bins = np.digitize(spect_dis_approach, range_approach) depart_bins = np.digitize(spect_dis_depart, range_depart) approach_spect = range_spect(approach_bins, spectrogram) depart_spect = range_spect(depart_bins, spectrogram) #print(approach_spect.shape) #print(depart_spect.shape) #print(spectrogram) #print(times) #print(times.shape) range_spectrogram = np.concatenate((approach_spect, depart_spect), axis=1) ship.spect = range_spectrogram normal = True #ranges = get_ranges(approach_bins,depart_bins,range_approach,range_depart) #print(range_spectrogram) #print(ranges) #plt.yscale('log') #make y scale log to match the new decibel units #axes = plt.gca() #get axes object #axes.set_ylim([10,1000]) #set upper limit of data on axes to be 1000 # plt.pcolormesh(ranges,frequencies,range_spectrogram,vmin=60,vmax=110 ) #plot the data and add color # plt.set_cmap('jet') # plt.ylabel('Frequency [Hz]') # plt.xlabel('Distance [km]') # locs, ticks = plt.xticks() #get current time ticks # new_ticks = get_ticks(ranges,locs) # plt.xticks(locs,new_ticks) # plt.colorbar() #plt.xticks(locs, new_ticks) # Set locations and labels to the distance #plt.savefig(destination) #save spectrogram at destination #plt.imshow(spectrogram) #plt.show() #show plot #plt.close() except: print("bad ship") up.one_jar(folder, ship, True) normal = False pass # print("bad ships:") # print(bad_apples) # print(len(bad_apples)) if normal: up.one_jar(folder, ship, False)
def add_data(folder): ships = up.unpickle_ships(folder) #add month or whatever info you wanna give it ship.encoded.append(ship.month) up.store(ships, folder)
def histograms(): ships = up.unpickle_ships(folder) month_histogram(ships) speed_histogram(ships)
def generate(folder): ships = up.unpickle_ships(folder) bad_apples = [] i = 0 for ship in ships: try: wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file destination = destination_folder + ship.year_month +'\\' + ship.id + '.png' #the destination for the spectrogram print(txtfilepath) converted_times,start,cpa_index,cpa_time,normal = convert_time(ship) #convert all times and find the file start time and cpa time print(normal) after_start = ship.distance[start:] #find all distances after file_time closest_range = np.min(np.abs(after_start)) # find closest point of approach (cpa) cpa_index = after_start.index(closest_range) if len(after_start) < 20: normal = False raise NameError('Too Few Distances') if normal: #for now only do the ones that are normal print('here 1') pre_cpa = after_start[:cpa_index] pre_times = converted_times[:cpa_index] approach_inter = interpolate.interp1d(pre_times,pre_cpa, axis=0, fill_value="extrapolate") post_cpa = after_start[cpa_index:] #find all distances after cpa time post_times = converted_times[cpa_index:] depart_inter = interpolate.interp1d(post_times,post_cpa, axis=0, fill_value="extrapolate") print('here 2') sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate sound_length = len(samples)//sample_rate frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(10e3), noverlap = 0, nfft = 10e3, mode='psd') #generate spectrogram uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ? uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results spectrogram = spectrogram + np.transpose(uppc) #add tf results to spect results range_step = .01 # step size of 1m if normal: approach_times = np.arange(0,cpa_time) range_approach = ((np.arange(pre_cpa[0], closest_range, -range_step))) # make a vector of distances between first range and cpa depart_times = np.arange(cpa_time,sound_length) else: depart_times = np.arange(,sound_length) range_depart = (np.arange(closest_range, post_cpa[len(post_cpa)-1], range_step)) # make a vector of distances between cpa and last range #range_desired = np.append(range_approach,range_depart)# stick them together #number_range_samples = len(range_desired)# total length is the number of samples we expect. if normal: print('here 3') spect_dis_approach = approach_inter(approach_times) approach_bins = np.digitize(spect_dis_approach,range_approach) approach_spect = range_spect(approach_bins,spectrogram) print('here 4') spect_dis_depart = depart_inter(depart_times) depart_bins = np.digitize(spect_dis_depart,range_depart) #depart_bins = depart_bins + last_bin depart_spect = range_spect(depart_bins,spectrogram) if normal: print('here 5') range_spectrogram = np.concatenate((approach_spect,depart_spect),axis=1) else: print('here 6') range_spectrogram = depart_spect ship.spect = range_spectrogram normal = True #plt.savefig(destination) #save spectrogram at destination #plt.close() except: i+=1 print("bad ship" + str(i)) up.one_jar(folder,ship,True) normal = False pass if normal: up.one_jar(folder,ship,False)
thirty_sec = 30 * 1000 cpa_time = cpa_time * 1000 print(cpa_time) if cpa_time > 0: pre_cpa = og_wav[(cpa_time - thirty_sec):cpa_time] post_cpa = og_wav[cpa_time:(cpa_time + thirty_sec)] new_wav = (pre_cpa + post_cpa) return new_wav, bad else: bad = True return None, bad ships = up.unpickle_ships(rootdir) ships = ships[1500:] i = 0 for ship in ships: try: wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file print(wavfilepath) destination = destination_folder + ship.year_month + '\\' + ship.id + '.wav' result_array, start_index, cpa_index, cpa_time, normal = convert_time( ship) cpa_wav, bad = extract_cpa(cpa_time, wavfilepath) if bad: i += 1 print('bad' + str(i))