def normalize(folder):
x = 0
y = 0
for ships in up.unpickle_batch(folder, 50, 4200, 9000):
#print(np.size(ships))
#print(ships[0].spect.shape)
sizes = get_sizes(ships)
#avg_size = int(np.mean(sizes))
square_size = np.size(ships[0].spect, 0)
for i in tqdm(range(len(ships)),
"This Batch: ",
bar_format="{l_bar}%s{bar}%s{r_bar}" %
(Fore.YELLOW, Fore.RESET)):
try:
pad_length = (square_size - sizes[i])
beg_pad, end_pad = calc_pad(square_size, sizes[i])
if (pad_length >= 0):
ships[i].spect = np.pad(ships[i].spect,
[(0, 0), (beg_pad, end_pad)],
mode='constant')
else:
ships[i].spect = ships[i].spect[:,
abs(beg_pad):sizes[i] +
end_pad]
# print(ships[i].spect.shape)
# print(x)
x += 1
up.one_jar(folder, ships[i], False)
except:
y += 1
up.one_jar(folder, ships[i], True)
tqdm.write(str(y))
pass
def generate(rootdir):
print('here')
ships = up.unpickle_ships(rootdir)
i = len(ships)-1
j = 0
while i > 0:
ship = ships[i]
try:
wavfilepath = os.path.join(ship.filepath , ship.id + '.wav') #the original wav file
print(wavfilepath)
#destination = destination_folder + ship.month +'\\' + ship.id + '.png' #the destination for the spectrogram
sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate
frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(1024), noverlap = 0, nfft = 1024, mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results
spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results
ship.spect = spectrogram + np.transpose(uppc) #add tf results to spect results
except:
j+=1
print("bad ship " + str(j))
print(sys.exc_info()[0])
up.one_jar(rootdir,ship,True)
pass
up.one_jar(rootdir,ship,False)
ships.pop(i)
print(i)
i-=1
def find_done(folder):
i = 0
for ships in up.unpickle_batch(folder, 100, 6500, 9000):
for ship in ships:
if ship.spect is None:
i += 1
up.one_jar(folder, ship, True)
tqdm.write(str(i))
pass
def all_speeds(folder):
depths = [0, 100, 200]
ships = up.unpickle_ships(folder)
for ship in ships:
ship.sound_speed = []
up.one_jar(folder, ship, False)
for depth in depths:
ships = up.unpickle_ships(folder)
new_ships = specific_depth(ships, depth)
up.store(new_ships, folder)
def normalize(folder):
folder = 'D:\PickledData\\'
ships = up.unpickle_ships(folder)
sizes = get_sizes(ships)
max_size = max(sizes)
for i in range(len(ships)):
pad_length = max_size - sizes[i]
ships[i].spect = np.pad(ships[i].spect, [(0, 0), (0, pad_length)],
mode='constant')
print(ships[i].spect.shape)
print(i)
up.one_jar(folder, ships[i], False)
def match_ships(rootdir,csv):
i=0
data_frame = get_data(csv)
for ships in up.unpickle_batch(rootdir,100,9000,9100):
for ship in ships:
# if isinstance(ship.sal, int):
print(ship.id + ': ' + str(i))
found,sample = find_best_sample(data_frame,ship.cpa_datetime)
sample = fix_sample(sample)
if found:
ship.depth = sample.depths
ship.temp = sample.temps
ship.sal = sample.sals
ship.IMO = sample.date
up.one_jar(rootdir,ship,False)
print('pickled: ' + str(i))
i+=1
def all_speeds(folder):
highest = 0
lowest = 99999 #temps to check validity of data
speed_list = []
for ships in up.unpickle_batch(folder, 100, 0, 9000):
for ship in ships:
ship.depth = np.array([float(i) for i in ship.depth])
ii = np.where(ship.depth == 1)[0]
if len(ii) > 0:
index = ii[0]
depth_400 = float(ship.depth[index])
temp_400 = float(ship.temp[index])
sal_400 = float(ship.sal[index])
# print(depth_400)
# print(temp_400)
# print("sal" + str(sal_400))
ship.sound_speed = calc_speed(
depth_400, sal_400,
temp_400) #for this run only cpa refers to sound speed
s = ship.sound_speed
if s > highest:
highest = s
if s < lowest:
lowest = s
speed_list.append(s)
else:
print(ship.id)
up.one_jar(folder, ship, False)
print(highest)
print(lowest)
mean, dev = std_dev(speed_list)
print(mean)
print(dev)
u_speeds = np.unique(np.array(speed_list))
print(u_speeds)
print(len(u_speeds))
def main(rootdir,destination):
i = 0
mmsis = extract_mmsi('D:\VZDATAALL.csv')
for ships in up.unpickle_batch(rootdir, 100, 400, 500):
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
destination = destination_folder + ship.year_month +'\\' + ship.id + '.wav'
skip = get_length(ship,mmsis)
converted_times, start_index, cpa_index,cpa_time,normal = convert_time(ship)
cpa_sog = ship.SOG[cpa_index]
if (not normal):
i+=1
print('bad ' + str(i))
elif skip:
i+=1
print('mmsi not included ' + str(i))
else:
distances, new_times = new_distances(ship, start_index, converted_times)
pre, post = find_ship_passage(ship,distances,new_times,cpa_time,cpa_sog)
pass_wav = cut_wav(pre,post,wavfilepath)
pass_wav.export(destination,format="wav")
print(wavfilepath)
pass_wav.export(destination,format="wav")
except:
up.one_jar(rootdir,ship,True)
print('something went wrong')
pass
def generate(folder):
ships = up.unpickle_ships(folder)
bad_apples = []
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file
destination = destination_folder + ship.year_month + '\\' + ship.id + '.png' #the destination for the spectrogram
print(txtfilepath)
converted_times, start, cpa_index, cpa_time = convert_time(
ship
) #convert all times and find the file start time and cpa time
#print(start)
#print(converted_times)
#print(cpa_time)
#print(cpa_index)
after_start = ship.distance[
start:] #find all distances after file_time
# print(len(after_start))
# print(after_start)
closest_range = np.min(
np.abs(after_start)) # find closest point of approach (cpa)
cpa_index = after_start.index(closest_range)
# print(cpa_index)
pre_cpa = after_start[:cpa_index]
post_cpa = after_start[
cpa_index:] #find all distances after cpa time
# print(len(pre_cpa))
pre_times = converted_times[:cpa_index]
# print(len(pre_times))
post_times = converted_times[cpa_index:]
# print(post_times)
#print(pre_cpa)
#print(post_cpa)
approach_inter = interpolate.interp1d(pre_times,
pre_cpa,
axis=0,
fill_value="extrapolate")
depart_inter = interpolate.interp1d(post_times,
post_cpa,
axis=0,
fill_value="extrapolate")
sample_rate, samples = wavfile.read(
wavfilepath
) #get original wav file samples at the original sample rate
sound_length = len(samples) // sample_rate
#print(sound_length)
approach_times = np.arange(0, cpa_time)
depart_times = np.arange(cpa_time, sound_length)
frequencies, times, spectrogram = signal.spectrogram(
samples,
sample_rate,
window=np.hanning(10e3),
noverlap=0,
nfft=10e3,
mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,
frequencies) #get the transfer function results
spectrogram = 10 * np.log10(
spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(
uppc, np.size(spectrogram, 1), 1
) #copy tf results several times to make it same size as spect results
spectrogram = spectrogram + np.transpose(
uppc) #add tf results to spect results
range_step = .01 # step size of 1m
range_approach = (
(np.arange(pre_cpa[0], closest_range, -range_step))
) # make a vector of distances between first range and cpa
range_depart = (
np.arange(closest_range, post_cpa[len(post_cpa) - 1],
range_step)
) # make a vector of distances between cpa and last range
range_desired = np.append(range_approach,
range_depart) # stick them together
number_range_samples = len(
range_desired
) # total length is the number of samples we expect.
#print(spectrogram.shape)
spect_dis_approach = approach_inter(approach_times)
spect_dis_depart = depart_inter(depart_times)
approach_bins = np.digitize(spect_dis_approach, range_approach)
depart_bins = np.digitize(spect_dis_depart, range_depart)
approach_spect = range_spect(approach_bins, spectrogram)
depart_spect = range_spect(depart_bins, spectrogram)
#print(approach_spect.shape)
#print(depart_spect.shape)
#print(spectrogram)
#print(times)
#print(times.shape)
range_spectrogram = np.concatenate((approach_spect, depart_spect),
axis=1)
ship.spect = range_spectrogram
normal = True
#ranges = get_ranges(approach_bins,depart_bins,range_approach,range_depart)
#print(range_spectrogram)
#print(ranges)
#plt.yscale('log') #make y scale log to match the new decibel units
#axes = plt.gca() #get axes object
#axes.set_ylim([10,1000]) #set upper limit of data on axes to be 1000
# plt.pcolormesh(ranges,frequencies,range_spectrogram,vmin=60,vmax=110 ) #plot the data and add color
# plt.set_cmap('jet')
# plt.ylabel('Frequency [Hz]')
# plt.xlabel('Distance [km]')
# locs, ticks = plt.xticks() #get current time ticks
# new_ticks = get_ticks(ranges,locs)
# plt.xticks(locs,new_ticks)
# plt.colorbar()
#plt.xticks(locs, new_ticks) # Set locations and labels to the distance
#plt.savefig(destination) #save spectrogram at destination
#plt.imshow(spectrogram)
#plt.show() #show plot
#plt.close()
except:
print("bad ship")
up.one_jar(folder, ship, True)
normal = False
pass
# print("bad ships:")
# print(bad_apples)
# print(len(bad_apples))
if normal:
up.one_jar(folder, ship, False)
def generate(folder):
ships = up.unpickle_ships(folder)
bad_apples = []
i = 0
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
txtfilepath = ship.filepath + ship.id + '.txt' #the original txt file
destination = destination_folder + ship.year_month +'\\' + ship.id + '.png' #the destination for the spectrogram
print(txtfilepath)
converted_times,start,cpa_index,cpa_time,normal = convert_time(ship) #convert all times and find the file start time and cpa time
print(normal)
after_start = ship.distance[start:] #find all distances after file_time
closest_range = np.min(np.abs(after_start)) # find closest point of approach (cpa)
cpa_index = after_start.index(closest_range)
if len(after_start) < 20:
normal = False
raise NameError('Too Few Distances')
if normal: #for now only do the ones that are normal
print('here 1')
pre_cpa = after_start[:cpa_index]
pre_times = converted_times[:cpa_index]
approach_inter = interpolate.interp1d(pre_times,pre_cpa, axis=0, fill_value="extrapolate")
post_cpa = after_start[cpa_index:] #find all distances after cpa time
post_times = converted_times[cpa_index:]
depart_inter = interpolate.interp1d(post_times,post_cpa, axis=0, fill_value="extrapolate")
print('here 2')
sample_rate, samples = wavfile.read(wavfilepath) #get original wav file samples at the original sample rate
sound_length = len(samples)//sample_rate
frequencies, times, spectrogram = signal.spectrogram(samples,sample_rate, window = np.hanning(10e3), noverlap = 0, nfft = 10e3, mode='psd') #generate spectrogram
uppc = tf.get_tf(ship.harp,frequencies) #get the transfer function results
spectrogram = 10*np.log10(spectrogram) #convert to/from decibels ?
uppc = npmb.repmat(uppc,np.size(spectrogram,1),1) #copy tf results several times to make it same size as spect results
spectrogram = spectrogram + np.transpose(uppc) #add tf results to spect results
range_step = .01 # step size of 1m
if normal:
approach_times = np.arange(0,cpa_time)
range_approach = ((np.arange(pre_cpa[0], closest_range, -range_step))) # make a vector of distances between first range and cpa
depart_times = np.arange(cpa_time,sound_length)
else:
depart_times = np.arange(,sound_length)
range_depart = (np.arange(closest_range, post_cpa[len(post_cpa)-1], range_step)) # make a vector of distances between cpa and last range
#range_desired = np.append(range_approach,range_depart)# stick them together
#number_range_samples = len(range_desired)# total length is the number of samples we expect.
if normal:
print('here 3')
spect_dis_approach = approach_inter(approach_times)
approach_bins = np.digitize(spect_dis_approach,range_approach)
approach_spect = range_spect(approach_bins,spectrogram)
print('here 4')
spect_dis_depart = depart_inter(depart_times)
depart_bins = np.digitize(spect_dis_depart,range_depart)
#depart_bins = depart_bins + last_bin
depart_spect = range_spect(depart_bins,spectrogram)
if normal:
print('here 5')
range_spectrogram = np.concatenate((approach_spect,depart_spect),axis=1)
else:
print('here 6')
range_spectrogram = depart_spect
ship.spect = range_spectrogram
normal = True
#plt.savefig(destination) #save spectrogram at destination
#plt.close()
except:
i+=1
print("bad ship" + str(i))
up.one_jar(folder,ship,True)
normal = False
pass
if normal:
up.one_jar(folder,ship,False)
post_cpa = og_wav[cpa_time:(cpa_time + thirty_sec)]
new_wav = (pre_cpa + post_cpa)
return new_wav, bad
else:
bad = True
return None, bad
ships = up.unpickle_ships(rootdir)
ships = ships[1500:]
i = 0
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
print(wavfilepath)
destination = destination_folder + ship.year_month + '\\' + ship.id + '.wav'
result_array, start_index, cpa_index, cpa_time, normal = convert_time(
ship)
cpa_wav, bad = extract_cpa(cpa_time, wavfilepath)
if bad:
i += 1
print('bad' + str(i))
else:
cpa_wav.export(destination, format="wav")
except:
up.one_jar(rootdir, ship, True)
pass