def normalize(folder):
x = 0
y = 0
for ships in up.unpickle_batch(folder, 50, 4200, 9000):
#print(np.size(ships))
#print(ships[0].spect.shape)
sizes = get_sizes(ships)
#avg_size = int(np.mean(sizes))
square_size = np.size(ships[0].spect, 0)
for i in tqdm(range(len(ships)),
"This Batch: ",
bar_format="{l_bar}%s{bar}%s{r_bar}" %
(Fore.YELLOW, Fore.RESET)):
try:
pad_length = (square_size - sizes[i])
beg_pad, end_pad = calc_pad(square_size, sizes[i])
if (pad_length >= 0):
ships[i].spect = np.pad(ships[i].spect,
[(0, 0), (beg_pad, end_pad)],
mode='constant')
else:
ships[i].spect = ships[i].spect[:,
abs(beg_pad):sizes[i] +
end_pad]
# print(ships[i].spect.shape)
# print(x)
x += 1
up.one_jar(folder, ships[i], False)
except:
y += 1
up.one_jar(folder, ships[i], True)
tqdm.write(str(y))
pass
def find_done(folder):
i = 0
for ships in up.unpickle_batch(folder, 100, 6500, 9000):
for ship in ships:
if ship.spect is None:
i += 1
up.one_jar(folder, ship, True)
tqdm.write(str(i))
pass
def match_all(rootdir):
i = 0
mmsis = extract_mmsi('D:\VZDATAALL.csv')
for ships in up.unpickle_batch(rootdir, 8700, 0, 8700):
for ship in ships:
get_info(ship, mmsis)
types = get_all_types(ships)
for ship in ships:
match_type(ship, types)
def data_generator(folder, mode):
if mode == 'train':
while True:
for ships in up.unpickle_batch(
folder, 50, 0,
6000): #the first 6000 ships in batches of 50
random.shuffle(ships)
train_spect = []
train_months = []
for ship in ships:
train_spect.append(ship.spect)
x_train = np.asarray(train_spect)
x_train = x_train.astype('float32')
x_train /= np.amax(x_train) #- 0.5
X_train = np.reshape(x_train, (-1, 501, 501, 1))
train_months.append(ship.month)
Y_train = np_utils.to_categorical(train_months, 12)
yield X_train, Y_train
if mode == 'test':
while True:
for ships in up.unpickle_batch(
folder, 50, 6000,
7200): #The last 1200 ships in batches of 50
random.shuffle(ships)
test_spect = []
test_months = []
for ship in ships:
test_spect.append(ship.spect)
x_test = np.asarray(test_spect)
x_test = x_test.astype('float32')
x_test /= np.amax(x_test) #- 0.5
X_test = np.reshape(x_test, (-1, 501, 501, 1))
test_months.append(ship.month)
Y_test = np_utils.to_categorical(test_months, 12)
yield X_test, Y_test
def match_ships(rootdir,csv):
i=0
data_frame = get_data(csv)
for ships in up.unpickle_batch(rootdir,100,9000,9100):
for ship in ships:
# if isinstance(ship.sal, int):
print(ship.id + ': ' + str(i))
found,sample = find_best_sample(data_frame,ship.cpa_datetime)
sample = fix_sample(sample)
if found:
ship.depth = sample.depths
ship.temp = sample.temps
ship.sal = sample.sals
ship.IMO = sample.date
up.one_jar(rootdir,ship,False)
print('pickled: ' + str(i))
i+=1
def all_speeds(folder):
highest = 0
lowest = 99999 #temps to check validity of data
speed_list = []
for ships in up.unpickle_batch(folder, 100, 0, 9000):
for ship in ships:
ship.depth = np.array([float(i) for i in ship.depth])
ii = np.where(ship.depth == 1)[0]
if len(ii) > 0:
index = ii[0]
depth_400 = float(ship.depth[index])
temp_400 = float(ship.temp[index])
sal_400 = float(ship.sal[index])
# print(depth_400)
# print(temp_400)
# print("sal" + str(sal_400))
ship.sound_speed = calc_speed(
depth_400, sal_400,
temp_400) #for this run only cpa refers to sound speed
s = ship.sound_speed
if s > highest:
highest = s
if s < lowest:
lowest = s
speed_list.append(s)
else:
print(ship.id)
up.one_jar(folder, ship, False)
print(highest)
print(lowest)
mean, dev = std_dev(speed_list)
print(mean)
print(dev)
u_speeds = np.unique(np.array(speed_list))
print(u_speeds)
print(len(u_speeds))
def main(rootdir,destination):
i = 0
mmsis = extract_mmsi('D:\VZDATAALL.csv')
for ships in up.unpickle_batch(rootdir, 100, 400, 500):
for ship in ships:
try:
wavfilepath = ship.filepath + ship.id + '.wav' #the original wav file
destination = destination_folder + ship.year_month +'\\' + ship.id + '.wav'
skip = get_length(ship,mmsis)
converted_times, start_index, cpa_index,cpa_time,normal = convert_time(ship)
cpa_sog = ship.SOG[cpa_index]
if (not normal):
i+=1
print('bad ' + str(i))
elif skip:
i+=1
print('mmsi not included ' + str(i))
else:
distances, new_times = new_distances(ship, start_index, converted_times)
pre, post = find_ship_passage(ship,distances,new_times,cpa_time,cpa_sog)
pass_wav = cut_wav(pre,post,wavfilepath)
pass_wav.export(destination,format="wav")
print(wavfilepath)
pass_wav.export(destination,format="wav")
except:
up.one_jar(rootdir,ship,True)
print('something went wrong')
pass
def match_all(rootdir):
i = 0
for ships in up.unpickle_batch(rootdir, 8700, 0, 8700):
types = get_all_types(ships)
for ship in ships:
match_type(ship, types)