label=cl)
plt.legend()
plt.title(str(z) + '_' + data_type + '_T-sne')
plt.show()
plt.savefig(path + "/" + str(z) + '_' + data_type + '_T-sne.png')
# GPU
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
unet_info.begin()
# data load
subseq = 2048
dbName = "SepsisDataset"
print("Load data")
X_train, X_test, Y_train, Y_test, N_FEATURES, classes = Data_Load.load_data(
dbName, 2048)
#get dense prediction results with overlap(transformed win data label's ground truth)
# y_test_resh = y_test.reshape(y_test.shape[0], y_test.shape[2], -1)
# y_test_resh_argmax = np.argmax(y_test_resh, axis=2)
# labels_test_unary = y_test_resh_argmax.reshape(y_test_resh_argmax.size)
# file_labels_test_unary = 'labels_gd_'+args.dataset+'_'+str(subseq)+'0317.npy'
# np.save(file_labels_test_unary,labels_test_unary)
# Setting
## 10-flod cross validation###
epochs = 1
batch_size = 64
optim_type = 'adam'
learning_rate = 0.02
sum_time = 0
net = 'unet'
def main_aircraft_processing(opts):
sat_dir = opts[0]
flt_fil = opts[1]
sensor = opts[2]
flt_typ = opts[3]
out_dir = opts[4]
beg_t = opts[5]
end_t = opts[6]
mode = opts[7]
comp = opts[8]
lat_bnd = opts[9]
lon_bnd = opts[10]
sat_cmap = opts[11]
bg_col = opts[12]
ac_se_col = opts[13]
ac_cmap = opts[14]
ac_mina = opts[15]
ac_maxa = opts[16]
ac_pos_col = opts[17]
txt_col = opts[18]
txt_size = opts[19]
txt_pos = opts[20]
cache_dir = opts[21]
res = opts[22]
linewid = opts[23]
dotsiz = opts[24]
singlep = opts[25]
if (singlep):
print("Beginning processing for single point.")
else:
print("Beginning processing for trajectory.")
if (flt_typ == 'CSV'):
ac_traj = indata.read_aircraft_csv(flt_fil, beg_t, end_t)
ac_traj2 = utils.interp_ac(ac_traj, '30S')
print("\t-\tLoaded aircraft trajectory.")
if (singlep):
plot_bounds = utils.calc_bounds_sp(ac_traj, lat_bnd, lon_bnd)
else:
plot_bounds = utils.calc_bounds_traj(ac_traj, lat_bnd, lon_bnd)
n_traj_pts2 = len(ac_traj2)
area = utils.create_area_def(plot_bounds, res)
start_t, end_t, tot_time = utils.get_startend(ac_traj, sensor, mode)
prev_time = datetime(1850, 1, 1, 0, 0, 0)
old_scn = None
sat_img = None
for i in range(2, n_traj_pts2):
cur_time = ac_traj2.index[i]
print('\t-\tNow processing', cur_time)
sat_time = utils.get_cur_sat_time(cur_time, sensor, mode)
if (sat_time != prev_time):
print('\t-\tLoading satellite data for', sat_time)
sat_img = indata.load_sat(sat_dir, sat_time, comp, sensor, area,
cache_dir, mode)
if (sat_img is None and old_scn is not None):
sat_img = old_scn
elif (sat_img is None):
print("ERROR: No satellite data for", sat_time)
old_scn = sat_img
prev_time = sat_time
else:
print('\t-\tSatellite data already loaded for', sat_time)
print('\t-\tPlotting and saving results')
fig = acplot.setup_plot(plot_bounds, bg_col, linewid)
if (sat_img is not None):
fig = acplot.overlay_sat(fig, sat_img, comp, sat_cmap)
fig = acplot.overlay_startend(fig, ac_traj2, ac_se_col, dotsiz)
if (not singlep):
fig = acplot.overlay_ac(fig, ac_traj2, i, ac_cmap, ac_mina,
ac_maxa, linewid)
fig = acplot.add_acpos(fig, ac_traj2, i, ac_pos_col, dotsiz)
fig = acplot.overlay_time(fig, cur_time, txt_col, txt_size, txt_pos)
acplot.save_output_plot(
out_dir + str(i - 1).zfill(4) + '_' + comp + '.png', fig, 600)
fig.clf()
fig.close()
print("Completed processing")
args.train_save_path = "./Dataset/" + "window_" + str(
args.n_window) + "/train/"
args.test_save_path = "./Dataset/" + "window_" + str(
args.n_window) + "/test/"
args.vocab_save_path = "./Dataset/" + "vocab/"
Check_Dir(args.train_save_path)
Check_Dir(args.test_save_path)
Check_Dir(args.vocab_save_path)
Print_Args(args)
# Train Files
Paired_Files = Data_Load.Pos_Neg_Pairing(args.train_pos,
args.train_neg,
window=args.n_window,
text=True)
pos_file_names = Paired_Files.Load_Pos_Names()
Pos_Generator = Data_Load.Doc_Generator(args.train_pos, pos_file_names)
word_count = Word_Counter(args.train_pos)
Vocab = Build_Vocab(word_count, ratio=1.0)
word2idx, idx2word = Word_Dictionary(Vocab)
savepath = Data_Load.Create_Path(args.vocab_save_path, 'Vocab')
Data_Load.Save_File(savepath, Vocab, types='json')
savepath = Data_Load.Create_Path(args.vocab_save_path, 'word2idx')
Data_Load.Save_File(savepath, word2idx, types='json')
savepath = Data_Load.Create_Path(args.vocab_save_path, 'idx2word')
def _load_data(self):
Data_Load._load_network(self)
Data_Load._AreaLinks(self, 0.0)
Data_Load._load_generator_data(self)
Data_Load._load_wind_data(self)
Data_Load._load_intial_data(self)
def main_aircraft_processing(opts):
"""Control routine for processing."""
sat_dir = opts[0]
flt_fil = opts[1]
sensor = opts[2]
flt_typ = opts[3]
out_dir = opts[4]
beg_t = opts[5]
end_t = opts[6]
mode = opts[7]
comp = opts[8]
lat_bnd = opts[9]
lon_bnd = opts[10]
sat_cmap = opts[11]
bg_col = opts[12]
ac_se_col = opts[13]
ac_cmap = opts[14]
ac_mina = opts[15]
ac_maxa = opts[16]
ac_pos_col = opts[17]
txt_col = opts[18]
txt_size = opts[19]
txt_pos = opts[20]
cache_dir = opts[21]
res = opts[22]
tag = opts[23]
linewid = opts[24]
dotsiz = opts[25]
singlep = opts[26]
print("Beginning processing")
verbose = True
if (flt_typ == 'CSV'):
ac_traj = indata.read_aircraft_csv(flt_fil, beg_t, end_t)
elif (flt_typ == 'FR24'):
ac_traj = indata.read_aircraft_fr24(flt_fil, beg_t, end_t)
else:
print("ERROR: Unsupported flight data type:", flt_typ)
quit()
ac_traj2 = utils.interp_ac(ac_traj, '30S')
if verbose:
print("\t-\tLoaded aircraft trajectory.")
if (singlep):
plot_bounds = utils.calc_bounds_sp(ac_traj, lat_bnd, lon_bnd)
else:
plot_bounds = utils.calc_bounds_traj(ac_traj, lat_bnd, lon_bnd)
n_traj_pts2 = len(ac_traj2)
area = utils.create_area_def(plot_bounds, res)
start_t, end_t, tot_time = utils.get_startend(ac_traj, sensor, mode)
prev_time = datetime(1850, 1, 1, 0, 0, 0)
old_scn = None
sat_img = None
for i in range(2, n_traj_pts2):
outf = out_dir + str(i - 1).zfill(4) + '_' + comp + '_' + tag + '.png'
if os.path.exists(outf):
continue
cur_time = ac_traj2.index[i]
if verbose:
print('\t-\tNow processing', cur_time)
sat_time = utils.get_cur_sat_time(cur_time, sensor, mode)
if sat_time != prev_time:
if verbose:
print('\t-\tLoading satellite data for', sat_time)
sat_img = indata.load_sat(sat_dir, sat_time, comp, sensor,
plot_bounds, cache_dir, mode)
if sat_img is None and old_scn is not None:
sat_img = old_scn
elif sat_img is None:
print("ERROR: No satellite data for", sat_time)
old_scn = sat_img
prev_time = sat_time
else:
if verbose:
print('\t-\tSatellite data already loaded for', sat_time)
if verbose:
print('\t-\tPlotting and saving results')
fig = acplot.setup_plot(plot_bounds, bg_col, linewid,
sat_img[comp].attrs['area'].to_cartopy_crs())
if sat_img is not None:
fig = acplot.overlay_sat(fig, sat_img, comp, sat_cmap)
# fig = acplot.overlay_startend(fig, ac_traj2, ac_se_col, dotsiz)
if not singlep:
fig = acplot.overlay_ac(fig, ac_traj2, i, ac_cmap, ac_mina,
ac_maxa, linewid)
fig = acplot.add_acpos(fig, ac_traj2, i, ac_pos_col, dotsiz)
fig = acplot.overlay_time(fig, cur_time, txt_col, txt_size, txt_pos)
acplot.save_output_plot(outf, fig, 90)
fig.clf()
fig.close()
print("Completed processing")
args.train_save_path = "./Dataset_GCDC/" + "window_" + str(args.n_window) + "/train/"
args.test_save_path = "./Dataset_GCDC/" + "window_" + str(args.n_window) + "/test/"
args.vocab_save_path = "./Dataset_GCDC/" + "vocab/"
Check_Dir(args.train_save_path)
Check_Dir(args.test_save_path)
Check_Dir(args.vocab_save_path)
Print_Args(args)
# Train Files
# Paired_Files = Data_Load.Pos_Neg_Pairing(args.train_pos, args.train_neg, window=args.n_window, text=True)
# pos_file_names = Paired_Files.Load_Pos_Names()
pos_file_names = os.listdir(args.train_pos)
Pos_Generator = Data_Load.Doc_Generator(args.train_pos, pos_file_names)
# TODO: NEED Changes 1. remove <para_break> 2. add space in <s> and word.
word_count = Word_Counter(args.train_pos)
# TODO: Add <bos> and <eos> <s> and </s>
Vocab = Build_Vocab(word_count, ratio=1.0)
word2idx, idx2word = Word_Dictionary(Vocab)
savepath = Data_Load.Create_Path(args.vocab_save_path, 'Vocab')
Data_Load.Save_File(savepath, Vocab, types='json')
savepath = Data_Load.Create_Path(args.vocab_save_path, 'word2idx')
Data_Load.Save_File(savepath, word2idx, types='json')
savepath = Data_Load.Create_Path(args.vocab_save_path, 'idx2word')
Data_Load.Save_File(savepath, idx2word, types='json')
def _load_data(self):
Data_Load._load_network(self)
Data_Load._load_generator_data(self)
Data_Load._load_wind_data(self)
Data_Load._load_intial_data(self)