Exemplo n.º 1
0
                    label=cl)
    plt.legend()
    plt.title(str(z) + '_' + data_type + '_T-sne')
    plt.show()
    plt.savefig(path + "/" + str(z) + '_' + data_type + '_T-sne.png')


# GPU
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
unet_info.begin()

# data load
subseq = 2048
dbName = "SepsisDataset"
print("Load data")
X_train, X_test, Y_train, Y_test, N_FEATURES, classes = Data_Load.load_data(
    dbName, 2048)
#get dense prediction results with overlap(transformed win data label's ground truth)
# y_test_resh = y_test.reshape(y_test.shape[0], y_test.shape[2], -1)
# y_test_resh_argmax = np.argmax(y_test_resh, axis=2)
# labels_test_unary = y_test_resh_argmax.reshape(y_test_resh_argmax.size)
# file_labels_test_unary = 'labels_gd_'+args.dataset+'_'+str(subseq)+'0317.npy'
# np.save(file_labels_test_unary,labels_test_unary)

# Setting
## 10-flod cross validation###
epochs = 1
batch_size = 64
optim_type = 'adam'
learning_rate = 0.02
sum_time = 0
net = 'unet'
Exemplo n.º 2
0
def main_aircraft_processing(opts):

    sat_dir = opts[0]
    flt_fil = opts[1]
    sensor = opts[2]
    flt_typ = opts[3]
    out_dir = opts[4]
    beg_t = opts[5]
    end_t = opts[6]
    mode = opts[7]
    comp = opts[8]
    lat_bnd = opts[9]
    lon_bnd = opts[10]
    sat_cmap = opts[11]
    bg_col = opts[12]
    ac_se_col = opts[13]
    ac_cmap = opts[14]
    ac_mina = opts[15]
    ac_maxa = opts[16]
    ac_pos_col = opts[17]
    txt_col = opts[18]
    txt_size = opts[19]
    txt_pos = opts[20]
    cache_dir = opts[21]
    res = opts[22]
    linewid = opts[23]
    dotsiz = opts[24]
    singlep = opts[25]

    if (singlep):
        print("Beginning processing for single point.")
    else:
        print("Beginning processing for trajectory.")

    if (flt_typ == 'CSV'):
        ac_traj = indata.read_aircraft_csv(flt_fil, beg_t, end_t)

    ac_traj2 = utils.interp_ac(ac_traj, '30S')

    print("\t-\tLoaded aircraft trajectory.")
    if (singlep):
        plot_bounds = utils.calc_bounds_sp(ac_traj, lat_bnd, lon_bnd)
    else:
        plot_bounds = utils.calc_bounds_traj(ac_traj, lat_bnd, lon_bnd)

    n_traj_pts2 = len(ac_traj2)

    area = utils.create_area_def(plot_bounds, res)

    start_t, end_t, tot_time = utils.get_startend(ac_traj, sensor, mode)

    prev_time = datetime(1850, 1, 1, 0, 0, 0)
    old_scn = None
    sat_img = None

    for i in range(2, n_traj_pts2):
        cur_time = ac_traj2.index[i]
        print('\t-\tNow processing', cur_time)

        sat_time = utils.get_cur_sat_time(cur_time, sensor, mode)
        if (sat_time != prev_time):
            print('\t-\tLoading satellite data for', sat_time)
            sat_img = indata.load_sat(sat_dir, sat_time, comp, sensor, area,
                                      cache_dir, mode)
            if (sat_img is None and old_scn is not None):
                sat_img = old_scn
            elif (sat_img is None):
                print("ERROR: No satellite data for", sat_time)
            old_scn = sat_img
            prev_time = sat_time
        else:
            print('\t-\tSatellite data already loaded for', sat_time)

        print('\t-\tPlotting and saving results')
        fig = acplot.setup_plot(plot_bounds, bg_col, linewid)

        if (sat_img is not None):
            fig = acplot.overlay_sat(fig, sat_img, comp, sat_cmap)

        fig = acplot.overlay_startend(fig, ac_traj2, ac_se_col, dotsiz)
        if (not singlep):
            fig = acplot.overlay_ac(fig, ac_traj2, i, ac_cmap, ac_mina,
                                    ac_maxa, linewid)
        fig = acplot.add_acpos(fig, ac_traj2, i, ac_pos_col, dotsiz)

        fig = acplot.overlay_time(fig, cur_time, txt_col, txt_size, txt_pos)
        acplot.save_output_plot(
            out_dir + str(i - 1).zfill(4) + '_' + comp + '.png', fig, 600)
        fig.clf()
        fig.close()

    print("Completed processing")
        args.train_save_path = "./Dataset/" + "window_" + str(
            args.n_window) + "/train/"
        args.test_save_path = "./Dataset/" + "window_" + str(
            args.n_window) + "/test/"
        args.vocab_save_path = "./Dataset/" + "vocab/"

        Check_Dir(args.train_save_path)
        Check_Dir(args.test_save_path)
        Check_Dir(args.vocab_save_path)

        Print_Args(args)

        # Train Files
        Paired_Files = Data_Load.Pos_Neg_Pairing(args.train_pos,
                                                 args.train_neg,
                                                 window=args.n_window,
                                                 text=True)
        pos_file_names = Paired_Files.Load_Pos_Names()
        Pos_Generator = Data_Load.Doc_Generator(args.train_pos, pos_file_names)

        word_count = Word_Counter(args.train_pos)
        Vocab = Build_Vocab(word_count, ratio=1.0)
        word2idx, idx2word = Word_Dictionary(Vocab)

        savepath = Data_Load.Create_Path(args.vocab_save_path, 'Vocab')
        Data_Load.Save_File(savepath, Vocab, types='json')

        savepath = Data_Load.Create_Path(args.vocab_save_path, 'word2idx')
        Data_Load.Save_File(savepath, word2idx, types='json')

        savepath = Data_Load.Create_Path(args.vocab_save_path, 'idx2word')
Exemplo n.º 4
0
 def _load_data(self):
     Data_Load._load_network(self)
     Data_Load._AreaLinks(self, 0.0)
     Data_Load._load_generator_data(self)
     Data_Load._load_wind_data(self)
     Data_Load._load_intial_data(self)
Exemplo n.º 5
0
def main_aircraft_processing(opts):
    """Control routine for processing."""
    sat_dir = opts[0]
    flt_fil = opts[1]
    sensor = opts[2]
    flt_typ = opts[3]
    out_dir = opts[4]
    beg_t = opts[5]
    end_t = opts[6]
    mode = opts[7]
    comp = opts[8]
    lat_bnd = opts[9]
    lon_bnd = opts[10]
    sat_cmap = opts[11]
    bg_col = opts[12]
    ac_se_col = opts[13]
    ac_cmap = opts[14]
    ac_mina = opts[15]
    ac_maxa = opts[16]
    ac_pos_col = opts[17]
    txt_col = opts[18]
    txt_size = opts[19]
    txt_pos = opts[20]
    cache_dir = opts[21]
    res = opts[22]
    tag = opts[23]
    linewid = opts[24]
    dotsiz = opts[25]
    singlep = opts[26]

    print("Beginning processing")

    verbose = True

    if (flt_typ == 'CSV'):
        ac_traj = indata.read_aircraft_csv(flt_fil, beg_t, end_t)
    elif (flt_typ == 'FR24'):
        ac_traj = indata.read_aircraft_fr24(flt_fil, beg_t, end_t)
    else:
        print("ERROR: Unsupported flight data type:", flt_typ)
        quit()

    ac_traj2 = utils.interp_ac(ac_traj, '30S')

    if verbose:
        print("\t-\tLoaded aircraft trajectory.")
    if (singlep):
        plot_bounds = utils.calc_bounds_sp(ac_traj, lat_bnd, lon_bnd)
    else:
        plot_bounds = utils.calc_bounds_traj(ac_traj, lat_bnd, lon_bnd)

    n_traj_pts2 = len(ac_traj2)

    area = utils.create_area_def(plot_bounds, res)

    start_t, end_t, tot_time = utils.get_startend(ac_traj, sensor, mode)

    prev_time = datetime(1850, 1, 1, 0, 0, 0)
    old_scn = None
    sat_img = None

    for i in range(2, n_traj_pts2):
        outf = out_dir + str(i - 1).zfill(4) + '_' + comp + '_' + tag + '.png'
        if os.path.exists(outf):
            continue
        cur_time = ac_traj2.index[i]
        if verbose:
            print('\t-\tNow processing', cur_time)

        sat_time = utils.get_cur_sat_time(cur_time, sensor, mode)
        if sat_time != prev_time:
            if verbose:
                print('\t-\tLoading satellite data for', sat_time)
            sat_img = indata.load_sat(sat_dir, sat_time, comp, sensor,
                                      plot_bounds, cache_dir, mode)
            if sat_img is None and old_scn is not None:
                sat_img = old_scn
            elif sat_img is None:
                print("ERROR: No satellite data for", sat_time)
            old_scn = sat_img
            prev_time = sat_time
        else:
            if verbose:
                print('\t-\tSatellite data already loaded for', sat_time)

        if verbose:
            print('\t-\tPlotting and saving results')

        fig = acplot.setup_plot(plot_bounds, bg_col, linewid,
                                sat_img[comp].attrs['area'].to_cartopy_crs())

        if sat_img is not None:
            fig = acplot.overlay_sat(fig, sat_img, comp, sat_cmap)

    #  fig = acplot.overlay_startend(fig, ac_traj2, ac_se_col, dotsiz)
        if not singlep:
            fig = acplot.overlay_ac(fig, ac_traj2, i, ac_cmap, ac_mina,
                                    ac_maxa, linewid)
        fig = acplot.add_acpos(fig, ac_traj2, i, ac_pos_col, dotsiz)

        fig = acplot.overlay_time(fig, cur_time, txt_col, txt_size, txt_pos)
        acplot.save_output_plot(outf, fig, 90)
        fig.clf()
        fig.close()

    print("Completed processing")
Exemplo n.º 6
0
        args.train_save_path = "./Dataset_GCDC/" + "window_" + str(args.n_window) + "/train/"
        args.test_save_path = "./Dataset_GCDC/" + "window_" + str(args.n_window) + "/test/"
        args.vocab_save_path = "./Dataset_GCDC/" + "vocab/"

        Check_Dir(args.train_save_path)
        Check_Dir(args.test_save_path)
        Check_Dir(args.vocab_save_path)

        Print_Args(args)

        # Train Files
        # Paired_Files = Data_Load.Pos_Neg_Pairing(args.train_pos, args.train_neg, window=args.n_window, text=True)
        # pos_file_names = Paired_Files.Load_Pos_Names()

        pos_file_names = os.listdir(args.train_pos)
        Pos_Generator = Data_Load.Doc_Generator(args.train_pos, pos_file_names)
        # TODO: NEED Changes 1. remove <para_break> 2. add space in <s> and word.
        word_count = Word_Counter(args.train_pos)
        # TODO: Add <bos> and <eos> <s> and </s>
        Vocab = Build_Vocab(word_count, ratio=1.0)
        word2idx, idx2word = Word_Dictionary(Vocab)
        savepath = Data_Load.Create_Path(args.vocab_save_path, 'Vocab')
        Data_Load.Save_File(savepath, Vocab, types='json')

        savepath = Data_Load.Create_Path(args.vocab_save_path, 'word2idx')
        Data_Load.Save_File(savepath, word2idx, types='json')

        savepath = Data_Load.Create_Path(args.vocab_save_path, 'idx2word')
        Data_Load.Save_File(savepath, idx2word, types='json')

 def _load_data(self):
     Data_Load._load_network(self)
     Data_Load._load_generator_data(self)
     Data_Load._load_wind_data(self)
     Data_Load._load_intial_data(self)