def halo_annotations(code,redshift, multiple_widths='yes',all_axis='no'): data_pf = ld.load_data(code, redshift) catalog_dir='/home/nmiller/THE/halo_catalogs/catalog/' code_catalog=(catalog_dir+str(code)) redshift_dir=(code_catalog+'/redshift'+str(redshift)) file_dir=(redshift_dir+'/redshift'+str(redshift)+'.0.h5') hpf = load(str(file_dir)) hc = HaloCatalog(halos_pf=hpf, output_dir="halo_catalogs/catalog") hc.load EnzoC = ( 0.3861618 , 0.46086884, 0.49156952) if (multiple_widths=='yes'): ProjectionPlot(data_pf, 'z', 'density', center=EnzoC).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos.png') ProjectionPlot(data_pf, 'z', 'density', center=EnzoC, width=(10000,'kpc')).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_width10k.png') ProjectionPlot(data_pf, 'z', 'density', center=EnzoC, width=(3000,'kpc')).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_width3k.png') ProjectionPlot(data_pf, 'z', 'density', center=EnzoC, width=(600,'kpc')).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_width600.png') else: ProjectionPlot(data_pf, 'z', 'density', center=EnzoC).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos.png') if (all_axis=='yes'): ProjectionPlot(data_pf, 'z', 'density', center=EnzoC).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_z.png') ProjectionPlot(data_pf, 'x', 'density', center=EnzoC).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_x.png') ProjectionPlot(data_pf, 'y', 'density', center=EnzoC).annotate_halos(hc).annotate_title(str(code)+str(redshift)+'Halos').save(str(code)+str(redshift)+'Halos_y.png') else: return
def star_creation(code, redshift): ds=ld.load_data(code, redshift) dd = ds.all_data() pm = dd[('io', 'particle_mass')] pi = dd[('io', 'particle_index')] if (code=='ramses'): ct = dd[('all', 'particle_age')] stars=(ct!=0) print 'done' else: ct = dd[('io', 'creation_time')] stars=(ct > 0) pm = pm[stars] pi = pi[stars] ct = ct[stars] plt.plot(pi,ct, '.') plt.title('Particle Index vs. Creation Time for %s %s' %(code,redshift)) plt.savefig(str(code)+str(redshift)+"index_vs_creation.png") plt.plot(pm,ct, '.') plt.title('Particle Mass vs. Creation Time for %s %s' %(code,redshift)) plt.savefig(str(code)+str(redshift)+"particlemass_vs_creation.png")
def show(args): params = vars(args) train_data, train_label, test_data, test_label = load_data( params["datadir"], 1, 1) indices = np.arange(params["start"], train_data.shape[0] - 1) show_data(train_data[params["start"]:], train_label[params["start"]:], indices)
def property_file(code,redshift,property_attribute): # if this is run on its own the propery must be in the form of those listed in possible_properties.txt data_pf = ld.load_data(code,redshift) halo_list = HaloFinder(data_pf) # make the file halofile = open(property_attribute+'.txt', 'w') if (property_attribute=='center_of_mass'): for halo in halo_list: halofile.write(str(halo.center_of_mass()) + "\n") elif (property_attribute=='total_mass'): for halo in halo_list: halofile.write(str(halo.total_mass()) + "\n") elif (property_attribute=='maximum_density'): for halo in halo_list: halofile.write(str(halo.maximum_density()) + "\n") elif (property_attribute=='particle_size'): for halo in halo_list: halofile.write(str(halo.get_size()) + "\n") elif (property_attribute=='max_radius'): for halo in halo_list: halofile.write(str(halo.maximum_radius()) + "\n") elif (property_attribute =='virial_mass'): for halo in halo_list: halofile.write(str(halo.virial_mass()) + "\n") elif (property_attribute=='virial_radius'): for halo in halo_list: halofile.write(str(halo.halo.virial_radius()) + "\n") elif (property_attribute=='maximum_density_location'): for halo in halo_list: halofile.write(str(halo.maximum_density_location()) + "\n") halofile.close() return
def star_creation(code, redshift): ds=ld.load_data(code, redshift) dd = ds.all_data() pm,pi,ct = star_prop(dd) plt.plot(pi,ct, '.') plt.title('Particle Index vs. Creation Time for %s %s' %(code,redshift)) plt.savefig(str(code)+str(redshift)+"index_vs_creation.png") plt.plot(pm,ct, '.') plt.title('Particle Mass vs. Creation Time for %s %s' %(code,redshift)) plt.savefig(str(code)+str(redshift)+"particlemass_vs_creation.png")
def load(self, img_rows=IMAGE_SIZE, img_cols=IMAGE_SIZE, img_channels=3, nb_classes=3): # 加载数据集到内存 images, labels = load_data(self.path_name) train_images, valid_images, train_labels, valid_labels = train_test_split(images, labels, test_size=0.3, random_state=random.randint(0, 100)) _, test_images, _, test_labels = train_test_split(images, labels, test_size=0.5, random_state=random.randint(0, 100)) # 当前的维度顺序如果为'th',则输入图片数据时的顺序为:channels,rows,cols,否则:rows,cols,channels # 这部分代码就是根据keras库要求的维度顺序重组训练数据集 if K.image_dim_ordering() == 'th': train_images = train_images.reshape(train_images.shape[0], img_channels, img_rows, img_cols) valid_images = valid_images.reshape(valid_images.shape[0], img_channels, img_rows, img_cols) test_images = test_images.reshape(test_images.shape[0], img_channels, img_rows, img_cols) self.input_shape = (img_channels, img_rows, img_cols) else: train_images = train_images.reshape(train_images.shape[0], img_rows, img_cols, img_channels) valid_images = valid_images.reshape(valid_images.shape[0], img_rows, img_cols, img_channels) test_images = test_images.reshape(test_images.shape[0], img_rows, img_cols, img_channels) self.input_shape = (img_rows, img_cols, img_channels) # 输出训练集、验证集、测试集的数量 print(train_images.shape[0], 'train samples') print(valid_images.shape[0], 'valid samples') print(test_images.shape[0], 'test samples') # 我们的模型使用categorical_crossentropy作为损失函数,因此需要根据类别数量nb_classes将 # 类别标签进行one-hot编码使其向量化,在这里我们的类别只有两种,经过转化后标签数据变为二维 train_labels = np_utils.to_categorical(train_labels, nb_classes) valid_labels = np_utils.to_categorical(valid_labels, nb_classes) test_labels = np_utils.to_categorical(test_labels, nb_classes) # 像素数据浮点化以便归一化 train_images = train_images.astype('float32') valid_images = valid_images.astype('float32') test_images = test_images.astype('float32') # 将其归一化,图像的各像素值归一化到0~1区间 train_images /= 255 valid_images /= 255 test_images /= 255 self.train_images = train_images self.valid_images = valid_images self.test_images = test_images self.train_labels = train_labels self.valid_labels = valid_labels self.test_labels = test_labels
def generate_cluster(name=None): #Load Dataset with inputs: node, dataset, transpose_flag node = "ABBV_sym" transpose_flag = 0 dataset = 'dataset/pharma_pharma_dataset.csv' args = load_data(node, dataset, transpose_flag) #Extract Clusters cluster = extract_cluster(args) print(cluster) #visualize that cluster on index.html return render_template('index.html', name=name)
def mstar_mhalo(code,redshift,st=0,end=10): ds = ld.load_data(code, redshift) result=[] for halo in xrange(int(st),int(end)): print halo sphere_center=nh._center_of_mass(code,redshift,halo) radius=nh._radius(code,redshift,halo) sp = ds.sphere(sphere_center, (int(radius), "kpc")) stars=s.find_stars(sp) m_star=s.find_M_star(sp) star_mass = sum(int(x) for x in m_star) baryon_mass, particle_mass = sp.quantities.total_quantity(["cell_mass", "particle_mass"]) baryon_mass=baryon_mass.in_units('Msun') particle_mass=particle_mass.in_units('Msun') total_mass=(baryon_mass+particle_mass) result.append([halo, total_mass, baryon_mass, particle_mass, star_mass]) return result
def mstar_mhalo(code,redshift,halos=all): ds = ld.load_data(code, redshift) print 'loaded' result=[] if (halos=='all'): print 'in loop' file_name=cd.get_output_file_name(code,'com',redshift) file_com=open(file_name,'r') number_halos=[] for line in file_com: l=line.split() number_halos.append(l) print len(number_halos) for halo in xrange(0,len(number_halos)): print halo sphere_center=nh._center_of_mass(code,redshift,halo) radius=nh._radius(code,redshift,halo) sp = ds.sphere(sphere_center, (int(radius), "kpc")) m_star=s.star_prop(sp,'pm') m_star=m_star.in_units('Msun') star_mass = sum(int(x) for x in m_star) #star_mass=star_mass.in_units('Msun') baryon_mass, particle_mass = sp.quantities.total_quantity(["cell_mass", "particle_mass"]) baryon_mass=baryon_mass.in_units('Msun') particle_mass=particle_mass.in_units('Msun') total_mass=(baryon_mass+particle_mass) result.append([halo, total_mass, baryon_mass, particle_mass, star_mass]) return result else: for halo in xrange(int(halos[0]),int(halos[1])): print halo sphere_center=nh._center_of_mass(code,redshift,halo) radius=nh._radius(code,redshift,halo) sp = ds.sphere(sphere_center, (int(radius), "kpc")) m_star=s.star_prop(sp,'pm') m_star=m_star.in_units('Msun') star_mass = sum(int(x) for x in m_star) #star_mass=star_mass.in_units('Msun') baryon_mass, particle_mass = sp.quantities.total_quantity(["cell_mass", "particle_mass"]) baryon_mass=baryon_mass.in_units('Msun') particle_mass=particle_mass.in_units('Msun') total_mass=(baryon_mass+particle_mass) result.append([halo, total_mass, baryon_mass, particle_mass, star_mass]) return result
def total_mass_sphere(code, redshift, sphere_center, radius): # Load the dataset. ds = ld.load_data(code, redshift) # Create a 1 Mpc radius sphere, centered on the max density. sp = ds.sphere(sphere_center, (int(radius), "kpc")) # Use the total_quantity derived quantity to sum up the # values of the cell_mass and particle_mass fields # within the sphere. baryon_mass, particle_mass = sp.quantities.total_quantity(["cell_mass", "particle_mass"]) massfile=(str(code)+str(redshift)+'spheremass.txt') ld.check_file(massfile) massfile = open(massfile, 'w') massfile.write("Total mass in sphere is %0.3e Msun (gas = %0.3e Msun, particles = %0.3e Msun)" % \ ((baryon_mass + particle_mass).in_units('Msun'), \ baryon_mass.in_units('Msun'), particle_mass.in_units('Msun'))) massfile.close() return
def display_data(): flags, atmospheric_pressure = load_data() sst = load_data_custom('TS') at = load_data_custom('T') #print(at) pressure_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 10 ] sst_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 13 ] at_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 11 ] side_by_side = list(zip(at, atmospheric_pressure, at_flags, pressure_flags)) print(len(side_by_side)) """ df_flags = pd.DataFrame(flags, columns=['time', 'lat', 'lon', 'PL_HD', 'PL_CRS', 'DIR', 'PL_WDIR', 'PL_SPD', 'SPD', 'PL_WSPD', 'P', 'T', 'RH', 'TS', 'SSPS',]) """ df_flag_and_obs = pd.DataFrame(side_by_side, columns=[ 'air temp', 'pressure observation', 'at flag', 'pressure flag' ]) with pd.option_context('display.max_rows', None, 'display.max_columns', None): print(df_flag_and_obs) #print(df_flag_and_obs) return df_flag_and_obs
def evaluate_data(): # Load dataset dataset = load_dataset.load_data() # Split-out validation dataset array = dataset.values X = array[:, 0:4] Y = array[:, 4] validation_size = 0.20 seed = 7 X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split( X, Y, test_size=validation_size, random_state=seed) # Test options and evaluation metric scoring = 'accuracy' # Spot Check Algorithms models = [] models.append(('LR', LogisticRegression())) models.append(('LDA', LinearDiscriminantAnalysis())) models.append(('KNN', KNeighborsClassifier())) models.append(('CART', DecisionTreeClassifier())) models.append(('NB', GaussianNB())) models.append(('SVM', SVC())) # evaluate each model in turn for name, model in models: kfold = model_selection.KFold(n_splits=10, random_state=seed) cv_results = model_selection.cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring) results.append(cv_results) names.append(name) msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std()) print(msg) return X_train, X_validation, Y_train, Y_validation
def setup(): global X_train, X_test, Y_train, Y_test, input_shape X, y = load_data(loc_) X_train = X y_train = y # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) if K.image_dim_ordering() == 'th': X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) # X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) # X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) X_train = X_train.astype('float32') X_train = X_train / 255 # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes)
PATCH_WIDTH = 100 PATCH_HEIGHT = 100 PATCH_SIZE = PATCH_WIDTH * PATCH_HEIGHT * 3 # processing command arguments phone, batch_size, train_size, learning_rate, num_train_iters, \ w_content, w_color, w_texture, w_tv, \ dped_dir, vgg_dir, eval_step = utils.process_command_args(sys.argv) np.random.seed(0) # loading training and test data print("Loading test data...") test_data, test_answ, train_data, train_answ = load_data(dataset, 0.1) print("data was loaded\n") TEST_SIZE = test_data.shape[0] num_test_batches = int(test_data.shape[0] / batch_size) # defining system architecture with tf.Graph().as_default(), tf.Session() as sess: # placeholders for training data phone_ = tf.placeholder(tf.float32, [None, PATCH_SIZE]) phone_image = tf.reshape(phone_, [-1, PATCH_HEIGHT, PATCH_WIDTH, 3]) dslr_ = tf.placeholder(tf.float32, [None, PATCH_SIZE])
def training(): batch_size = 10 noise_type = 'gaussian' noise_proportion = 0.2 noise_mean = 0 noise_std = 1 noise_lam = 1 noise_std_range = [1,5] noise_lam_range = [1,5] loss_type = 'l2_loss' study_rate = 1e-5 current_dir = Path('.') train_true,__,test_true=load_data(dataset='mias', DIR=current_dir) train_true = train_true.repeat().batch(batch_size) train_true = train_true.prefetch(buffer_size = tf.data.experimental.AUTOTUNE) test_true = test_true.repeat().batch(batch_size) test_true = test_true.prefetch(buffer_size = tf.data.experimental.AUTOTUNE) iter_train_handle = train_true.make_one_shot_iterator().string_handle() iter_val_handle = test_true.make_one_shot_iterator().string_handle() handle = tf.placeholder(tf.string, shape=[]) iterator = Iterator.from_string_handle(handle, train_true.output_types, train_true.output_shapes) next_batch = iterator.get_next() noise_args = {'proportion':noise_proportion} if noise_type == 'random': noise_fn = random_noise noise_args['std_range'] = noise_std_range noise_args['lam_range'] = noise_lam_range elif noise_type == 'poisson': noise_fn = poisson_noise noise_args['lam'] = noise_lam else: noise_fn = gaussian_noise noise_args['mean'] = noise_mean noise_args['std'] = noise_std true_img = tf.placeholder(tf.uint8, shape=[batch_size, 64, 64, 1]) noised_img = noise_fn(**noise_args, image=true_img) model_input = tf.cast(noised_img, dtype=tf.float32) denoised_img = QAE.build_QAE(model_input) if loss_type == 'l2_loss': train_loss = l2_loss( tf.cast(true_img, dtype=tf.float32), denoised_img) # val_loss = l2_loss( # tf.cast(test_true_img, # dtype=tf.float32), # test_denoised_img) total_train_loss = train_loss optimizer = tf.train.AdamOptimizer(\ learning_rate=study_rate).minimize( total_train_loss) tf.summary.scalar('train_l2_loss',train_loss) # tf.summary.scalar('val_l2_loss',val_loss) tf.summary.scalar('total_train_loss', total_train_loss) merged_summary = tf.summary.merge_all() train_writer = tf.summary.FileWriter( current_dir/'train_data') init_vars = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() with tf.Session().as_default() as sess: global_step = tf.train.get_global_step() handle_train, handle_val = sess.run( [iter_train_handle, iter_val_handle]) sess.run(init_vars) for step in range(500): train_true_img = sess.run(next_batch, feed_dict={handle: handle_train}) test_true_img = sess.run(next_batch, feed_dict={handle: handle_val}) _ = sess.run(optimizer, feed_dict={true_img:train_true_img}) t_summ = sess.run(merged_summary, feed_dict={true_img:train_true_img}) t_loss = sess.run(total_train_loss, feed_dict={true_img:train_true_img}) train_writer.add_summary(t_summ,step) print('Iter:{}, Training Loss {}'.format( step, t_loss)) if step%20 == 0: fig,axes = print('done')
import numpy as np from stradimod.utils import sigmoid, relu, tanh from stradimod.netwok import Network from stradimod.layers import Dense from load_dataset import load_data np.random.seed(1) x_train, y_train, x_test, y_test, classes = load_data() # === TEST 1 ===== print("====== TEST 1 =====") model = Network() model.add(Dense(5, relu)) model.add(Dense(5, relu)) model.add(Dense(1, sigmoid)) model.build(12288) model.train(x_train, y_train, epochs=3000, learning_rate=0.03) accuracy = model.predict(x_test, y_test) print(f"Accuracy: {accuracy*100}%") print("") # === TEST 2 ==== print("====== TEST 2 =====") model = Network() model.add(Dense(5, tanh)) model.add(Dense(5, tanh)) model.add(Dense(1, sigmoid)) model.build(12288)
print("batch_time: %dms - batch_loss: %.4f" % ((batch_end - batch_start)*1000, batch_loss)) epoch_end = time.time() # 打印epoch的信息 print("batchs: %d - epoch_time: %ds %dms/batch - loss: %.4f" % (batchs, epoch_end - epoch_start, (epoch_end-epoch_start)*1000/batchs, total_loss/batchs)) # 按配置json文件里的save_interval的数值来保存检查点 if epoch % save_interval == 0: manager.save() if __name__ == "__main__": configs = get_config() epochs = configs["train"]["train_epochs"] data_path = configs["train"]["data_path"] num_examples = configs["train"]["num_examples"] dataset_name = configs["preprocess"]["dataset_name"] batch_size = configs["train"]["batch_size"] #加载数据,并构建数据生成器 train_data = load_data(dataset_name, data_path, "train", num_examples) batchs = ceil(len(train_data[0]) / batch_size) train_data_generator = data_generator(train_data, "train", batchs, batch_size) # 加载模型 model = get_ds2_model() optimizer = tf.keras.optimizers.Adam() #训练 train(model, optimizer, train_data_generator, batchs, epochs)
def simulation_plot(code,redshift): ds=ld.load_data(code, redshift) ProjectionPlot(ds, 2, 'density', width=(25.0, 'mpc'),weight_field=None).save(str(code)+str(redshift)+'complete-density.png') ProjectionPlot(ds, 2, 'temperature', width=(25.0, 'mpc'),weight_field=None).save(str(code)+str(redshift)+'complete-temperature.png')
AL, cache = forward_propagation(A, parameters['W2'], parameters['b2'], activation = "sigmoid") # convert probabilities to 0 or 1 for i in range(0, AL.shape[1]): if AL[0,i] > 0.5: p[0,i] = 1 else: p[0,i] = 0 print("Accuracy: " + str(np.sum((p == y)/m))) return AL # Load very popular dataset of catvsnon-cat for the testing purpose. from load_dataset import load_data train_x_orig, train_y, test_x_orig, test_y, classes = load_data() train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0], -1).T test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T # Standardize data to have feature values between 0 and 1. train_x = train_x_flatten / 255. test_x = test_x_flatten / 255. ### CONSTANTS DEFINING THE MODEL #### n_x = 12288 # length of input features # num_px * num_px * 3 n_h = 7 # Length of hidden layer n_y = 1 # Length of output layer layer_dimention = (n_x, n_h, n_y) #Build the model
#başarıyı hesaplamak için kullanılan fonksiyon def calc_accuracy(prediction, actual): counter = 0 index = 0 while (index < len(prediction)): if (prediction[index] == actual[index]): counter += 1 index += 1 return counter / len(prediction) #veriseti elde edilir x_train, x_test, train_label, test_labels = load_data() #sınıflar one-hot-encoding'e çevrilir train_label = to_categorical(train_label, 10) test_labels = to_categorical(test_labels, 10) print("Data is ready...") """ Modelin tanımlandığı yer. Bizim ilk girişi ve çıkışı vermemiz yeterli. Ara adımlar keras tarafından hesaplanır. """ model = models.Sequential() #64 filtre, relu aktivasyon fonksiyonu, 3x3'lük filtreler. Padding yok ve stride = 1 model.add( layers.Conv2D(64, (3, 3), activation='relu', input_shape=(200, 200, 3))) model.add(layers.MaxPooling2D((2, 2)))
model = get_ds2_model() checkpoint = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager( checkpoint, directory=configs["checkpoint"]['directory'], max_to_keep=configs["checkpoint"]['max_to_keep']) if manager.latest_checkpoint: checkpoint.restore(manager.latest_checkpoint) test_data_path = configs["test"]["data_path"] num_examples = configs["test"]["num_examples"] dataset_name = configs["preprocess"]["dataset_name"] batch_size = configs["test"]["batch_size"] # 加载测试集数据生成器 test_data = load_data(dataset_name, test_data_path, "test", num_examples) batchs = ceil(len(test_data[0]) / batch_size) test_data_generator = data_generator(test_data, "test", batchs, batch_size) aver_wers = 0 aver_lers = 0 aver_norm_lers = 0 # 构建字符集对象 index_word = get_index_word() for batch, (input_tensor, labels_list) in zip(range(1, batchs + 1), test_data_generator): originals = labels_list results = [] y_pred = model(input_tensor)
# -*- coding: utf-8 -*- from mlp import NeuralNetMLP from load_dataset import load_data import matplotlib.pyplot as plt import os import pickle import numpy as np #Carrega o dataset de treino e teste path = os.getcwd() #Carrega as 60000 instancias de treino X_train, y_train = load_data(path, tipo='train') print('Rows: %d, columns: %d' % (X_train.shape[0], X_train.shape[1])) #Carrega as 10000 instancias de teste X_test, y_test = load_data(path, tipo='t10k') print('Rows: %d, columns: %d' % (X_test.shape[0], X_test.shape[1])) #Cria uma instância de rede neural com os seguintes parâmetros neural_network = NeuralNetMLP(n_output=10, n_features=X_train.shape[1], n_hidden=50, l2=0.1, l1=0.0, epochs=2000, eta=0.001, alpha=0.001, decrease_const=0.00001, shuffle=True, minibatches=50, random_state=1)
fp.write('{}\t{}\t{}\t{}\t{}\n'.format( i, loss[i], acc[i], val_loss[i], val_acc[i])) if __name__ == '__main__': epochs = 50 nb_classes = 6 batch_size = 32 original_size_image_path = '/home/deep/datasets/kth/images/original/' crop_size_image_path = '/home/deep/datasets/kth/images/crop/' output_path = 'results/' print('training with original size image') model_ori,_ = model_original_size() x_train, y_train, x_val, y_val = load_data(original_size_image_path,'original') y_train = keras.utils.to_categorical(y_train,nb_classes) y_val = keras.utils.to_categorical(y_val) model_ori.summary() lr = 0.005 sgd = keras.optimizers.SGD(lr=lr, nesterov=True, momentum=0.9) model_ori.compile(loss=keras.losses.categorical_crossentropy, optimizer=sgd, metrics=['accuracy']) history = model_ori.fit(x_train,y_train, batch_size=batch_size, epochs=epochs, verbose=1, callbacks=[onetenth_20_30_40(lr=lr)], validation_data=(x_val,y_val),
from tensorflow.keras import models from tensorflow.keras.layers import Conv2D,MaxPooling2D,Flatten,Dense,Dropout,BatchNormalization from tensorflow.keras.utils import to_categorical, normalize from tensorflow.keras.optimizers import Adam from tensorflow.keras.regularizers import l1,l2 import numpy as np from load_dataset import load_data, preprocess_data from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from time import process_time import matplotlib.pyplot as plt data_dir = '/home/kangle/dataset/PedBicCarData' train_data, train_label, test_data, test_label = load_data(data_dir, 2, 2) train_data, train_label, test_data, test_label = preprocess_data(train_data, train_label, test_data, test_label, 'cnn') train_data, val_data, train_label, val_label = train_test_split(train_data, train_label, test_size=0.1, random_state=42) print("Split training data into training and validation data:\n") print("training data: %d" % train_data.shape[0]) print("validation data: %d" % val_data.shape[0]) model = models.Sequential() model.add(Conv2D(16, [5, 5], input_shape=train_data.shape[1:], activation='relu', kernel_initializer='he_uniform', kernel_regularizer='l2', name='conv_1')) model.add(MaxPooling2D()) model.add(Conv2D(32, [5, 5], activation='relu', kernel_initializer='he_uniform', kernel_regularizer='l2', name='conv_2')) model.add(MaxPooling2D()) model.add(Flatten()) model.add(Dense(120, activation='relu', kernel_initializer='he_uniform', name='dense_1')) #model.add(BatchNormalization()) model.add(Dense(84, activation='relu', kernel_initializer='he_uniform', name='dense_2'))
from yt import * import load_dataset as ld code='ramses' redshift=1.0 ds=ld.load_data(code, redshift) dd = ds.all_data() pa = dd[('all', 'particle_age')] print pa
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) # 加载数据 data_loaders, dataset_sizes, dataset = load_data(args.batch_size) # 损失函数 criterion = nn.CrossEntropyLoss() # criterion = torch.nn.DataParallel(criterion) criterion = criterion.cuda() # PC-DARTS模型初始换 model = Network(args.init_channels, CLASSES, args.layers, criterion) model = model.cuda().half() # Arcface margin = ArcMarginProduct(512, CLASSES) # margin = torch.nn.DataParallel(margin) margin = margin.cuda().half() if MULTI_GPU: model = torch.nn.DataParallel(model) margin = torch.nn.DataParallel(margin) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) # pc-darts和margin的优化器 if MULTI_GPU: optimizer = torch.optim.SGD( # [{'params': model.module.parameters(), 'weight_decay': args.weight_decay}, [{ 'params': margin.module.parameters(), 'weight_decay': args.weight_decay }], args.learning_rate, momentum=args.momentum) # 架构参数的优化器 optimizer_a = torch.optim.Adam(model.module.arch_parameters(), lr=args.arch_learning_rate, betas=(0.5, 0.999), weight_decay=args.arch_weight_decay) else: optimizer = torch.optim.SGD([{ 'params': model.parameters(), 'weight_decay': args.weight_decay }, { 'params': margin.parameters(), 'weight_decay': args.weight_decay }], args.learning_rate, momentum=args.momentum) # 架构参数的优化器 optimizer_a = torch.optim.Adam(model.arch_parameters(), lr=args.arch_learning_rate, betas=(0.5, 0.999), weight_decay=args.arch_weight_decay) if resume: checkpoint = torch.load( '/data/face_recognition/PC-DARTS-master/checkpoints_ms1m/search-try-20200423-094408/PC-DARTS_FACE.pth.tar' ) model.load_state_dict(checkpoint['model_state_dict']) # margin.load_state_dict(checkpoint['margin_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) # # optimizer_a.load_state_dict(checkpoint['optimizer_a']) # start_epoch = checkpoint['epoch'] # optimizer.param_groups[0]['lr'] = 0.5 lr = args.learning_rate else: lr = args.learning_rate start_epoch = 0 # sys.exit() if fineturn: checkpoint = torch.load( '/data/face_recognition/PC-DARTS-master/checkpoints_ms1m/search-try-20200423-094408/PC-DARTS_FACE.pth.tar' ) model.load_state_dict(checkpoint['model_state_dict']) margin.load_state_dict(checkpoint['margin_state_dict']) lr = args.learning_rate print('ccccccccccccccccccc') print(lr) start_epoch = 0 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) for epoch in range(args.epochs): scheduler.step() current_lr = scheduler.get_lr()[0] logging.info('Epoch: %d lr: %e', epoch, current_lr) if epoch < 5 and args.batch_size > 256 and not fineturn: for param_group in optimizer.param_groups: param_group['lr'] = lr * (epoch + 1) / 5.0 logging.info('Warming-up Epoch: %d, LR: %e', epoch, lr * (epoch + 1) / 5.0) # print(optimizer) if MULTI_GPU: genotype = model.module.genotype() logging.info('genotype = %s', genotype) arch_param = model.module.arch_parameters() else: genotype = model.genotype() logging.info('genotype = %s', genotype) arch_param = model.arch_parameters() # training train_acc, train_obj = train(data_loaders['train_subset'], data_loaders['valid_subset'], model, margin, optimizer, optimizer_a, criterion, lr, epoch) logging.info('Train_acc %f', train_acc) # validation if epoch >= args.begin: infer(data_loaders, dataset, model, criterion, epoch) # utils.save(model, os.path.join(args.save, 'weights.pt')) # utils.save(margin, os.path.join(args.save, 'margin_weights.pt')) state = { 'epoch': epoch + 1, 'args': args, 'margin_state_dict': margin.state_dict(), 'model_state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'optimizer_a': optimizer_a.state_dict() } # filename = 'checkpoints/' + args.output_filename + '.pth.tar' filename = os.path.join(args.save, 'PC-DARTS_FACE') + '.pth.tar' torch.save(state, filename)
# Summarize the Dataset import load_dataset # Main program if __name__ == '__main__': # Load dataset dataset = load_dataset.load_data() # shape # print(dataset.shape) # head # print(dataset.head(20)) # descriptions # print(dataset.describe()) # class distribution print(dataset.groupby('class').size())
def train(args): params = vars(args) with open(params["config"], mode='r') as f: paramset = json.load(f) data_dir = paramset["root_dir"] samp_rate_t = paramset["sample_rate"]["sample_rate_t"] samp_rate_f = paramset["sample_rate"]["sample_rate_f"] train_data_raw, train_label_raw, test_data_raw, test_label_raw = load_data( data_dir, samp_rate_t, samp_rate_f) if "dimension_reduction" in paramset: dim_reducer = paramset["dimension_reduction"]["method"] num_components = paramset["dimension_reduction"]["n_components"] algo_map[dim_reducer]["parameters"]["n_components"] = num_components print('\nbegin dimensionality reduction process.') module = importlib.import_module(algo_map[dim_reducer]["module"]) reducer = getattr(module, algo_map[dim_reducer]["function"])( **algo_map[dim_reducer]["parameters"]) train_data, train_label, test_data, test_label = preprocess_data( train_data_raw, train_label_raw, test_data_raw, test_label_raw, dim_reducer) reducer.fit(train_data) train_data = reducer.transform(train_data) test_data = reducer.transform(test_data) print('\nafter dimensionality reduction:') print(train_data.shape) print(test_data.shape) print('\nbegin training process.') classify_method = paramset["classifier"]["method"] classify_parameter = paramset["classifier"]["parameter"] if "dimension_reduction" not in paramset: train_data, train_label, test_data, test_label = preprocess_data( train_data_raw, train_label_raw, test_data_raw, test_label_raw, classify_method) module = importlib.import_module(algo_map[classify_method]["module"]) if algo_map[classify_method]["module"] == "nnet_lib": #classifier = getattr(module, algo_map[classify_method]["function"])(train_data, train_label) classifier, history = getattr(module, "nnet_training")(train_data, train_label, classify_method, **classify_parameter) plot_learncurve(classify_method, history=history) else: classifier = getattr( module, algo_map[classify_method]["function"])(**classify_parameter) classifier.fit(train_data, train_label) plot_learncurve(classify_method, estimator=classifier, data=train_data, label=train_label, train_sizes=np.linspace(0.05, 0.2, 5)) print('\npredict for test data.') test_pred = classifier.predict(test_data) train_pred = classifier.predict(train_data) if len(test_pred.shape) > 1: test_pred = np.argmax(test_pred, axis=1) train_pred = np.argmax(train_pred, axis=1) test_label = np.argmax(test_label, axis=1) train_label = np.argmax(train_label, axis=1) print('\nevaluate the prediction(train data).') train_conf = confusion_matrix(train_label, train_pred) train_precision = precision_score(train_label, train_pred, average=None) train_recall = recall_score(train_label, train_pred, average=None) print(train_conf) print(train_precision) print(train_recall) print('\nevaluate the prediction(test data).') test_conf = confusion_matrix(test_label, test_pred) test_precision = precision_score(test_label, test_pred, average=None) test_recall = recall_score(test_label, test_pred, average=None) print(test_conf) print(test_precision) print(test_recall) pred_result = { "train_conf": train_conf, "train_precision": train_precision, "train_recall": train_recall, "test_conf": test_conf, "test_precision": test_precision, "test_recall": test_recall } print('\ngenerate report file \t') #logFile = write_log(paramset, pred_result) if algo_map[classify_method]["module"] == "nnet_lib": logFile = write_log(paramset, pred_result, classifier, history) else: logFile = write_log(paramset, pred_result) print(logFile) if params["show_misclassified"]: indices = [ i for i in range(len(test_label)) if test_pred[i] != test_label[i] ] show_data(test_data_raw[indices], test_label[indices], indices, test_pred[indices])
def plot_halo_sphere(code, redshift, halo_number, width=400): ds=ld.load_data(code, redshift) radius=nh._radius(code,redshift,halo_number) center=nh._center_of_mass(code,z,halo_number) halo_sphere = ds.sphere(center, (float(radius), "kpc")) ProjectionPlot(ds, 'density', width=(int(width), "kpc"), center='max', weight_field=None, data_source=halo_sphere).save('Halo'+str(halo_number)+'.png')
batch_size = 128 nb_classes = 5 nb_epoch = 20 # Directory location of Train and Test Datasets loc_ = "/home/samiran/Desktop/suhitCS726/Dataset-CS726" img_rows, img_cols = 96, 128 # number of convolutional filters to use nb_filters = 64 # size of pooling area for max pooling pool_size = (2, 2) # convolution kernel size kernel_size = (3, 3) X, y = load_data(loc_) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) if K.image_dim_ordering() == 'th': X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) X_train = X_train.astype('float32')
import pandas as pd import matplotlib.pyplot as plt import numpy as np from load_dataset import load_data_custom, load_data flags, atmospheric_pressure = load_data() sst = load_data_custom('TS') at = load_data_custom('T') #print(at) pressure_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 10 ] sst_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 13 ] at_flags = [ value for sublist in flags for counter, value in enumerate(sublist) if counter == 11 ] side_by_side = list(zip(at, atmospheric_pressure, at_flags, pressure_flags)) #print(side_by_side) #Z cleared data cleared_p_data = [] cleared_at_data = []
def inference(data_dir, regu, training): with tf.variable_scope("input"): meta, train_data, test_data = load_data(data_dir) CAPATCHA_SIZE = meta["captcha_size"] IMAGE_WIDTH = meta["width"] IMAGE_HEIGHT = meta["height"] IMAGE_SIZE = IMAGE_WIDTH * IMAGE_HEIGHT x = tf.placeholder(dtype=tf.float32, shape=[None, IMAGE_HEIGHT, IMAGE_WIDTH]) y_ = tf.placeholder(dtype=tf.float32, shape=[None, CAPATCHA_SIZE]) input = tf.reshape(x, [-1, IMAGE_HEIGHT, IMAGE_WIDTH, 1]) tf.summary.image("input", input, max_outputs=10) with tf.variable_scope("layer1_conv"): weight = tf.get_variable("weight", shape=[3, 3, 1, 64], initializer=tf.truncated_normal_initializer()) bias = tf.get_variable("bias", shape=[64], initializer=tf.constant_initializer(0.1)) conv = tf.nn.conv2d(input, weight, strides=[1, 1, 1, 1], padding="SAME") activated_conv = tf.nn.relu(tf.nn.bias_add( conv, bias)) #不能直接使用加法,,因为矩阵上不同位置上的节点都要加上同样的偏置项 with tf.variable_scope("layer2-pool"): pool = tf.nn.max_pool(activated_conv, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") with tf.variable_scope("layer3_conv"): weight = tf.get_variable("weight", shape=[3, 3, 64, 128], initializer=tf.truncated_normal_initializer()) bias = tf.get_variable("bias", shape=[128], initializer=tf.constant_initializer(0.1)) conv = tf.nn.conv2d(pool, weight, strides=[1, 1, 1, 1], padding="SAME") activated_conv = tf.nn.relu(tf.nn.bias_add(conv, bias)) with tf.variable_scope("layer4-pool"): pool = tf.nn.max_pool(activated_conv, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") pool_shape = pool.get_shape().as_list() nodes = pool_shape[1] * pool_shape[2] * pool_shape[3] reshaped = tf.reshape(pool, [-1, nodes]) with tf.variable_scope("layer5-fc1"): weight = tf.get_variable( "weight", shape=[nodes, 512], initializer=tf.truncated_normal_initializer(stddev=0.1)) bias = tf.get_variable("bias", shape=[512], initializer=tf.constant_initializer(0.1)) if regu != None: regularizer = tf.contrib.layers.l2_regularizer(0.001) tf.add_to_collection("loss", regularizer(weight)) fc = tf.nn.relu(tf.matmul(reshaped, weight) + bias) if training: fc = tf.nn.dropout(fc, keep_prob=0.5) with tf.variable_scope("layer6-fc2"): weight = tf.get_variable( "weight", shape=[512, CAPATCHA_SIZE], initializer=tf.truncated_normal_initializer(stddev=0.1)) bias = tf.get_variable("bias", shape=[CAPATCHA_SIZE], initializer=tf.constant_initializer(0.1)) if regu != None: regularizer = tf.contrib.layers.l2_regularizer(0.001) tf.add_to_collection("loss", regularizer(weight)) logit = tf.matmul(fc, weight) + bias with tf.variable_scope("loss"): cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=logit)) loss = cross_entropy + tf.get_collection("loss") train_step = tf.train.AdamOptimizer(0.001).minimize(loss) variable_summary(loss, "loss") with tf.variable_scope("accuracy"): pred_correct = tf.equal(tf.argmax(logit, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(pred_correct, tf.float32)) variable_summary(accuracy, "accuracy") with tf.Session() as sess: tf.global_variables_initializer().run() train_writer = tf.summary.FileWriter("./log/train", sess.graph) test_writer = tf.summary.FileWriter("./log/test", sess.graph) merged = tf.summary.merge_all() for i in range(max_step): xs, ys = train_data.next_batch(batch_size) summary_step, _ = sess.run([merged, train_step], feed_dict={ x: xs, y_: ys }) train_writer.add_summary(summary_step, i) if i % 10 == 0: valid_summary, train_accuracy = sess.run([merged, accuracy], feed_dict={ x: xs, y_: ys }) train_writer.add_summary(valid_summary, i) # test_x, test_y = test_data.images,test_data.labels#会报错ResourceExhaustedError,内存不足,虽然测试用理论上是可以用全数据 test_x, test_y = test_data.next_batch(2000) test_summary, test_accuracy = sess.run([merged, accuracy], feed_dict={ x: test_x, y_: test_y }) test_writer.add_summary(test_summary, i) print( 'step %s, training accuracy = %.2f, testing accuracy = %.2f' % (i, train_accuracy, test_accuracy)) train_writer.close() test_writer.close() test_x, test_y = test_data.next_batch(200) test_accuracy = accuracy.eval(feed_dict={x: test_x, y_: test_y}) print("test_accuracy:%.2f" % test_accuracy)