def train(): """ Function that trains the model """ # Parameters for DataGrnerator constructor params = { 'dim': (15, 16, 3200), 'batch_size': FLAGS.batch_size, 'n_classes': 2, 'n_channels': 1, 'shuffle': True } # Get list of filenames for all training data and a dictionary of labels corresponding to each file name data, labels = get_data() with tf.name_scope('InputBatches'): # Generators for training and validation data generator_obj = {} training_generator = DataGenerator(data['train'], labels, **params) validation_generator = DataGenerator(data['valid'], labels, **params) # Stores the training and validation batches in a dict generator_obj['train'] = training_generator generator_obj['valid'] = validation_generator # Train the model training_fn(generator_obj)
def train_with_datagen(self): tensorboard = TensorBoard(log_dir="logs\\{}{}".format(self.model_name, time.time())) epoch_path = str(self.model_name) + '-{epoch:02d}.model' checkpoint = ModelCheckpoint(epoch_path, period=4) # training_boards = list(np.load('known_scores(2.7).npy').item().items()) # val_boards = list(np.load('test_set(166438)_old.npy').item().items()) training_boards = list(np.load('train(18000000)_new.npy').item().items()) val_boards = list(np.load('test(2000000)_new.npy').item().items()) # training_boards = list(np.load('expanded_train(3600000).npy').item().items()) # val_boards = list(np.load('expanded_test(400000).npy').item().items()) # training_boards = list(np.load('train_set(5000000)_FICS.npy').item().items()) # val_boards = list(np.load('test_set(500000)_FICS.npy').item().items()) if self.num_classes > 0: print('Ternary classification') training_generator = DataGenerator(training_boards, batch_size=128, categorical=True) validation_generator = DataGenerator(val_boards, batch_size=128, categorical=True) self.ternary_classifier() else: training_generator = DataGenerator(training_boards, batch_size=128) validation_generator = DataGenerator(val_boards, batch_size=128) self.regression_with_encoder() self.model.fit_generator(generator=training_generator, validation_data=validation_generator, use_multiprocessing=True, workers=2, epochs=32, verbose=True, callbacks=[tensorboard, checkpoint])
def main(): """ Script for evaluating MobileNet with testing dataset (self-driving car data) """ description = "Evaluate MobileNet with test dataset" parser = argparse.ArgumentParser(description=description) parser.add_argument('-model', '--trained_model', type=str, help='MobileNet trained model') parser.add_argument("-data_path", "--data", type=str, help="Path to dataset directory") user_args = parser.parse_args() batch_size = 16 path = os.path.join(user_args.data, 'test_labels.npy') labels = np.load(path) data_gen = DataGenerator(user_args.data) print('Evaluating model on {} samples'.format(labels.shape[0])) print('Class distribution:') count = np.unique(labels, return_counts=True) for i in range(3): print('{} : {}'.format(count[0][i], count[1][i])) model = load_model(user_args.trained_model) scores = model.evaluate_generator(data_gen.npy_generator(usage='test', batch_size=batch_size), steps=np.ceil(labels.shape[0] / batch_size).astype(int), verbose=1) print("Model performance: {}".format(scores[1]))
def generate_synthetic_data(mode: str, num_rows: int, description_filepath: str, synthetic_data_filepath: str): ''' Generates the synthetic data and saves it to the data/ directory. Keyword arguments: mode -- what type of synthetic data num_rows -- number of rows in the synthetic dataset description_filepath -- filepath to the data description synthetic_data_filepath -- filepath to where synthetic data written ''' generator = DataGenerator() if mode == 'random': generator.generate_dataset_in_random_mode(num_rows, description_filepath) elif mode == 'independent': generator.generate_dataset_in_independent_mode(num_rows, description_filepath) elif mode == 'correlated': generator.generate_dataset_in_correlated_attribute_mode( num_rows, description_filepath) generator.save_synthetic_data(synthetic_data_filepath)
def main(): go_file = TraceTxt() sce = Scenery() gra = Graphics() config = Config() fac_aps = FactoryAp() fac_users = FactoryUser() data_generator = DataGenerator() arq = go_file.abreArquivo('result.txt') arq_supervisioned = go_file.abreArquivo('supervisioned.txt') aps = fac_aps.createAps(config) users = fac_users.createUsers(config) dados = [] sp = SupervisedData() valuesClean = sp.get_values(config, aps) #chama o gerador de ddos apssando a lista de usuarios e aps(com os tributs de localizacao) for i in range(0, config.ciclo): dados, users = data_generator.createData(aps, config, users) #gera grafico sce.gerarGrafico(users, aps, i, config, valuesClean) #gera arquivo go_file.escreveArquivoScenery(i, arq, dados) go_file.escreveArquivoSupervisionado(arq_supervisioned, valuesClean) gra.plotagemPower(valuesClean, config) gra.heatMaps(aps, config) #gra.heatMapSinr(aps,config) go_file.fechaArquivo(arq)
def create_train_valid_data(negative_file, positive_files, num_of_classes, kernel_size=None, custom_file_limit=None): """Create generators for train an validation""" lines_from_all_files = RBNSreader.read_files(positive_files, file_limit=custom_file_limit) if USE_SHUFFLED_SEQS: lines_from_all_files += create_negative_seqs(lines_from_all_files) else: lines_from_all_files += RBNSreader.read_file(negative_file, 0, custom_file_limit * 2) np.random.shuffle(lines_from_all_files) valid_n = int(valid_p * len(lines_from_all_files)) validation_data = lines_from_all_files[:valid_n] train_data = lines_from_all_files[valid_n:] print('validation size', len(validation_data)) print('train size', len(train_data)) print('kernel size', kernel_size) train_gen = DataGenerator(train_data, num_of_classes=num_of_classes, kernel_sizes=kernel_size, max_sample_size=MAX_SAMPLE_SIZE, batch_size=BATCH_SIZE, shuffle=True) valid_gen = None if valid_n > 0: valid_gen = DataGenerator(validation_data, num_of_classes=num_of_classes, kernel_sizes=kernel_size, max_sample_size=MAX_SAMPLE_SIZE, batch_size=BATCH_SIZE, shuffle=True) return train_gen, valid_gen
def partition_data(self, data_split): self.data_split = data_split self.partition = {} self.partition["training"] = [] self.partition["validation"] = [] for ID in self.list_IDs: guitarist = int(ID.split("_")[0]) if guitarist == data_split: self.partition["validation"].append(ID) else: self.partition["training"].append(ID) self.training_generator = DataGenerator(self.partition['training'], data_path=self.data_path, batch_size=self.batch_size, shuffle=True, spec_repr=self.spec_repr, con_win_size=self.con_win_size) self.validation_generator = DataGenerator(self.partition['validation'], data_path=self.data_path, batch_size=len(self.partition['validation']), shuffle=False, spec_repr=self.spec_repr, con_win_size=self.con_win_size) self.split_folder = self.save_folder + str(self.data_split) + "/" if not os.path.exists(self.split_folder): os.makedirs(self.split_folder)
def get_data(self): with tf.name_scope('data'): # Training Data Preparation # DataGenerator to load data and extract patches for training datagen = DataGenerator() imgs = datagen.load_imgs_from_directory(directory = self.dir_train,filter='decorr*.mat',max_files=None) # Extracting overlapping training patches 256x256 images = datagen.generate_patches_from_list(imgs, shape=(256,256),num_patches_per_img=2000,augment=False) np.random.shuffle(images) X_train_noisy = images #####TRAINING #Compute mask for training images to exclude them in the loss computation indexes = np.where(X_train_noisy > self.clip) self.mask_train = np.ones_like(X_train_noisy,dtype=np.bool) self.mask_train[indexes] = False print('Clipping...') #Replace high backscatters with the median X_train_noisy = np.clip(X_train_noisy, 0, None) self.X_train_noisy_clipped = X_train_noisy medians = np.median(X_train_noisy,axis=[1,2],keepdims=True) self.X_train_noisy_clipped = np.where(self.X_train_noisy_clipped > self.clip,medians, X_train_noisy) print('Normalizing...') #Normalization self.X_train_noisy_clipped = (self.X_train_noisy_clipped.astype(np.float32)) self.X_train_noisy_clipped /= self.norm self.training_data_wrapper=DataWrapper(self.X_train_noisy_clipped,self.mask_train,self.batch_size,shape=(self.img_rows,self.img_cols)) self.training_data_iter=iter_sequence_infinite(self.training_data_wrapper) ######TEST IMAGES images_test = datagen.load_imgs_from_directory(directory = self.dir_test,filter='decorr_complex_tsx_SLC_0.mat') images_test = np.array(images_test) #cropping some test images images_test = np.array([images_test[0,0,i:i+1000,j:j+1000,:] for i,j in zip([5000,5500,4000,3000,0,1000,5500],[4000,5000,3500,7000,5000,5000,500])]) self.images_test = images_test #Compute mask for test images to be able to place the point targets back into the denoised estimate indexes = np.where(self.images_test > self.clip) self.mask_test = np.ones_like(self.images_test,dtype=np.bool) self.mask_test[indexes] = False #Clipping high backscattering FOR TEST self.images_test_clipped = np.clip(self.images_test, 0, None) medians = np.median(self.images_test,axis=[1,2],keepdims=True) self.images_test_clipped = np.where(self.images_test_clipped > self.clip,medians, self.images_test) #Normalization self.images_test_clipped = (self.images_test_clipped.astype(np.float32)) / self.norm
def main(): """ Script for evaluating MobileNet with testing dataset (self-driving car data) """ description = "Evaluate MobileNet with test dataset" parser = argparse.ArgumentParser(description=description) parser.add_argument('-model', '--trained_model', type=str, help='MobileNet trained model') parser.add_argument("-data", "--data", type=str, help="Path to dataset Images") parser.add_argument("-labels", "--labels", type=str, help="Path to dataset Labels") user_args = parser.parse_args() data = np.load(user_args.data) labels = np.load(user_args.labels) data_gen = DataGenerator('') prep_data, prep_labels = data_gen.preprocess_data(data, labels, balance='equals') del data print('Evaluating model on {} samples'.format(prep_labels.shape[0])) print('Class distribution:') for i in range(3): print('{} : {}'.format(i, np.sum(prep_labels[:, i]).astype(int))) model = load_model(user_args.trained_model) evaluate(model, prep_data, prep_labels)
def run_models(): # train_data, train_target, train_id = read_and_normalize_train_data() X_train, Y_train = get_images(train_dir,train_label) X_valid, Y_valid = get_images(val_dir,val_label) # callbacks functions checkpoint = ModelCheckpoint(weights_dir, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) earlystop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=3, verbose=1, mode='auto') csv_logger = CSVLogger(log_dir) callbacks = [checkpoint, earlystop, csv_logger] model, model_h, model_w = get_model() # Generators training_generator = DataGenerator(**params).generate(Y_train, X_train, model_h, model_w ) validation_generator = DataGenerator(**params).generate(Y_valid, X_valid, model_h, model_w ) # Train model on dataset model.fit_generator(generator=training_generator, steps_per_epoch=len(X_train) // batch_size, epochs=epochs, validation_data=validation_generator, validation_steps=len(X_valid) // batch_size, verbose=1), callbacks=callbacks)
def trainDataGenerator(model, batch_size, epochs, x_train, y_train, x_test, y_test): # Parameters params = { 'dim': (240, 320, 1), 'batch_size': 64, 'n_classes': 6, 'n_channels': 1, 'shuffle': True } dataSetPath = os.path.join(os.path.expanduser('~'), '.keras/datasets/skeleton') # Generators training_generator = DataGenerator(dataSetPath, x_train, y_train, **params) validation_generator = DataGenerator(dataSetPath, x_test, y_test, **params) ## TRAINING model.fit_generator(generator=training_generator, validation_data=validation_generator, epochs=epochs, verbose=1, shuffle=True, callbacks=[ TensorBoard(log_dir='/tmp/skeletonmodel', histogram_freq=0, write_graph=True, write_images=False) ]) model_path = '../test/skeletonmodel.h5' model.save(model_path) uploadFileToDrive(model_path)
def __init__(self, m = 20, n = 100, T = 1000, shutN = 10, shutC = 5, offP = 0.6, \ MapType = 'clust', ReqType = 'CS2', gaN = 5, upp = False, lP = 0): self.m = m # number of stations self.n = n # number of requests self.shutN = shutN # number of shuttles self.T = T # running time self.shutC = shutC # capacity of shuttle self.offP = offP # ratio of offline requests self.gaN = gaN # number of GA steps self.upp = upp # convert dists to upper bound self.lP = lP # acceptable Late time Policy self.MG = MapGenerator(self.m, MapType, self.upp) self.RG = RequestGenerator(self.MG, ReqType, self.n, self.T, self.offP) self.DG = DataGenerator(self.MG, self.RG, self.shutN, self.gaN, self.lP) self.requests = self.RG.requests[:] print(self.MG) print(self.RG) print('Stations : {m} | Requests : {r} | Shuttles : {s}\nTime : {t} | Off proportion : {o} | Capacity : {c}\n'\ .format(m=self.m, r=self.n, s=self.shutN, t=self.T, o=self.offP, c=self.shutC)) print('------------------------------------') self.rDS = self.RG.rDS() self.times = [] pass
def generate_data(train_imgs, val_imgs, labels_df, albumentations_train, batch_size, train_dir, resized_dims): """ Args: - train_imgs: list of images to be used for training - val_imgs: list of images to be used for validation - albumentations_train: data augmentation from albumentations package - batch_size: batch size for the generator - train_dir: directory containing training images - resized_dims: dimensions to which images will be resized to Returns: - data_generator_train: generator object for training images - data_generator_val: generator object for validation images """ data_generator_train = DataGenerator(train_imgs, label_vector=labels_df, dir_imgs=train_dir, resized_dims=resized_dims, batch_size=batch_size, augmentation=albumentations_train, shuffle=True) data_generator_val = DataGenerator(val_imgs, label_vector=labels_df, dir_imgs=train_dir, batch_size=batch_size, resized_dims=resized_dims, shuffle=False) return (data_generator_train, data_generator_val)
def __init__(self, caller, gyro, acc, grav, ext): super(CaptureThread, self).__init__() self.caller = caller self.gyro = gyro self.acc = acc self.grav = grav self.gen = DataGenerator() self.extra = ext
def train(dataset_path, copy_n): path_checkpoints = '/content/gdrive/Team Drives/Models/model-improvement-{epoch:02d}-{val_acc:.2f}.hdf5' file_train = os.path.join(dataset_path, 'train_data.npy') file_valid = os.path.join(dataset_path, 'valid_data.npy') x_train = np.load(file_train, mmap_mode='r') x_train_samples = x_train.shape[0] x_valid = np.load(file_valid, mmap_mode='r') x_valid_samples = x_valid.shape[0] del x_train, x_valid batch_size = 64 #epochs = 15 epochs = 25 base_model = pretrained.mobilenet.MobileNet(weights='imagenet') model = pretrained.mobilenet.MobileNet(weights=None, classes=3) # global frozen_weights # frozen_weights = base_model.get_layer('conv_pw_1').get_weights() keep(model, base_model) frozen(model, base_model, n=copy_n) del base_model sgd = optimizers.SGD(lr=0.01, decay=0.0005, momentum=0.9) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) stopper = EarlyStopping(monitor='val_acc', min_delta=0.0001, patience=3, verbose=1) checkpoint = ModelCheckpoint(path_checkpoints, monitor='val_acc', verbose=1, save_best_only=True, mode='max') # custom_callback = CustomCallback() data_gen = DataGenerator(dataset_path) model.fit_generator( data_gen.npy_generator(usage='train', batch_size=batch_size), steps_per_epoch=np.ceil(x_train_samples / batch_size).astype(int), validation_data=data_gen.npy_generator(usage='valid', batch_size=batch_size), validation_steps=np.ceil(x_valid_samples / batch_size).astype(int), callbacks=[stopper, checkpoint], epochs=epochs, verbose=1) model.save( "/content/gdrive/Team Drives/Models/A{}B+-mobilenet.h5".format(copy_n)) print("Saved model to disk")
def continue_model(infile, outdir, initial, data_amount=1000000, data_offset=0, validation_amount=100000, validation_offset=1000000, batch_size=8, slide = 5, epochs=15, verbosity = 1): ###FIX NUMPY LOAD FOR DICTIONARIES np_load_old = np.load np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k) mp.set_start_method("spawn",force=True) os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" ###Tensorflow session #config = tf.compat.v1.ConfigProto() #config.gpu_options.allow_growth = True #tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=config)) outdir = outdir % os.environ["SLURM_JOBID"] if not os.path.exists(outdir): os.makedirs(outdir) model = keras.models.load_model(initial) if len(model.layers[0].output_shape[0])==4: depth = 1 else: depth = model.layers[0].output_shape[0][1] multiple_inputs=False if len(model._input_layers) > 1: multiple_inputs=True #Initialize generators data_gen = DataGenerator(infile, data_amount=data_amount, batch_size=batch_size, frames_per_sample=depth, offset=data_offset, sliding_window=slide, labels_structured=multiple_inputs) validation_gen = DataGenerator(infile, data_amount=validation_amount, batch_size=batch_size, frames_per_sample=depth, offset=validation_offset, sliding_window=slide, labels_structured=multiple_inputs) #os.remove(initial) callbacks = [keras.callbacks.ModelCheckpoint(filepath= (outdir+'model_progress_{epoch:02d}.h5')), Logger.JupyterProgbarLogger(count_mode='steps',notebook=False)] history = model.fit_generator(generator=data_gen, validation_data=validation_gen, epochs=epochs, verbose=verbosity, use_multiprocessing=True, workers=16, callbacks=callbacks) print(model.summary())
def __init__(self, caller, gyro, acc, grav, classifier, ext=None): super(ProcessThread, self).__init__() self.caller = caller self.gyro = gyro self.acc = acc self.grav = grav self.gen = DataGenerator() self.extra = ext self.classifier = classifier
def __create_generator(self): train_start_seq = self.mysql_client.train_start_seq train_end_seq = self.mysql_client.train_end_seq validation_start_seq = self.mysql_client.validation_start_seq validation_end_seq = self.mysql_client.validation_end_seq self.train_generator = DataGenerator(train_start_seq, train_end_seq) self.validation_generator = DataGenerator(validation_start_seq, validation_end_seq)
def trainTheModelPart1(trData, trLabels, testData, testLabels, cv, model): ''' the network or networks for part 1 are trained here ''' if CONST_p1.part1Debuging: #show log if debuging from utils.logGeneration import findHowManyImagesInEachClass findHowManyImagesInEachClass(trData, "train") findHowManyImagesInEachClass(testData, "test") optimizer = CONST_p1.optimizer loss = CONST_p1.loss trWeights = [ CONST_p1.part1weightDict[trLabels[i]] for i in trLabels ] #creating array of weights for the data since the data is highly unbalanced #create data generator if CONST_p1.part1usePretrainedNetwork: params = CONST_p1.data_generator_params_pretrainedNet else: params = CONST_p1.data_generator_params training_generator = DataGenerator(trData, trLabels, **params) test_generator = DataGenerator(testData, testLabels, **params) #crete model if os.path.exists(CONST_p1.part1trainedModelSavePath ): #check if the model already exists model = keras.models.load_model(CONST_p1.part1trainedModelSavePath) else: if CONST_p1.part1usePretrainedNetwork: model = CONST_p1.model_pretrainedNet else: model = CONST_p1.model #compile model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) #fit history = model.fit_generator( generator=training_generator, validation_data=test_generator, class_weight=trWeights, **CONST_p1.training_parameters #callbacks=[keras.callbacks.EarlyStopping(monitor= 'val_acc',patience=3,mode = "max",baseline=80)]#if early stopping used ) #save and plot model.save(CONST_p1.part1trainedModelSavePath) plotAccAndLoss(history, cv) #clean del (history) model.reset_states() del (model) tf.reset_default_graph() keras.backend.clear_session()
def _done_convergence(self, area: Area, t_end: int): print('\n*********** Convergence is Done ! t = {} ***********'.format( t_end)) print('Targets Cells Probabilities: {}'.format( [self.p_S['t'][location] for location in area.targets_locations])) DataGenerator.tabulate_matrix(self.p_S['t']) # self._plot_target_searching(area=area, t=t_end) # self._plot_metrics(t=t_end) save_pickle_object(obj_name='p_S_converged', obj=self.p_S['t'])
def train(dataset_path): path_checkpoints = '/content/gdrive/Team Drives/Models/best_model.hdf5' # path_checkpoints = '/content/gdrive/Team Drives/Models/model-improvement-{epoch:02d}-{val_acc:.2f}.hdf5' file_train = os.path.join(dataset_path, 'train_labels.npy') file_valid = os.path.join(dataset_path, 'valid_labels.npy') x_train = np.load(file_train, mmap_mode='r') x_train_samples = x_train.shape[0] x_valid = np.load(file_valid, mmap_mode='r') x_valid_samples = x_valid.shape[0] del x_train, x_valid batch_size = 64 epochs = 25 # Model inputs = Input(shape=(10800,)) #working with already flatten image x = Dense(1333, activation='relu')(inputs) x = Dense(200, activation='relu')(x) predictions = Dense(3, activation='softmax')(x) model = Model(inputs=inputs, outputs=predictions) sgd = optimizers.SGD(lr=0.01, decay=0.0005, momentum=0.9) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) trainable_count = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)])) print("Total number of parameters: {}".format(trainable_count)) stopper = EarlyStopping(monitor='val_acc', min_delta=0.0001, patience=3, verbose=1) checkpoint = ModelCheckpoint(path_checkpoints, monitor='val_acc', verbose=1, save_best_only=True, mode='max') custom_callback = CustomCallback() global data_gen data_gen = DataGenerator(dataset_path) model.fit_generator(data_gen.npy_generator(usage='train', batch_size=batch_size), steps_per_epoch=np.ceil( x_train_samples / batch_size).astype(int), validation_data=data_gen.npy_generator( usage='valid', batch_size=batch_size), validation_steps=np.ceil( x_valid_samples / batch_size).astype(int), callbacks=[stopper, checkpoint, custom_callback], epochs=epochs, verbose=1)
def test_generator(args): generator = DataGenerator(args.data_root, "train", 90, (0.7, 1.2), (1, 3), 0.3, 0.3, 5, 32, (224, 224), 3, True, True, True) n_samples = 100 i = 0 for img in generator.generate(): i += 1 if i > n_samples: break
def main(): NewDataset = DataGenerator("./dataset/comments_cleaned_1st_phase.csv", "./dataset/digi_cleaned_phase2.csv" ) # This line doesn't have a valid save_path start_time = time.time() NewDataset.Data_Import() NewDataset.Data_cleaner() NewDataset.Save_Dataset() print("============================") print("Finished in %.2f s" % (time.time() - start_time))
def main(): dataGen = DataGenerator(iterationNum=flag.batch_size) print(dataGen.iterationNum) random_image_reconstruction = dataGen.get_random_images(10) print(random_image_reconstruction.shape) model = B_VAE( [None, dataGen.image_shape[0], dataGen.image_shape[1]],gamma= flag.gamma, capacity_limit= flag.capacity_limit,capacity_change_duration= flag.capacity_duration) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = load_checkpoints(sess) if flag.training: train(sess,model,dataGen, saver)
def main(): params = { 'dim': (8, 8), 'batch_size': 256, 'n_classes': 3, 'n_channels': 7, 'shuffle': True } IDs, num_IDs = setIDs('IDs_shuf.csv') pivot = int(np.floor(num_IDs * 0.8)) partitions = {'train': IDs[:pivot], 'validation': IDs[pivot:]} train_generator = DataGenerator(partitions['train'], **params) validation_generator = DataGenerator(partitions['validation'], **params) #X, y = train_generator._DataGenerator__data_generation(partitions['train'][0:params['batch_size']]) #print(X[0]) model = Sequential() model.add( Conv2D(32, kernel_size=(4, 4), data_format='channels_first', batch_size=params['batch_size'], batch_input_shape=(params['batch_size'], 7, 8, 8))) model.add(Activation('relu')) model.add(Conv2D(64, kernel_size=(2, 2), data_format='channels_first')) model.add(Flatten()) model.add(Dense(3)) model.add(Activation('softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) filepath = "/data/models" checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, mode='max') model.fit_generator(generator=train_generator, validation_data=validation_generator, use_multiprocessing=True, workers=16, epochs=10, verbose=1, callbacks=[checkpoint])
def main(): gains = [40, 50, 60] tx_beams = np.arange(0, 24) num_samples_tot_gain_tx_beam = 10000 # Order is gain * indexes = np.arange( 0, num_samples_tot_gain_tx_beam * len(tx_beams) * len(gains) ) batch_size = 32 data_path = '/media/michele/rx-12-tx-tm-0-rx-tm-1.h5' num_blocks_per_frame = 15 how_many_blocks_per_frame = 1 num_samples_per_block = 2048 num_tx_beams = len(tx_beams) input_size = 1024 dg = DataGenerator( indexes, batch_size, data_path, num_tx_beams, num_blocks_per_frame, input_size, num_samples_per_block, how_many_blocks_per_frame, shuffle=False, is_2d=False ) batch_gain_tx_beam = num_samples_tot_gain_tx_beam / batch_size # for [i_g, val_g] in enumerate(gains): # print("Gain: " + str(val_g)) # for [i_t, val_t] in enumerate(tx_beams): # print("Beam idx: " + str(val_t)) # batch_index = (i_g * len(tx_beams) * batch_gain_tx_beam) + i_t * batch_gain_tx_beam # print("Batch idx: " + str(batch_index)) # [batch, batch_y] = dg.__getitem__(batch_index) # print("tx_beam %d y % s" % (val_t, batch_y[0])) # # print(batch_y[0]) for i in range(dg.__len__()): print("Batch idx: " + str(i)) [batch, batch_y] = dg.__getitem__(i) print("tx_beam %s %s y %s %s" % (batch[0][0], batch[-1][0], batch_y[0], batch_y[-1])) print("batch_x_size: %s, batch_y_size: %s" % (str(batch.shape), str(batch_y.shape)))
def bayesian_updating_iteration(params, until_convergence: bool = True, sensitivity_analysis: bool = False): start_time = dt.now() DataGenerator.param_validation(params=params) targets_locations = eval(params['TARGET_LOCATIONS']) if targets_locations is None and params.getint('NUM_TARGETS'): targets_locations = Area.generate_targets( num_targets=params.getint('NUM_TARGETS')) area = Area(num_cells_axis=params.getint('N'), t_interval=params.getint('T_INTERVAL'), pta=params.getfloat('pta'), alpha=params.getfloat('alpha'), targets_locations=targets_locations) agent = Agent(current_location=eval(params['AGENT_POSITION']), lambda_strength=params.getint('LAMBDA_SENSOR'), p_S=Agent.get_p_S_from_initial_prior( prior=params.getfloat('INITIAL_PRIOR_P'), area=area), entropy_updates=[], information_gain_updates=[], converge_iter=200 - 1, p_S_history=[]) """ 2 Running modes: until convergence (all the targets identified and not False Positives) or until infinity """ if not sensitivity_analysis: agent.bayesian_update(area=area, until_convergence=until_convergence, verbose=True, stop_after_iteration=200) print('Bayesian Updating Done in {}'.format(dt.now() - start_time)) else: target_prob = agent.get_prob_history(targets_locations[2]) # non_target_loc = random.choice([loc for loc in zip(range(params.getint('N')), range(params.getint('N'))) if loc not in targets_locations]) # Randomly selected non targets list_of_loc = [(9, 10), (7, 15), (1, 16)] agent_pos = eval(params['AGENT_POSITION']) non_target_prob = {} for loc in list_of_loc: dist = np.round( np.sqrt( np.power(loc[0] - agent_pos[0], 2) + np.power(loc[1] - agent_pos[1], 2))) non_target_prob[dist] = agent.get_prob_history(loc) ent_upd, info_gain_upd, conv_iter = agent.get_metrics() return ent_upd, info_gain_upd, conv_iter, target_prob, non_target_prob
def __BuildGenerators(self): dataTrain, labelsTrain, dataTest, labelsTest = TrainTestSplit( self.data, self.labels, .95) self.train_generator = DataGenerator(dataTrain, labelsTrain, self.batch_size, self.vocab_size, self.max_input_len, self.max_output_len, self.token_index) self.valid_generator = DataGenerator(dataTest, labelsTest, self.batch_size, self.vocab_size, self.max_input_len, self.max_output_len, self.token_index)
def train_model(num_classes=5, num_samples=1, meta_batch_size=16, random_seed=1234, itr_cnt=25000, cell_count=128, lr=0.001, model=None, logDir='./logs', modelDir='./models', shuffle=True): random.seed(random_seed) np.random.seed(random_seed) tf.random.set_seed(random_seed) data_generator = DataGenerator(num_classes, num_samples + 1) o = model if o is None: o = MANN2(num_classes, num_samples + 1, cell_count=cell_count) optim = tf.keras.optimizers.Adam(learning_rate=lr) modelName = f'K={num_samples}&N={num_classes}&CellCount={cell_count}&BS={meta_batch_size}' writer = tf.summary.create_file_writer(f'{logDir}/{modelName}/') with writer.as_default(): for step in range(itr_cnt): i, l = data_generator.sample_batch('train', meta_batch_size, shuffle=shuffle) _, ls = train_step(i, l, o, optim) if (step + 1) % 100 == 0: print("*" * 5 + "Iter " + str(step + 1) + "*" * 5) i, l = data_generator.sample_batch('test', 100) pred, tls = train_step(i, l, o, optim, eval=True) pred = tf.reshape( pred, [-1, num_samples + 1, num_classes, num_classes]) pred = tf.math.argmax(pred[:, -1, :, :], axis=2) l = tf.math.argmax(l[:, -1, :, :], axis=2) test_accuracy = tf.reduce_mean( tf.cast(tf.math.equal(pred, l), tf.float32)).numpy() tf.summary.scalar('Test Accuracy', test_accuracy, step=step) tf.summary.scalar('Training Loss', ls.numpy(), step=step) tf.summary.scalar('Test Loss', tls.numpy(), step=step) writer.flush() o.save_weights(f'{modelDir}/{modelName}') print("Test Accuracy", test_accuracy, "Train Loss:", ls.numpy(), "Test Loss:", tls.numpy())
def test(self): # Generate training set test_set = self.Data.unseen_pairs test_gen = DataGenerator(pairs=test_set, batch_size=self.param_grid['batch_size'], nr_files=len(self.Data.all_files), nr_tests=len(self.Data.all_tests), negative_ratio=self.param_grid['negative_ratio']) X, y = next(test_gen.data_generation(test_set)) pred = self.model.predict(X) pred[pred < 0.5] = 0 pred[pred >= 0.5] = 1 return y, pred
def generate_graph(self): self.params = { 'graph' : None, 'start_idx' : int(self.start_airport.text()), 'end_idx' : int(self.destination_airport.text()), 'max_flights' : int(self.max_flights.text()), 'cost_weight' : int(self.cost_weight.text()), 'time_weight' : int(self.time_weight.text()), 'pop_size' : int(self.pop_size.text()), 'generations' : int(self.generation.text()), 'mutation_rate' : float(self.mutation_rate.text()), 'tournament_size' : int(self.tournament_size.text()), 'elitism' : bool(self.elitism.currentText()), 'dest_min' : int(self.dest_min.text()), 'dest_max' : int(self.dest_max.text()), 'max_flights' : 4, } data = DataGenerator() DataGenerator.DESTINATIONS_MIN = self.params['dest_min'] DataGenerator.DESTINATIONS_MAX = self.params['dest_max'] # if input_graph_file is not None: # data.load_saved_graph(input_graph_file) # # else: #TODO ilosc lotnisk data.load_new_data(10) data.create_graph() # if graph_save_file is not None: # data.save_graph(graph_save_file) testsuite_airports = data.get_airports() testsuite_graph = data.get_graph() self.graph = GraphManager(self.params['max_flights']) self.graph.set_graph(testsuite_graph, testsuite_airports) airports_parser = Testsuite_airports_parser(testsuite_airports)
from matplotlib.ticker import LinearLocator, FormatStrFormatter from matplotlib import cm import numpy as np def take_integer_from_user(message): inp = raw_input(message) while True: try: int(inp) break except ValueError: inp = raw_input('You must specify a number: ') return int(inp) fig_swiss = plt.figure() dg = DataGenerator() inp = take_integer_from_user('Press 1 for photos, 2 for Swiss Roll: ') if inp == 1 : a = dg.generateFacesPoints("C:\\Users\\Elrassam\\Desktop\\Faces\\s") numOfPoints = 400 elif inp == 2: numOfPoints = take_integer_from_user('Enter the number of points: ') a=np.zeros((numOfPoints,3)) ins = open( "1500.txt", "r" ) i = 0 for line in ins: j = 0 nums = line.split(' ') for nu in nums: a[i,j] = nu j += 1
import random, time import numpy as np #A test program to test data generator from DataGenerator import DataGenerator from GridMessageProtocol import * dg=DataGenerator(n=1, m=8, randseed=1024) #print "0:0" #print "-1:1" while True: x=dg.generate_measurement(t=time.time(), id_range=(0,8)) alist=[item for sublist in x for item in sublist] print "0:%f" %alist[5]
# Customize output #################################### CLASSNAMES = ['A','B','C','D'] NUMBER_OF_CLASSES = len(CLASSNAMES) NUMBER_OF_DIMENSIONS = 10 NUMBER_OF_SAMPLES_PER_CLASS = 2000 #################################### if __name__ == '__main__': # GENERATE DEPENDENT DATA generate = DataGenerator() samples = [] trees = {} for c in CLASSNAMES: tree = generate.generateTree(['X'+`i` for i in range(NUMBER_OF_DIMENSIONS)]) data = generate.generateDependentData(NUMBER_OF_SAMPLES_PER_CLASS, c, tree) sample = data['samples'] for s in sample: s.append(data['name']) samples.append(s) trees[c] = tree saveJSONtoFile("flare.json", trees['A'].getJSONFormat()) testingSet = []
if int(raw_input("Jesli chcesz domysle wartosci -1, jesli nie -2\n")) == 2: per = float(raw_input("podaj liczbe okresow\n")) fr = float(raw_input("podaj frame_rate\n")) fq = float(raw_input("podaj czestotliwosc\n")) ns = float(raw_input("podaj poziom szumu\n")) am = float(raw_input("podaj amplitude\n")) off = float(raw_input("podaj offset\n")) else: per = 4 fr = 44100 fq = 200 ns = 100 am = 1 off = 0 generator = DataGenerator() generator.change_number_of_periods(per) generator.change_framerate(fr) generator.change_frequency(fq) generator.change_noise(ns) generator.change_amplitude(am) generator.change_offset(off) def my_sin(x, freq, amplitude, phase, offset): #musi byc list comprehention bo wywala blad przy mnozeniu listy przez float'a return np.sin([i * freq + phase for i in x]) * amplitude + offset p0 = sy.array([1,1,1,1]) def func(x, a, b, c):
T = times.shape[0] times.shape = (T,1) n = state.shape[0] x0 = state[:,0].T xt = state[:,1].T x0.shape = ( 1,n ) xt.shape = ( 1,n ) x = x0.repeat( T, 0 ) + sp.dot( times-t0, xt ) return x if __name__ == "__main__": import matplotlib.pyplot as plt m = 8 n = 8 dg = DataGenerator(n=n,m=m) se = StateEstimator(dg.EstimationMatrix) time_offset = time.time() # time_offset = 0.0 times = sp.arange(time_offset+0.0, time_offset+1.7, 1.0/3.0).reshape(-1,1) T = times.shape[0] print "==times.shape[0]==" print times.shape[0] # state = sp.hstack( [ (sp.pi-times)*sp.cos(2*times), (1+times)*sp.sin(3*times)] ) state = sp.vstack( [ dg.generate_state(t).T for t in times ] ) print "==state.shape==" print state.shape print "===state===" print state