def extract_regression_parameters(reg_file): test_artifact = read_zipped_json(reg_file) all_options = [] for i, test in enumerate(test_artifact): layers = extract_layers(test['model']['layers']) options = test['model']['training_options'] drate = options.get('dropout_rate', 0.0) bn = options.get('batch_normalization', False) if drate and not bn: layers = add_dropouts(layers, drate) for key in [ 'randomize', 'activation_functions', 'layer_sizes', 'activation_function', 'learn_residuals', 'batch_normalizaiton', 'dropout_rate' ]: options.pop(key, None) make_model(layer_sequence({'layers': layers}), (6, )) options['layers'] = layers options['seed'] = i all_options.append(options) return all_options
def make_and_run(input_model, _layers=None, lookback=12, epochs=4, **kwargs): input_features = [ 'tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm', 'wind_speed_mps', 'rel_hum' ] # column in dataframe to bse used as output/target outputs = ['blaTEM_coppml'] data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=lookback, lr=0.001, inputs=input_features, outputs=outputs, epochs=epochs, **kwargs) nn_config['layers'] = _layers df = pd.read_csv('../data/all_data_30min.csv') _model = input_model(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals) _model.build_nn() _ = _model.train_nn(indices='random') _ = _model.predict(use_datetime_index=False) return _model
def make_and_run(input_model, _layers=None, lookback=12, epochs=4, **kwargs): data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=lookback, lr=0.001, epochs=epochs, **kwargs) nn_config['layers'] = _layers df = pd.read_csv("../data/nasdaq100_padding.csv") _model = input_model(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals ) _model.build_nn() _ = _model.train_nn(indices='random') _ = _model.predict(use_datetime_index=False) return _model
def create_model(args): # define model # 加载模型结构 resnet = utils.make_model(args) model = resnet_model.resnext101_32x8d() # 加载模型参数 # 固定参数进行训练 一定要注意添加的位置 model.fc = nn.Sequential(nn.Dropout(0.5), nn.Linear(2048, args.num_classes)) summary(model.cuda(), input_size=(3, 288, 288)) exit() # 读取当前模型参数 model_dict = model.state_dict() # 将预训练的网络中不属于模型的层的参数剃掉 pretrained_dict = resnet.state_dict() pretrained_dict = { k: v for k, v in pretrained_dict.items() if k in model_dict } model_dict.update(pretrained_dict) model.load_state_dict(model_dict) # model = model.module print("load model's parameters success") exit() return model
def make_multi_model(input_model, from_config=False, config_path=None, weights=None, batch_size=8, lookback=19, lr=1.52e-5, ignore_nans=True, **kwargs): data_config, nn_config, total_intervals = make_model( batch_size=batch_size, lookback=lookback, lr=lr, ignore_nans=ignore_nans, **kwargs) data_config['inputs'] = [ 'tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)', 'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3', 'ORGP_INppm' ] data_config['outputs'] = [ 'obs_chla_1', 'obs_chla_3', 'obs_chla_8' # , 'obs_chla_12' ] fpath = "D:\\playground\\paper_with_codes\\dl_ts_prediction\\data" df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv')) df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv')) df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv')) # df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv')) df_1.index = pd.to_datetime(df_1['date']) df_3.index = pd.to_datetime(df_3['date']) df_8.index = pd.to_datetime(df_8['date']) # df_12.index = pd.to_datetime(df_12['date']) if from_config: _model = input_model.from_config( config_path=config_path, data=[ df_1, df_3, df_8 #, df_12 ]) _model.build_nn() _model.load_weights(weights) else: _model = input_model( data_config=data_config, nn_config=nn_config, data=[ df_1, df_3, df_8 # , df_12 ]) return _model
def stylize(args): content_image = utils.load_image_to_tensor(args.content_image,args.cuda) content_image.unsqueeze_(0) content_image = Variable(content_image) model = utils.make_model(args) model.load_state_dict(torch.load(args.model)) output_image = model(content_image) output_image = output_image.data output_image.squeeze_(0) utils.save_tensor_to_image(output_image,args.output_image,args.cuda)
def __init__(self, options): """ Args: options: configs.Config类的实例 """ self.options = options input_shape = (options.target_size[0], options.target_size[1], len(options.use_chans)) print('Making %s...'%options.model_name) if utils.is_py3(): #本地运行 best_model = os.path.join(options.output_path, options.weight_path) self.model = utils.make_model(self.options.model_name, input_shape) if tf.gfile.Exists(best_model): print('Load weights from %s.'%best_model) self.model.load_weights(best_model) else: #PAI上运行 path = os.path.join(options.output_path, options.weight_path) self.model = utils.make_model(self.options.model_name, input_shape) if tf.gfile.Exists(path): utils.copy_from_oss([path], using_cache=False) best_model = options.weight_path print('Load weights from %s.'%best_model) self.model.load_weights(best_model) self.log_path = '%s-log.txt'%options.run_name if utils.is_py3(): self.log_path = os.path.join(options.output_path, self.log_path) else: utils.copy_from_oss( [os.path.join(options.output_path, self.log_path)], using_cache=False) if self.options.init_epoch < 0: self.options.init_epoch = get_initial_epoch(self.log_path) self.metrics = [utils.f1_score] self.loss_function = utils.dice_coef_loss
def LeNet5(batch_size, num_particles): input_shape = _input_shape(batch_size) return make_model( stax.serial( GeneralConv(('NCHW', 'OIHW', 'NHWC'), out_chan=6, filter_shape=(5, 5), strides=(1, 1), padding="VALID"), Relu, MaxPool(window_shape=(2, 2), strides=(2, 2), padding="VALID"), Conv(out_chan=16, filter_shape=(5, 5), strides=(1, 1), padding="SAME"), Relu, MaxPool(window_shape=(2, 2), strides=(2, 2), padding="SAME"), Conv(out_chan=120, filter_shape=(5, 5), strides=(1, 1), padding="VALID"), Relu, MaxPool(window_shape=(2, 2), strides=(2, 2), padding="SAME"), Flatten, Dense(84), Relu, Dense(10), LogSoftmax), input_shape, num_particles)
import pandas as pd from utils import make_model from models import Model if __name__ == "__main__": input_features = [ 'tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm', 'wind_speed_mps', 'rel_hum' ] # column in dataframe to bse used as output/target outputs = ['blaTEM_coppml'] data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=1, inputs=input_features, outputs=outputs, lr=0.0001) df = pd.read_csv('../data/all_data_30min.csv') model = Model(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals) model.build_nn() history = model.train_nn(indices='random') y, obs = model.predict(st=0,
def main(args): """Main driver for evaluating different models. Can be used in both training and testing mode. """ if args.seed is None: args.seed = np.random.randint(1234567890) np.random.seed(args.seed) torch.manual_seed(args.seed) # Use the factory method to make both model and environment (e.g. every # call to make_env() will spawn a new environment using same parameters). # Note that models aren't very large and can be run quickly on CPU. env_creator = make_env(args.max_steps, args.is_test, args.render) model_creator = make_model(args, torch.device('cpu')) envs = [] for _ in range(args.remotes): envs.append( EnvWrapper.remote(env_creator, model_creator, args.seed_env)) # Trainable model does a significant amount of more work, so put on GPU device = torch.device( 'cpu' if args.no_cuda or not torch.cuda.is_available() else 'cuda') model = make_model(args, device)() if args.checkpoint is not None: model.load_checkpoint(args.checkpoint) # Train if not args.is_test: checkpoint_dir = os.path.join('checkpoints', args.model) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) # Some methods have specialized memory implementations memory = make_memory(args.model, args.buffer_size) memory.load(**vars(args)) # Perform a validation step every full pass through the data iters_per_epoch = args.buffer_size // args.batch_size # Keep a running average of n-epochs worth of rollouts step_queue = deque(maxlen=1 * args.rollouts * args.remotes) reward_queue = deque(maxlen=step_queue.maxlen) loss_queue = deque(maxlen=iters_per_epoch) best = -np.inf results = [] start = time.time() for episode in range(args.max_epochs * iters_per_epoch): loss = model.train(memory, **vars(args)) loss_queue.append(loss) if episode % args.update_iter == 0: model.update() # Validation step; # Here we take the weights from the current network, and distribute # them to all remote instances. While the network trains for another # epoch, these instances will run in parallel & evaluate the policy. # If an epoch finishes before remote instances, training will be # halted until outcomes are returned if episode % iters_per_epoch == 0: cur_episode = '%d' % (episode // iters_per_epoch) model.save_checkpoint(os.path.join(checkpoint_dir, cur_episode)) # Collect results from the previous epoch. # Note that all steps of a rollout are returned, but only the # last one will have the reward for that episode. for device in ray.get(results): for ep in device: # (s0, act, r, s1, terminal, timestep) step_queue.append(ep[-1][-1]) reward_queue.append(ep[-1][2]) # Update weights of remote network & perform rollouts results = test(envs, model.get_weights(), args.rollouts, args.explore) print('Epoch: %s, Step: %2.4f, Reward: %1.2f, Loss: %2.4f, '\ 'Took:%2.4fs' % (cur_episode, np.mean(step_queue), np.mean(reward_queue), np.mean(loss_queue), time.time() - start)) if np.mean(reward_queue) > best: best = np.mean(reward_queue) model.save_checkpoint(os.path.join(checkpoint_dir, 'best')) start = time.time() print('---------- Testing ----------') # Note if this happens naturally after the model has been trained # (and not from the --test flag), this will be wrong as it will use # the training objects instead of testing. results = test(envs, model.get_weights(), args.rollouts, args.explore) steps, rewards = [], [] for device in ray.get(results): for ep in device: # (s0, act, r, s1, terminal, timestep) steps.append(ep[-1][-1]) rewards.append(ep[-1][2]) print('Average across (%d) episodes: Step: %2.4f, Reward: %1.2f' % (len(rewards), np.mean(steps), np.mean(rewards)))
# this file tests that given activations are working both as layers as well as activation functions withing a layer from models import Model from utils import make_model import pandas as pd data_config, nn_config, _ = make_model() layers = {} for lyr in [ 'PRELU', "RELU", "TANH", "ELU", "LEAKYRELU", "THRESHOLDRELU", "SELU", 'sigmoid', 'hardsigmoid', 'crelu', 'relu6', 'softmax', 'softplus', 'softsign' ]: layers[lyr] = {'config': {}} layers["Dense"] = {'config': {'units': 1}} nn_config['layers'] = layers nn_config['epochs'] = 2 data_config['lookback'] = 1 df = pd.read_csv("../data/nasdaq100_padding.csv") model = Model(data_config, nn_config, df) model.build_nn() model.train_nn()
def train(args): torch.manual_seed(args.seed) train_loader_LR, train_loader_HR, dataset_len = utils.make_dataset(args) transformer = utils.make_model(args) if args.updata: transformer.load_state_dict(torch.load(args.modeldir)) vgg = utils.make_vggmodel(args) optimizer = Adam(transformer.parameters(), lr=args.lr) mse_loss = torch.nn.MSELoss() if args.cuda: mse_loss.cuda() for e in range(args.epochs): log_msg = "pix_weight = "+str(args.pix_weight)+" content_weight = "+str(args.content_weight) print(log_msg) transformer.train() agg_content_loss = 0 agg_pix_loss = 0 count = 0 for batch_id, ((x, x_),(style,y_)) in enumerate(zip(train_loader_LR,train_loader_HR)): n_batch = len(x) count += n_batch optimizer.zero_grad() pix_x_v = Variable(x) pix_s_v = Variable(style,requires_grad=False) pix_loss = 0 content_loss = 0 if args.cuda: x=x.cuda() style=style.cuda() pix_x_v=pix_x_v.cuda() pix_s_v=pix_s_v.cuda() output = transformer(pix_x_v) pix_loss = args.pix_weight * mse_loss(output, pix_s_v) vgg_s = Variable(style.clone(),requires_grad=False) vgg_x = utils.init_vgg_input(output) vgg_s = utils.init_vgg_input(vgg_s) feature_x = vgg(vgg_x) feature_s = vgg(vgg_s) f_s_v = Variable(feature_s[1].data, requires_grad=False) content_loss = args.content_weight * mse_loss(feature_x[1], f_s_v) total_loss = 0 if args.pix_weight>0: total_loss+=pix_loss if args.content_weight>0: total_loss+=content_loss total_loss.backward() optimizer.step() agg_content_loss += content_loss.data[0] agg_pix_loss += pix_loss.data[0] up = 10000 if(batch_id + 1) % args.log_interval == 0: mesg = "{}\tEpoch {}:\t[{}/{}]\tcontent: {:.6f}\tpix: {:.6f}\ttotal: {:.6f}".format(\ time.ctime(), e + 1, count, dataset_len,\ agg_content_loss*up / (batch_id + 1),\ agg_pix_loss *up/ (batch_id + 1),\ (agg_content_loss + agg_pix_loss)*up / (batch_id + 1)) print(mesg) if e%2 == 0: transformer.eval() transformer.cpu() save_model_filename = "epoch_" + str(args.epochs) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str( args.content_weight) + "_SRCNN_" + str(args.srcnn) + ".model" save_model_path = os.path.join(args.save_model_dir, save_model_filename) torch.save(transformer.state_dict(), save_model_path) transformer.eval() transformer.cpu() save_model_filename = "epoch_" + str(args.epochs) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str( args.content_weight) + "_SRCNN_" + str(args.srcnn) + ".model" save_model_path = os.path.join(args.save_model_dir, save_model_filename) torch.save(transformer.state_dict(), save_model_path) print("\nDone, trained model saved at", save_model_path)
def main(): """ The main executable function """ parser = make_argument_parser() args = parser.parse_args() input_dirs = args.inputdirs tf = args.factor valid_chroms = args.validchroms valid_input_dirs = args.validinputdirs test_chroms = args.testchroms epochs = args.epochs patience = args.patience learningrate = args.learningrate seed = args.seed utils.set_seed(seed) dropout_rate = args.dropout L = args.seqlen w = args.motifwidth utils.L = L utils.w = w utils.w2 = w / 2 negatives = args.negatives assert negatives > 0 meta = args.meta gencode = args.gencode motif = args.motif num_motifs = args.kernels num_recurrent = args.recurrent num_dense = args.dense features = ['bigwig'] if tf: print 'Single-task training:', tf singleTask = True if meta: print 'Including metadata features' features.append('meta') if gencode: print 'Including genome annotations' features.append('gencode') else: print 'Multi-task training' singleTask = False #Cannot use any metadata features assert not meta assert not gencode if args.outputdir is None: clobber = True output_dir = args.outputdirc else: clobber = False output_dir = args.outputdir try: # adapted from dreme.py by T. Bailey os.makedirs(output_dir) except OSError as exc: if exc.errno == errno.EEXIST: if not clobber: print >> sys.stderr, ( 'output directory (%s) already exists ' 'but you specified not to clobber it') % output_dir sys.exit(1) else: print >> sys.stderr, ('output directory (%s) already exists ' 'so it will be clobbered') % output_dir print 'Loading genome' genome = utils.load_genome() if valid_input_dirs: print 'You specified at least one validation input directory' assert singleTask # This option only works for single-task training print 'Loading ChIP labels' if singleTask: chip_bed_list, nonnegative_regions_bed_list = \ utils.load_chip_singleTask(input_dirs, tf) if valid_input_dirs: valid_chip_bed_list, valid_nonnegative_regions_bed_list = \ utils.load_chip_singleTask(valid_input_dirs, tf) num_tfs = 1 else: assert len( input_dirs) == 1 # multi-task training only supports one cell line input_dir = input_dirs[0] tfs, positive_windows, y_positive, nonnegative_regions_bed = \ utils.load_chip_multiTask(input_dir) num_tfs = len(tfs) print 'Loading bigWig data' bigwig_names, bigwig_files_list = utils.load_bigwigs(input_dirs) num_bigwigs = len(bigwig_names) if valid_input_dirs: valid_bigwig_names, valid_bigwig_files_list = utils.load_bigwigs( valid_input_dirs) assert valid_bigwig_names == bigwig_names if not singleTask: bigwig_files = bigwig_files_list[0] if meta: print 'Loading metadata features' meta_names, meta_list = utils.load_meta(input_dirs) if valid_input_dirs: valid_meta_names, valid_meta_list = utils.load_load( valid_input_dirs) assert valid_meta_names == meta_names else: # meta option was not selected, pass empty metadata features to the functions meta_list = [[] for bigwig_files in bigwig_files_list] if valid_input_dirs: valid_meta_list = [[] for bigwig_files in valid_bigwig_files_list] print 'Making features' if singleTask: if not valid_input_dirs: #validation directories not used, must pass placeholder values valid_chip_bed_list = None valid_nonnegative_regions_bed_list = None valid_bigwig_files_list = None valid_meta_list = None datagen_train, datagen_valid = \ utils.make_features_singleTask(chip_bed_list, nonnegative_regions_bed_list, bigwig_files_list, bigwig_names, meta_list, gencode, genome, epochs, negatives, valid_chroms, test_chroms, valid_chip_bed_list, valid_nonnegative_regions_bed_list, valid_bigwig_files_list, valid_meta_list) else: datagen_train, datagen_valid = \ utils.make_features_multiTask(positive_windows, y_positive, nonnegative_regions_bed, bigwig_files, bigwig_names, genome, epochs, valid_chroms, test_chroms) print 'Building model' if num_recurrent == 0: print 'You specified 0 LSTM units. Omitting BLSTM layer' if num_recurrent < 0: print 'You specified less than 0 LSTM units. Replacing BLSTM layer with global max-pooling layer' if meta or gencode: num_meta = 0 if meta: num_meta = len(meta_names) if gencode: num_meta += 6 model = utils.make_meta_model(num_tfs, num_bigwigs, num_meta, num_motifs, num_recurrent, num_dense, dropout_rate) else: model = utils.make_model(num_tfs, num_bigwigs, num_motifs, num_recurrent, num_dense, dropout_rate) if motif: assert singleTask # This option only works with single-task training motifs_db = utils.load_motif_db('resources/HOCOMOCOv9.meme') if tf in motifs_db: print 'Injecting canonical motif' pwm = motifs_db[tf] pwm += 0.001 pwm = pwm / pwm.sum(axis=1)[:, np.newaxis] pwm = np.log2(pwm / 0.25) utils.inject_pwm(model, pwm) output_tf_file = open(output_dir + '/chip.txt', 'w') if singleTask: output_tf_file.write("%s\n" % tf) else: for tf in tfs: output_tf_file.write("%s\n" % tf) output_tf_file.close() output_feature_file = open(output_dir + '/feature.txt', 'w') for feature in features: output_feature_file.write("%s\n" % feature) output_feature_file.close() output_bw_file = open(output_dir + '/bigwig.txt', 'w') for bw in bigwig_names: output_bw_file.write("%s\n" % bw) output_bw_file.close() if meta: output_meta_file = open(output_dir + '/meta.txt', 'w') for meta_name in meta_names: output_meta_file.write("%s\n" % meta_name) output_meta_file.close() model_json = model.to_json() output_json_file = open(output_dir + '/model.json', 'w') output_json_file.write(model_json) output_json_file.close() train(datagen_train, datagen_valid, model, epochs, patience, learningrate, output_dir)
self.k_model = self.compile(model_inputs=inputs, outputs=predictions) return if __name__ == "__main__": input_features = ['tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm', 'wind_speed_mps', 'rel_hum'] # column in dataframe to bse used as output/target outputs = ['blaTEM_coppml', 'sul1_coppml'] data_config, nn_config, total_intervals = make_model(batch_size=4, lookback=15, inputs=input_features, outputs=outputs, lr=0.0001, epochs=20, val_fraction=0.3, # TODO why less than 0.3 give error here? test_fraction=0.3 ) cwd = os.getcwd() df = pd.read_csv(os.path.join(os.path.dirname(cwd), "data\\all_data_30min.csv")) df.index = pd.to_datetime(df['Date_Time2']) model = MultiSite(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals )
def make_multi_model(input_model, from_config=False, config_path=None, weights=None, batch_size=8, lookback=19, lr=1.52e-5, **kwargs): data_config, nn_config, total_intervals = make_model(batch_size=batch_size, lookback=lookback, lr=lr, ignore_nans=True, **kwargs) lookback = data_config['lookback'] data_config['inputs'] = ['tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)', 'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3', 'ORGP_INppm'] data_config['outputs'] = ['obs_chla_1', 'obs_chla_3', 'obs_chla_8', 'obs_chla_12'] data_config['val_fraction'] = 0.2 fpath = "D:\\playground\\paper_with_codes\\dl_ts_prediction\\data" df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv')) df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv')) df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv')) df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv')) chl_1_nonan_idx = df_1[:-lookback][~df_1['obs_chla_1'][:-lookback].isna().values].index # 175 chl_3_nonan_idx = df_3[:-lookback][~df_3['obs_chla_3'][:-lookback].isna().values].index # 181 chl_8_nonan_idx = df_8[:-lookback][~df_8['obs_chla_8'][:-lookback].isna().values].index # 380 chl_12_nonan_idx = df_12[:-lookback][~df_12['obs_chla_12'][:-lookback].isna().values].index # 734 # len(list(set().union(chl_1_nonan_idx.to_list(), chl_3_nonan_idx.to_list(), chl_8_nonan_idx.to_list(), chl_12_nonan_idx.to_list() # ))) = 1162 train_idx_chl_1, test_idx_chl_1 = train_test_split(chl_1_nonan_idx, test_size=data_config['val_fraction'], random_state=313) train_idx_chl_3, test_idx_chl_3 = train_test_split(chl_3_nonan_idx, test_size=data_config['val_fraction'], random_state=313) train_idx_chl_8, test_idx_chl_8 = train_test_split(chl_8_nonan_idx, test_size=data_config['val_fraction'], random_state=313) train_idx_chl_12, test_idx_chl_12 = train_test_split(chl_12_nonan_idx, test_size=data_config['val_fraction'], random_state=313) _train_idx = list(set().union(train_idx_chl_1.to_list(), train_idx_chl_3.to_list(), train_idx_chl_8.to_list(), train_idx_chl_12.to_list() )) # 863 _test_idx = list(set().union(test_idx_chl_1.to_list(), test_idx_chl_3.to_list(), test_idx_chl_8.to_list(), test_idx_chl_12.to_list() )) # 406 df_1.index = pd.to_datetime(df_1['date']) df_3.index = pd.to_datetime(df_3['date']) df_8.index = pd.to_datetime(df_8['date']) df_12.index = pd.to_datetime(df_12['date']) if from_config: _model = input_model.from_config(config_path=config_path, data=[df_1, df_3, df_8, df_12]) _model.build_nn() _model.load_weights(weights) else: _model = input_model(data_config=data_config, nn_config=nn_config, data = [df_1, df_3, df_8, df_12] ) _model.build_nn() return _model, _train_idx, _test_idx
dirpath, cached=True) X_train = X_train.loc[:, [ 'sim_time', 'left_pwm', 'right_pwm', 'theta(t-1)_cos', 'theta(t-1)_sin' ]].values.reshape(-1, timestep, p) X_test = X_test.loc[:, [ 'sim_time', 'left_pwm', 'right_pwm', 'theta(t-1)_cos', 'theta(t-1)_sin' ]].values.reshape(-1, timestep, p) theta_y_train = y_train.loc[:, ['theta_cos', 'theta_sin']].values.reshape( -1, timestep, J) theta_y_test = y_test.loc[:, ['theta_cos', 'theta_sin']].values.reshape( -1, timestep, J) # pdb.set_trace() num_batches = 16 theta_model = make_model(num_batches, timestep, layers_dims, lr=1e-4) train_trial_names = load_obj(dirpath, 'train_trial_names') test_trial_names = load_obj(dirpath, 'test_trial_names') iterations = 3 epochs = 30 # learning curver train_loss_history = [] test_loss_history = [] # decoded_y_train = decode_angles(y_train) # y_test = decode_angles(y_test) for it in range(iterations): print("iteration %d" % it) history = theta_model.fit(X_train, theta_y_train,
def main(args): """Main driver for evaluating different models. Can be used in both training and testing mode. """ if args.seed is None: args.seed = np.random.randint(1234567890) np.random.seed(args.seed) torch.manual_seed(args.seed) # Make the remote environments; the models aren't very large and can # be run fairly quickly on the cpus. Save the GPUs for training env_creator = make_env(args.max_steps, args.is_test, args.render) model_creator = make_model(args, torch.device('cpu')) envs = [] for _ in range(args.remotes): envs.append(EnvWrapper(env_creator, model_creator, args.seed_env)) # We'll put the trainable model on the GPU if one's available device = torch.device( 'cpu' if args.no_cuda or not torch.cuda.is_available() else 'cuda') model = make_model(args, device)() if args.checkpoint is not None: model.load_checkpoint(args.checkpoint) # Train if not args.is_test: checkpoint_dir = os.path.join('checkpoints', args.model) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) # Some methods have specialized memory implementations memory = make_memory(args.model, args.buffer_size) memory.load(**vars(args)) # Perform a validation step every full pass through the data iters_per_epoch = args.buffer_size // args.batch_size # Keep a running average of n-epochs worth of rollouts step_queue = deque(maxlen=1 * args.rollouts * args.remotes) reward_queue = deque(maxlen=step_queue.maxlen) loss_queue = deque(maxlen=iters_per_epoch) start = time.time() for episode in range(args.max_epochs * iters_per_epoch): loss = model.train(memory, **vars(args)) loss_queue.append(loss) if episode % args.update_iter == 0: model.update() # Validation step; # Here we take the weights from the current network, and distribute # them to all remote instances. While the network trains for another # epoch, these instances will run in parallel & evaluate the policy. # If an epoch finishes before remote instances, training will be # halted until outcomes are returned if (episode + 1) % iters_per_epoch == 0: cur_episode = '%d' % (episode // iters_per_epoch) model.save_checkpoint(os.path.join(checkpoint_dir, cur_episode)) # Collect results from the previous epoch for device in test(envs, model.get_weights(), args.rollouts, args.explore): for ep in device: # (s0, act, r, s1, terminal, timestep) step_queue.append(ep[-1][-1]) reward_queue.append(ep[-1][2]) print('Epoch: %s, Step: %2.4f, Reward: %1.2f, Loss: %2.4f, '\ 'Took:%2.4fs' % (cur_episode, np.mean(step_queue), np.mean(reward_queue), np.mean(loss_queue), time.time() - start)) start = time.time() print('---------- Testing ----------') steps, rewards = [], [] for device in test(envs, model.get_weights(), args.rollouts, args.explore): for ep in device: # (s0, act, r, s1, terminal, timestep) steps.append(ep[-1][-1]) rewards.append(ep[-1][2]) print('Average across (%d) episodes: Step: %2.4f, Reward: %1.2f' % (args.rollouts * args.remotes, np.mean(steps), np.mean(rewards)))
print(out.isna().sum()) # put four chunks of missing intervals intervals = [(100, 200), (1000, 8000), (10000, 31000)] for interval in intervals: st, en = interval[0], interval[1] out[st:en] = np.nan df["NDX"] = out print("{} nan values created in NDX column".format(out.isna().sum())) # verify the missing values print(df[98:108]) data_config, nn_config, _ = make_model(batch_size=32, lookback=5, lr=0.0001) model = DualAttentionModel(data_config=data_config, nn_config=nn_config, data=df, intervals=[(0, 99), (200, 999), (8000, 9999), (31000, 40561)]) model.build_nn() history = model.train_nn(indices='random') y, obs = model.predict(indices=model.test_indices, use_datetime_index=False) # tr_y, tr_obs = model.predict(indices=model.train_indices, pref='train', use_datetime_index=False) model.view_model(st=0, save=True) # takes a lot of time to save all plots
"Dropout": { 'config': { 'rate': 0.3 } }, "Dense": { 'config': { 'units': 1 } } } data_config, nn_config, total_intervals = make_model(layers=layers, batch_size=12, lookback=15, lr=8.95e-5, ignore_nans=False, inputs=input_features, outputs=outputs, epochs=10) fname = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data\\data_30min.csv") df = pd.read_csv(fname) # must be 2d dataframe model = CustomModel(data_config=data_config, nn_config=nn_config, data=df) model.build_nn() history = model.train_nn(indices='random', tensorboard=True) test_pred, test_obs = model.predict(indices=model.test_indices)
from models import IMVLSTMModel from utils import make_model import pandas as pd lookback = 10 epochs = 50 df = pd.read_csv("../data/nasdaq100_padding.csv") data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=lookback, lr=0.001, use_predicted_output=True, epochs=epochs) model = IMVLSTMModel(data_config=data_config, nn_config=nn_config, data=df) model.build_nn() h = model.train_nn(st=0, en=1000) x, y = model.predict(st=0, en=1000) model.plot_activations()
split=True) # X columns: ['sim_time', 'left_pwm', 'right_pwm', 'model_pos_x(t-1)', 'model_pos_y(t-1)', 'theta(t-1)_cos', 'theta(t-1)_sin'] # pdb.set_trace() # X_train = X_train.values.reshape( # -1, timestep, p) # X_test = X_test.values.reshape( # -1, timestep, p) # y_train = y_train.values.reshape( # -1, timestep, J) # y_test = y_test.values.reshape( # -1, timestep, J) if model_cached: model = load_model(dirpath, model_fname) else: model = make_model(None, layers_dims) iterations = 300 epochs = 30 # learning curver train_loss_history = [] test_loss_history = [] # decoded_y_train = decode_angles(y_train) # y_test = decode_angles(y_test) # train_trial_names = load_obj(dirpath, 'train_trial_names') # test_trial_names = load_obj(dirpath, 'test_trial_names') test_fname = 'v_shape_path_2.csv' testfile = os.path.join(dirpath, test_fname)
embedding_std = model_config["embedding_weight_std"] max_positions = model_config["max_positions"] freeze_embedding = model_config["freeze_embedding"] trainable_positional_encodings = model_config[ "trainable_positional_encodings"] use_memory_mask = model_config["use_memory_mask"] query_position_rate = model_config["query_position_rate"] key_position_rate = model_config["key_position_rate"] window_backward = model_config["window_backward"] window_ahead = model_config["window_ahead"] key_projection = model_config["key_projection"] value_projection = model_config["value_projection"] dv3 = make_model( n_speakers, speaker_dim, speaker_embed_std, embed_dim, padding_idx, embedding_std, max_positions, n_vocab, freeze_embedding, filter_size, encoder_channels, n_mels, decoder_channels, r, trainable_positional_encodings, use_memory_mask, query_position_rate, key_position_rate, window_backward, window_ahead, key_projection, value_projection, downsample_factor, linear_dim, use_decoder_states, converter_channels, dropout) summary(dv3) # =========================loss========================= loss_config = config["loss"] masked_weight = loss_config["masked_loss_weight"] priority_freq = loss_config["priority_freq"] # Hz priority_bin = int(priority_freq / (0.5 * sample_rate) * linear_dim) priority_freq_weight = loss_config["priority_freq_weight"] binary_divergence_weight = loss_config["binary_divergence_weight"] guided_attention_sigma = loss_config["guided_attention_sigma"] criterion = TTSLoss( masked_weight=masked_weight,
}, 'Dropout_1': { 'rate': 0.3 }, 'Dense_2': { 'units': 16, 'activation': 'relu' }, 'Dense_3': { 'units': 9 } } data_config, nn_config, _ = make_model( inputs=['input_' + str(i) for i in range(cols - 1)], outputs=['input_' + str(cols - 1)], lookback=1, layers=layers, epochs=100) # Define Quantiles quantiles = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995] # Initiate Model model = QuantileModel(data_config, nn_config, data) # Assign loss for the model model.loss = qloss # the quantiles must also be assigned to the model for post-processing purpose model.quantiles = quantiles # Build the NN
# this example shows how to build the Models from `from_checkout` class method # first we will train and save a simple model and load it from config file import pandas as pd import os from utils import make_model from dl4seq.main import Model data_config, nn_config, total_intervals = make_model(lookback=1) fname = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data\\nasdaq100_padding.csv") df = pd.read_csv(fname) model = Model(data_config=data_config, nn_config=nn_config, data=df, ) model.build_nn() history = model.train_nn(indices='random') # for clarity, delete the model, although it is overwritten del model # Load the `Model` from checkpoint, provide the checkpoint cpath = "provide complete path of config file" model = Model.from_config(cpath, data=df)
print('Test f1 of %s: %0.3f.' % (options.run_name, f1)) print('Scoring time cost: %0.2f(min).' % ((time.time() - t0) / 60)) if __name__ == '__main__': options = None if utils.is_py3(): # Python3 options = configs.get_config('local_config_end2end.json') else: # Python2 options = configs.get_config(argparse.ArgumentParser()) test_files = json.load(open(options.train_val_test_config))['test'] input_shape = (options.target_size[0], options.target_size[1], len(options.use_chans)) model = utils.make_model(options.model_name, input_shape) if utils.is_py3(): p = os.path.join(options.output_path, options.weight_path) print("Load weights from '%s'." % p) model.load_weights(p) for k in range(len(test_files)): test_files[k] = os.path.join(options.input_path, options.data_path, test_files[k]) else: path = os.path.join(options.output_path, options.weight_path) if tf.gfile.Exists(path): print('Copy %s to %s.' % (path, options.weight_path)) tf.gfile.Copy(path, options.weight_path, overwrite=True) else: print('There is not pretrained model: %s' % path) model.load_weights(options.weight_path)
def main(): """We're going to create a beautiful app with Streamlit""" menu = [ "Accueil", "Pandas Profile", "D-Tale", "Visualisation", "Nuage de mots", "Machine Learning", "A propos" ] selection = st.sidebar.selectbox("Fonctions", menu) if selection == "Pandas Profile": components.html(ha.alert_panda_prof(), height=190) my_data = st.file_uploader("Charger le fichier CSV", type=['csv']) if my_data is not None: df = pd.read_csv(my_data) st.dataframe(df.head(10)) eda_profil = ProfileReport(df, title='Pandas Profiling Report...', explorative=True) st_profile_report(eda_profil) elif selection == "D-Tale": components.html(ha.alert_dtale(), height=190) data_file = st.file_uploader("Charger le fichier CSV", type=['csv']) if data_file is not None: df = pd.read_csv(data_file) st.dataframe(df.head()) d = dtale.show(df) d.open_browser() if st.button("Générer le rapport"): report = sv.analyze(df) report.show_html() utils.st_display_sweetviz("SWEETVIZ_REPORT.html") components.html(ha.alert_warning(), 1000) elif selection == "Visualisation": # Image d'entête image = Image.open('dataviz.png') col2, col1 = st.beta_columns([1, 3]) col2.image( "https://idoc-projets.ias.u-psud.fr/redmine/attachments/download/121/sunburst.gif", caption='', width=None, use_column_width=True) col1.image(image, caption='', width=None, use_column_width=True) # Visualisation catégorie / rating / reviews cat1, cat2 = st.beta_columns(2) datas = utils.lire_dataset(my_db_clean) fig = px.histogram(datas, x='Rating', y='Category', title='Somme des notes par catégorie', color='Category') cat1.plotly_chart(fig) fig = px.histogram(datas, x='Reviews', y='Category', title='Somme des commentaires par catégorie', color='Category') cat2.plotly_chart(fig) # Visualisation sunburst / pie perc1, perc2 = st.beta_columns(2) fig = px.sunburst(datas, path=['Type', 'Category', 'Genres'], title='Types, Catégories et genres') perc1.plotly_chart(fig) fig = px.pie(datas, names='Type', title='Pourcentage apllication gratuites/Payantes', color_discrete_sequence=px.colors.sequential.RdBu) perc2.plotly_chart(fig) # Visualisation histo hist1, hist2 = st.beta_columns(2) hist_data = [list(datas['Rating'])] group_labels = ['Rating'] fig = ff.create_distplot(hist_data, group_labels) hist1.plotly_chart(fig) counts, bins = np.histogram(datas.Rating, bins=range(0, 6, 1)) bins = 0.5 * (bins[:-1] + bins[1:]) fig = px.bar(x=bins, y=counts, labels={ 'x': 'Rating', 'y': 'Count' }, title='Distribution des notes') hist2.plotly_chart(fig) elif selection == "Nuage de mots": # General img1, img2 = st.beta_columns(2) img1.image('datas/wordcloud/general.png') img2.image('datas/wordcloud/free_app.png') # Free App img3, img4 = st.beta_columns(2) img3.image('datas/wordcloud/free_app_pos.png') img4.image('datas/wordcloud/free_app_neg.png') # Paid app img5, img6 = st.beta_columns(2) img5.image('datas/wordcloud/paid_app.png') img6.image('datas/wordcloud/paid_app_pos.png') img7, img8 = st.beta_columns(2) img7.image('datas/wordcloud/paid_app_neg.png') elif selection == "Machine Learning": image = Image.open('machine learning.jpg') col1, col2 = st.beta_columns([3, 1]) col2.image( "https://static.wixstatic.com/media/bb7b70_d5fde322f7914060b7d997ba9d506a50~mv2.gif", caption='', width=None, use_column_width=True) col1.image(image, caption='', width=None, use_column_width=True) if st.checkbox("Afficher le dataset"): datas = utils.lire_dataset(my_db) st.write(datas.head()) if st.checkbox("Afficher graph valeurs manquantes"): col1, col2 = st.beta_columns([2, 1]) df = datas.isnull() fig = px.imshow(df) col1.plotly_chart(fig) col2.write(datas.isnull().sum()) col2.write( "On peut voir que la colonne **Rating** contient la plupart des valeurs manquantes. A sa suite on a **Current Ver**, **Adroid ver** et **Type**." ) if st.checkbox("Afficher DB ok"): datas = utils.lire_dataset(my_db_clean) st.write(datas.head()) mat1, mat2 = st.beta_columns(2) fig = px.scatter_matrix( datas, dimensions=["Rating", "Reviews", "Size", "Installs", "Price"], color="Type", symbol="Type", title="Matrix de dispersion des variables continues") fig.update_traces(diagonal_visible=False) mat1.plotly_chart(fig) fig = px.imshow( datas[["Rating", "Reviews", "Size", "Installs", "Price"]].corr(), labels=dict(x="", y="", color="Corrélation"), ) mat2.plotly_chart(fig) if st.checkbox("Make model"): #mon_score = utils.transform_var_model(my_db_clean) mon_score = utils.make_model(my_db_clean) st.success(mon_score) elif selection == "A propos": #st.subheader("Team presentation") components.html(hp.pied_de_page(), height=800) #components.iframe('http://www.ingemedia.net/',height=1000, scrolling=True) else: components.html(hp.entete_de_page(), height=1600)
# Compute gradients trainable_vars = self.trainable_variables gradients = tape.gradient(loss, trainable_vars) # Update weights self.optimizer.apply_gradients(zip(gradients, trainable_vars)) # Update metrics (includes the metric that tracks the loss) self.compiled_metrics.update_state(y, y_pred) # Return a dict mapping metric names to current value return {m.name: m.result() for m in self.metrics} data_config, nn_config, total_intervals = make_model(lstm_units=64, dropout=0.4, rec_dropout=0.5, lstm_act='relu', batch_size=32, lookback=1, lr=8.95e-5) df = pd.read_csv('../data/nasdaq100_padding.csv') model = Model(data_config=data_config, nn_config=nn_config, data=df ) model.KModel = CustomModel model.build_nn()