def main(): parser = argparse.ArgumentParser() arg = parser.add_argument arg('--lr',type=float,default=0.1) arg('--n_epochs',type=int, default = 5) arg('--batch-size',type=int, default= 32) arg('--data_dir',type=str,default = 'chest_xray') arg('--model',type=str, default ='chexnet',choices = model_list.keys()) arg('--root',type=str,default ='runs/debug', help = 'checkpoint root') args = parser.parse_args() train_loader= generate_trainloaders(data_dir= args.data_dir, batch_size= args.batch_size) root = Path(args.root) root.mkdir(exist_ok = True, parents = True) model = model_list[args.model] loss = CrossEntropyLoss() utils.fit( init_optimizer= lambda lr:SGD(model.parameters(),lr = args.lr), args= args, model = model, train_loader = train_loader, criterion= loss, n_epochs= args.n_epochs, train_on_gpu=check_gpu(), dir_save = args.root, lr = args.lr, base_model=args.model )
def rollout_mcmc(x, bounds, func_policy, depth_h, _queries, _values, N_q = 5, n_sample=10, decay_rate=.9, ARD_Flag = False, length_scale = None ): if len(x.shape) == 1: x = np.array([x]) kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) gp_model = fit(_queries, _values, kernel) U = ei(x,bounds,gp_model) if depth_h == 0: return U else: queriesori = np.copy(_queries) valuesori = np.copy(_values) Udelays = np.array([]) _mu, _sig = gp_model.predict(x) for i in range(n_sample): _Udelay = 0 _queriesf = np.copy(queriesori) _valuesf = np.copy(valuesori) #kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) #gp_model = fit(_queries, _values, kernel) #predict 1st step #mu, sig = gp_model.predict(x) y_next = np.array(np.random.normal(_mu, _sig)) _queriesf = np.concatenate([_queriesf,x]) _valuesf = np.concatenate([_valuesf,y_next]) for j in range(depth_h): _remain_h = depth_h - j - 1 #kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) gp_model = fit(_queriesf, _valuesf, kernel) x_next = func_policy(gp_model, _remain_h, bounds) _Udelay += decay_rate * ei(x_next,bounds,gp_model) _queriesf = np.concatenate([_queriesf,x_next]) mu, sig = gp_model.predict(x_next) next_y = np.array(np.random.normal(mu, sig)) _valuesf = np.concatenate([_valuesf,next_y]) Udelays = np.append(Udelays, _Udelay) U += np.mean(Udelays) return U
def rollout_utility_archive(x, bounds, func_policy, depth_h, _queries, _values, N_q, n_sample=None, decay_rate=0.9, ARD_Flag=False, length_scale=None): #print(depth_h) global U if len(x.shape) == 1: x = np.array([x]) kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) gp_model = fit(_queries, _values, kernel) #todo:memo if depth_h == 0: U += ei(x, bounds, gp_model) else: U += ei(x, bounds, gp_model) _queries = np.concatenate([_queries, x]) points, weights = gauss_hermite(x, gp_model, N_q) for i in range(N_q): val = np.array([[points[0][i]]]) _values = np.concatenate([_values, val]) kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) #print("X",_queries) #print("Y",_values) _gp_model = fit(_queries, _values, kernel) #todo:memo #print(i,"afterfit_afterker") x_next = func_policy(_gp_model, depth_h, bounds) U = U + weights[i] * decay_rate * rollout_utility_archive( x_next, bounds, func_policy, depth_h - 1, _queries, _values, N_q, decay_rate, ARD_Flag=ARD_Flag, length_scale=length_scale) _values = np.copy(_values[:-1, :]) _queries = np.copy(_queries[:-1, :]) _U = U U = 0 return _U
def load_fragments(): from residue import Residue import os ref = Residue('REF') ref.append_atom(Atom( 'N', [0.000, 0.300, 0.000])) ref.append_atom(Atom('CA', [1.181, -0.540, 0.000])) ref.append_atom(Atom( 'C', [2.440, 0.300, 0.000])) # load all residues this_dir, this_filename = os.path.split(__file__) DATA_PATH = os.path.join(this_dir, "static", "residues.pdb") residue_container = Molecule.LoadFromFile(DATA_PATH) for fragment in residue_container.iter_residues(): fit(ref, ('N','CA','C'), fragment, ('N','CA','C')) FragmentProvider.FRAGMENTS[fragment.name] = fragment
def place_catwalk_connections(instances, point_a, point_b): """Place catwalk sections to connect two straight points.""" diff = point_b - point_a # The horizontal unit vector in the direction we are placing catwalks direction = diff.copy() direction.z = 0 distance = direction.len() - 128 direction = direction.norm() if diff.z > 0: angle = INST_ANGLE[direction.as_tuple()] # We need to add stairs for stair_pos in range(0, int(diff.z), 128): # Move twice the vertical horizontally # plus 128 so we don't start in point A loc = point_a + (2 * stair_pos + 128) * direction # Do the vertical offset loc.z += stair_pos VMF.create_ent(classname="func_instance", origin=loc.join(" "), angles=angle, file=instances["stair"]) # This is the location we start flat sections at point_a = loc + 128 * direction point_a.z += 128 elif diff.z < 0: # We need to add downward stairs # They point opposite to normal ones utils.con_log("down from", point_a) angle = INST_ANGLE[(-direction).as_tuple()] for stair_pos in range(0, -int(diff.z), 128): utils.con_log(stair_pos) # Move twice the vertical horizontally loc = point_a + (2 * stair_pos + 256) * direction # Do the vertical offset plus additional 128 units # to account for the moved instance loc.z -= stair_pos + 128 VMF.create_ent(classname="func_instance", origin=loc.join(" "), angles=angle, file=instances["stair"]) # Adjust point A to be at the end of the catwalks point_a = loc # Remove the space the stairs take up from the horiz distance distance -= abs(diff.z) * 2 # Now do straight sections utils.con_log("Stretching ", distance, direction) angle = INST_ANGLE[direction.as_tuple()] loc = point_a + (direction * 128) # Figure out the most efficent number of sections for segment_len in utils.fit(distance, [512, 256, 128]): VMF.create_ent( classname="func_instance", origin=loc.join(" "), angles=angle, file=instances["straight_" + str(segment_len)], ) utils.con_log(loc) loc += segment_len * direction
def fit(self, inputs, outputs, num_epochs=10): """Train projector given masked faces as input and transparency masks as outputs. # Arguments * `inputs` - a batch of faces to train on * `outputs` - the ID of every input face * `num_epochs` - number of epochs to run, defaults to 10 """ self.train() return utils.fit(self, inputs, outputs, num_epochs=num_epochs)
def preprocess(self, training_data_path): '''''' def tokenizer(iterator): for value in iterator: yield value tokens, labels = read_data(training_data_path) model_dir = self.params[constants.PARAM_KEY_MODEL_DIR] if self.label_vocab is None: logging.info("generating label vocabulary ...") self.label_vocab = learn.preprocessing.VocabularyProcessor( max_document_length=self.params[ constants.PARAM_KEY_MAX_DOCUMENT_LEN], tokenizer_fn=tokenizer) self.label_vocab.fit(labels) logging.info("label vocabulary size = %d", len(self.label_vocab.vocabulary_)) label_vocab_path = os.path.join(model_dir, constants.FILENAME_LABEL_VOCAB) deeptext.utils.serialization.save(self.label_vocab, label_vocab_path) self.label_ids = self.preprocess_label_transform(labels) self.params[constants.PARAM_KEY_LABEL_VOCAB_SIZE] = len( self.label_vocab.vocabulary_) self.tokens = tokens print constants.TENSOR_NAME_TOKENS print self.params[constants.PARAM_KEY_MAX_DOCUMENT_LEN] print self.params[constants.PARAM_KEY_EMBEDDING_SIZE] self.word2vec_model = fit( self.tokens, os.path.join(model_dir, constants.FILENAME_WORD2VEC_MODEL)) self.tensor_tokens = tf.placeholder( dtype=tf.float32, name=constants.TENSOR_NAME_TOKENS, shape=[ None, self.params[constants.PARAM_KEY_MAX_DOCUMENT_LEN], self.params[constants.PARAM_KEY_EMBEDDING_SIZE] ]) self.tensor_labels = tf.placeholder_with_default( self.label_ids, name=constants.TENSOR_NAME_LABELS, shape=[None, self.params[constants.PARAM_KEY_MAX_DOCUMENT_LEN]]) print 'build_model' self.build_model(self.tensor_tokens, self.tensor_labels)
def grow(chain, *frag_names): ''' grow(chain, *fragment_names) Grow a peptide chain by appending fragments to the C-terminus of the chain. ''' if not frag_names: return chain if chain.count_residues() == 0: fragment = FragmentProvider.get_fragment(frag_names[0]) chain.append_residue(fragment, is_head=True) return grow(chain, *frag_names[1:]) tail = chain.residues[-1] for n,frag_name in enumerate(frag_names, start=1): fragment = FragmentProvider.get_fragment(frag_name) if n == 1: R,T = fit(fragment, ('N','CA','C'), tail, ('N','CA','C')) invR = np.linalg.inv(R) # calculate new fragment's coordinates (flipped and translated), xyz = fragment.get_coordinates()*[1,(-1)**n,1] + [n*3.638,0,0] xyz = np.dot(invR, np.transpose(xyz - T)).T fragment.set_coordinates(xyz) chain.append_residue(fragment) # take the tail residue back to its original position xyz = tail.get_coordinates() xyz = np.dot(invR, np.transpose(xyz - T)).T tail.set_coordinates(xyz) return chain
sigma_depth = (2. * sigma) / np.sqrt(I) # If initial SNR estimate is larger than 5-sigma, perform the fit: if tdepth / sigma_depth > 5: print( '\t >> Performing fit for ' + sector + '; expected depth precision FOR ALL SECTORS: ', sigma_depth * 1e6, ' giving SNR:', tdepth / sigma_depth) if not os.path.exists(planet): os.mkdir(planet) full_path = planet + '/' + sector utils.fit(t[sector], f[sector], ferr[sector], sector, period, period_err, t0, t0_err, ecc, omega, GPmodel = 'ExpMatern', outpath = full_path, \ method = method, in_transit_length = factor*tdur) utils.fit(t[sector], f[sector], ferr[sector], sector, period, period_err, t0, t0_err, ecc, omega, GPmodel = 'QP', outpath = full_path, \ method = method, in_transit_length = factor*tdur) if fit_catwoman: utils.fit(t[sector], f[sector], ferr[sector], sector, period, period_err, t0, t0_err, ecc, omega, GPmodel = 'ExpMatern', outpath = full_path, \ method = method, in_transit_length = factor*tdur, fit_catwoman = fit_catwoman) utils.fit(t[sector], f[sector], ferr[sector], sector, period, period_err, t0, t0_err, ecc, omega, GPmodel = 'QP', outpath = full_path, \ method = method, in_transit_length = factor*tdur, fit_catwoman = fit_catwoman) good_sectors.append(sector) else: print( '\t WARNING: ', sector,
def place_catwalk_connections(vmf: VMF, instances, point_a: Vec, point_b: Vec): """Place catwalk sections to connect two straight points.""" diff = point_b - point_a # The horizontal unit vector in the direction we are placing catwalks direction = diff.copy() direction.z = 0 distance = direction.len() - 128 direction = direction.norm() if diff.z > 0: angle = INST_ANGLE[direction.as_tuple()] # We need to add stairs for stair_pos in range(0, int(diff.z), 128): # Move twice the vertical horizontally # plus 128 so we don't start in point A loc = point_a + (2 * stair_pos + 128) * direction # Do the vertical offset loc.z += stair_pos vmf.create_ent( classname='func_instance', origin=loc.join(' '), angles=angle, file=instances['stair'], ) # This is the location we start flat sections at point_a = loc + 128 * direction point_a.z += 128 elif diff.z < 0: # We need to add downward stairs # They point opposite to normal ones LOGGER.debug('down from {}', point_a) angle = INST_ANGLE[(-direction).as_tuple()] for stair_pos in range(0, -int(diff.z), 128): LOGGER.debug(stair_pos) # Move twice the vertical horizontally loc = point_a + (2 * stair_pos + 256) * direction # type: Vec # Do the vertical offset plus additional 128 units # to account for the moved instance loc.z -= (stair_pos + 128) vmf.create_ent( classname='func_instance', origin=loc.join(' '), angles=angle, file=instances['stair'], ) # Adjust point A to be at the end of the catwalks point_a = loc # Remove the space the stairs take up from the horiz distance distance -= abs(diff.z) * 2 # Now do straight sections LOGGER.debug('Stretching {} {}', distance, direction) angle = INST_ANGLE[direction.as_tuple()] loc = point_a + (direction * 128) # Figure out the most efficent number of sections for segment_len in utils.fit( distance, [512, 256, 128] ): vmf.create_ent( classname='func_instance', origin=loc.join(' '), angles=angle, file=instances['straight_' + str(segment_len)], ) loc += (segment_len * direction)
sort_within_batch=True, repeat=False) validdl = data.BucketIterator(dataset=(validds), batch_size=6, sort_key=lambda x: len(x.text), device=device, sort_within_batch=True, repeat=False) train_batch_it = BatchGenerator(traindl, 'text', 'Category') valid_batch_it = BatchGenerator(validdl, 'text', 'Category') vocab_size = len(txt_field.vocab) model = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out, trainds.fields['text'].vocab.vectors, dropout=dropout).to(device) opt = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), 1e-3) fit(model=model, train_dl=train_batch_it, val_dl=valid_batch_it, loss_fn=F.nll_loss, opt=opt, epochs=train_epochs)
def make_straight( vmf: VMF, origin: Vec, normal: Vec, dist: int, config: Config, is_start=False, ) -> None: """Make a straight line of instances from one point to another.""" angles = round(normal, 6).to_angle() orient = Matrix.from_angle(angles) # The starting brush needs to stick out a bit further, to cover the # point_push entity. start_off = -96 if is_start else -64 p1, p2 = Vec.bbox( origin + Vec(start_off, -config.trig_radius, -config.trig_radius) @ orient, origin + Vec(dist - 64, config.trig_radius, config.trig_radius) @ orient, ) solid = vmf.make_prism(p1, p2, mat='tools/toolstrigger').solid motion_trigger(vmf, solid.copy()) push_trigger(vmf, origin, normal, [solid]) off = 0 for seg_dist in utils.fit(dist, config.inst_straight_sizes): vmf.create_ent( classname='func_instance', origin=origin + off * orient.forward(), angles=angles, file=config.inst_straight[seg_dist], ) off += seg_dist # Supports. if config.inst_support: for off in range(0, int(dist), 128): position = origin + off * normal placed_support = False for supp_dir in [ orient.up(), orient.left(), -orient.left(), -orient.up() ]: try: tile = tiling.TILES[ (position - 128 * supp_dir).as_tuple(), supp_dir.norm().as_tuple() ] except KeyError: continue # Check all 4 center tiles are present. if all(tile[u, v].is_tile for u in (1, 2) for v in (1, 2)): vmf.create_ent( classname='func_instance', origin=position, angles=Matrix.from_basis(x=normal, z=supp_dir).to_angle(), file=config.inst_support, ) placed_support = True if placed_support and config.inst_support_ring: vmf.create_ent( classname='func_instance', origin=position, angles=angles, file=config.inst_support_ring, )
model_config_path = os.path.join(args.model_dir, 'model_config.json') if args.model_config is not None: model_config_path = args.model_config model_config = utils.load_json(model_config_path) game_step = model_config['game_step'] the_model = model.MinimalModel(**model_config) eval_config = utils.load_json(args.eval_config) if args.mode == 'train': train_config_path = os.path.join(args.model_dir, 'train_config.json') if args.train_config is not None: train_config_path = args.train_config train_config = utils.load_json(train_config_path) train_data = data.RandomDataset(game_step, **train_config['data']) valid_datasets = [ ('random_data', data.RandomDataset(game_step, **eval_config)) ] if game_step == 1: valid_datasets.append( ('minimal_data', data.MinimalDataset()) ) utils.fit(the_model, train_data, valid_datasets, **train_config['schedule']) elif args.mode == 'test': test_data = data.RandomDataset(game_step, **eval_config) accuracy = utils.eval(the_model, test_data) print(accuracy) else: assert False, 'invalid mode'
def main(): parser = argparse.ArgumentParser(description='relaxation terms regression') # parser.add_argument('-p', '--process', type=str, # choices=["shear", "bulk", "conductivity", "thermal_diffusion", "mass_diffusion"], # default="shear,bulk,conductivity,thermal_diffusion,mass_diffusion", # help='Comma-separated names of transport properties whose regression is performed') parser.add_argument('-a', '--algorithm', type=str, choices=[ 'DT', 'RF', 'ET', 'GP', 'KN', 'SVM', 'KR', 'GB', 'HGB', 'MLP' ], default='DT', help='regression algorithm') args = parser.parse_args() # process = args.process.split(',') # print("Process: ", colored(process[0], 'green')) algorithm = args.algorithm.split(',') print("Algorithm: ", colored(algorithm[0], 'blue')) src_dir = "." print("SRC: ", colored(src_dir, 'yellow')) output_dir = src_dir + "/.." print("OUTPUT: ", colored(output_dir, 'red')) n_jobs = 2 # Import database dataset = np.loadtxt("../data/transposed_reshaped_data.txt") # with open('../data/TCs_air5.txt') as f: # lines = (line for line in f if not line.startswith('#')) # dataset = np.loadtxt(lines, skiprows=1) print(dataset.shape) # if (process[0] == "shear"): # x = dataset[:,0:7] # T, P, x_N2, x_O2, x_NO, x_N, x_O # y = dataset[:,7:8] # shear viscosity # elif (process[0] == "bulk"): # x = dataset[:,0:7] # T, P, x_N2, x_O2, x_NO, x_N, x_O # y = dataset[:,8:9] # bulk viscosity # elif (process[0] == "conductivity"): # x = dataset[:,0:7] # T, P, x_N2, x_O2, x_NO, x_N, x_O # y = dataset[:,9:10]# thermal conductivity # elif (process[0] == "thermal_diffusion"): # x = dataset[:,0:7] # T, P, x_N2, x_O2, x_NO, x_N, x_O # y = dataset[:,10:] # thermal diffusion, D_Ti # elif (process[0] == "mass_diffusion"): # x = dataset[:,0:7] # T, P, x_N2, x_O2, x_NO, x_N, x_O # y = dataset[:,:] # mass diffusion TODO x = dataset[:, 0:50] # ni_n[47], na_n[1], V, T y = dataset[:, 50:] # RD_mol[47], RD_at[1] print(x.shape) print(y.shape) print("### Phase 1: PRE_PROCESSING ###") ######################################## # 1.0) create directory tree model, scaler, figure = utils.mk_tree(algorithm[0], output_dir) # 1.1) train/test split dataset x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.75, test_size=0.25, random_state=69) # 1.2) scale data and save scalers sc_x = StandardScaler() sc_y = StandardScaler() sc_x.fit(x_train) x_train = sc_x.transform(x_train) x_test = sc_x.transform(x_test) sc_y.fit(y_train) y_train = sc_y.transform(y_train) y_test = sc_y.transform(y_test) print('Training Features Shape:', x_train.shape) print('Training Labels Shape:', y_train.shape) print('Testing Features Shape:', x_test.shape) print('Testing Labels Shape:', y_test.shape) dump(sc_x, open(scaler + "/scaler_x.pkl", 'wb')) dump(sc_y, open(scaler + "/scaler_y.pkl", 'wb')) print("### Phase 2: PROCESSING ###") #################################### # 2.0) estimator selection if (algorithm[0] == 'DT'): est, hyper_params = estimators.est_DT() elif (algorithm[0] == 'ET'): est, hyper_params = estimators.est_ET() elif (algorithm[0] == 'SVM'): est, hyper_params = estimators.est_SVM() elif (algorithm[0] == 'KR'): est, hyper_params = estimators.est_KR() elif (algorithm[0] == 'KN'): est, hyper_params = estimators.est_KN() elif (algorithm[0] == 'MLP'): est, hyper_params = estimators.est_MLP() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() elif (algorithm[0] == 'HGB'): est, hyper_params = estimators.est_HGB() elif (algorithm[0] == 'RF'): est, hyper_params = estimators.est_RF() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() else: print("Algorithm not implemented ...") # 2.1) search for best hyper-parameters combination # Exhaustive search over specified parameter values for the estimator # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html gs = GridSearchCV(est, cv=10, param_grid=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # Randomized search on hyper parameters # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV # class sklearn.model_selection.RandomizedSearchCV(estimator, param_distributions, *, n_iter=10, scoring=None, n_jobs=None, refit=True, # cv=None, verbose=0, pre_dispatch='2*n_jobs', random_state=None, error_score=nan, # return_train_score=False) #gs = RandomizedSearchCV(est, cv=10, n_iter=10, param_distributions=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', # refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # 2.2) training utils.fit(x_train, y_train, gs) # 2.3) prediction y_regr = utils.predict(x_test, gs) print("### Phase 3: POST-PROCESSING ###") ######################################### # 3.0) save best hyper-parameters results = pd.DataFrame(gs.cv_results_) # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html #compression_opts = dict(method='zip', archive_name='GridSearchCV_results.csv') #results.to_csv('GridSearchCV_results.zip', index=False, compression=compression_opts) results.to_csv(model + "/../" + "GridSearchCV_results.csv", index=False, sep='\t', encoding='utf-8') # results print screen print("Best: %f using %s" % (gs.best_score_, gs.best_params_)) means = gs.cv_results_['mean_test_score'] stds = gs.cv_results_['std_test_score'] params = gs.cv_results_['params'] for mean, stdev, param in zip(means, stds, params): print("%f (%f) with: %r" % (mean, stdev, param)) # 3.1) compute score metrics utils.scores(sc_x, sc_y, x_train, y_train, x_test, y_test, model, gs) # 3.2) back to original values (unscaling) x_test_dim = sc_x.inverse_transform(x_test) y_test_dim = sc_y.inverse_transform(y_test) y_regr_dim = sc_y.inverse_transform(y_regr) # 3.3) make plots utils.draw_plot(x_test_dim, y_test_dim, y_regr_dim, figure) # 3.4) save model to disk dump(gs, model + "/model.sav")
def main(): parser = argparse.ArgumentParser(description='Omega integrals') parser.add_argument('-p', '--process', type=str, choices=["omega11", "omega12", "omega13", "omega22", "omegas"], default="omegas", help='Comma-separated names of omega integrals whose regression is performed') parser.add_argument('-a', '--algorithm', type=str, choices=['DT', 'RF', 'ET', 'GP', 'KN', 'SVM', 'KR', 'GB', 'HGB', 'MLP'], default='DT', help='transport algorithm') parser.add_argument('-l', '--load_model', type=str2bool, nargs='?', choices=[False, True], default=False, const=True, help='Load saved model') args = parser.parse_args() process = args.process.split(',') print("Process: ", colored(process[0], 'green')) algorithm = args.algorithm.split(',') print("Algorithm: ", colored(algorithm[0],'blue')) load_model = args.load_model print("Load: ", colored(load_model,'magenta')) src_dir = "." print("SRC: ", colored(src_dir,'yellow')) output_dir = src_dir+"/.." print("OUTPUT: ", colored(output_dir,'red')) n_jobs = 2 # Import database with open('../data/omega_integrals_encoded.txt') as f: lines = (line for line in f if not line.startswith('#')) dataset = np.loadtxt(lines, skiprows=1) print(dataset.shape) x = dataset[:,0:3] # c, d, T y = dataset[:,3:] # Ω(1,1), Ω(1,2), Ω(1,3), Ω(2,2) print(x.shape) print(y.shape) print("### Phase 1: PRE_PROCESSING ###") ######################################## # 1.0) create directory tree model, scaler, figure = utils.mk_tree(process[0], algorithm[0], output_dir) # 1.1) train/test split dataset x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.75, test_size=0.25, random_state=69) # 1.2) scale data and save scalers sc_x = StandardScaler() sc_y = StandardScaler() sc_x.fit(x_train) x_train = sc_x.transform(x_train) x_test = sc_x.transform(x_test) sc_y.fit(y_train) y_train = sc_y.transform(y_train) y_test = sc_y.transform(y_test) print('Training Features Shape:', x_train.shape) print('Training Labels Shape:', y_train.shape) print('Testing Features Shape:', x_test.shape) print('Testing Labels Shape:', y_test.shape) dump(sc_x, open(scaler+"/scaler_x_"+process[0]+'.pkl', 'wb')) dump(sc_y, open(scaler+"/scaler_y_"+process[0]+'.pkl', 'wb')) print("### Phase 2: PROCESSING ###") #################################### # 2.0) estimator selection if (algorithm[0] == 'DT'): est, hyper_params = estimators.est_DT() elif (algorithm[0] == 'ET'): est, hyper_params = estimators.est_ET() elif (algorithm[0] == 'SVM'): est, hyper_params = estimators.est_SVM() elif (algorithm[0] == 'KR'): est, hyper_params = estimators.est_KR() elif (algorithm[0] == 'KN'): est, hyper_params = estimators.est_KN() elif (algorithm[0] == 'MLP'): est, hyper_params = estimators.est_MLP() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() elif (algorithm[0] == 'HGB'): est, hyper_params = estimators.est_HGB() elif (algorithm[0] == 'RF'): est, hyper_params = estimators.est_RF() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() else: print("Algorithm not implemented ...") # 2.1) search for best hyper-parameters combination # Exhaustive search over specified parameter values for the estimator # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html gs = GridSearchCV(est, cv=3, param_grid=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # Randomized search on hyper parameters # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV # class sklearn.model_selection.RandomizedSearchCV(estimator, param_distributions, *, n_iter=10, scoring=None, n_jobs=None, refit=True, # cv=None, verbose=0, pre_dispatch='2*n_jobs', random_state=None, error_score=nan, # return_train_score=False) #gs = RandomizedSearchCV(est, cv=10, n_iter=10, param_distributions=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', # refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # 2.2) training utils.fit(x_train, y_train, gs) # 2.3) prediction y_regr = utils.predict(x_test, gs) print("### Phase 3: POST-PROCESSING ###") ######################################### # 3.0) save best hyper-parameters results = pd.DataFrame(gs.cv_results_) # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html #compression_opts = dict(method='zip', archive_name='GridSearchCV_results.csv') #results.to_csv('GridSearchCV_results.zip', index=False, compression=compression_opts) results.to_csv(model+"/../"+"GridSearchCV_results.csv", index=False, sep='\t', encoding='utf-8') # results print screen print("Best: %f using %s" % (gs.best_score_, gs.best_params_)) means = gs.cv_results_['mean_test_score'] stds = gs.cv_results_['std_test_score'] params = gs.cv_results_['params'] for mean, stdev, param in zip(means, stds, params): print("%f (%f) with: %r" % (mean, stdev, param)) # 3.1) compute score metrics utils.scores(sc_x, sc_y, x_train, y_train, x_test, y_test, model, gs) # 3.2) back to original values (unscaling) x_test_dim = sc_x.inverse_transform(x_test) y_test_dim = sc_y.inverse_transform(y_test) y_regr_dim = sc_y.inverse_transform(y_regr) # 3.3) make plots utils.draw_plot(x_test_dim, y_test_dim, y_regr_dim, figure, process[0], algorithm[0]) # 3.4) save model to disk dump(gs, model+"/model_"+process[0]+".sav")
print('Trained model loss function loaded...') print(f"Previously trained for {epochs} number of epochs...") # train for more epochs epochs = new_epochs print(f"Train for {epochs} more epochs...") # train data loader train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=False) train_loss, train_accuracy = [], [] val_loss, val_accuracy = [], [] for epoch in range(epochs): print(f"Epoch {epoch+1} of {epochs}") train_epoch_loss, train_epoch_accuracy = fit(model, train_loader, optimizer, criterion, train_data) val_epoch_loss, val_epoch_accuracy = validate(model, val_loader, optimizer, criterion, val_data) train_loss.append(train_epoch_loss) train_accuracy.append(train_epoch_accuracy) val_loss.append(val_epoch_loss) val_accuracy.append(val_epoch_accuracy) print( f"Train Loss: {train_epoch_loss:.4f}, Train Acc: {train_epoch_accuracy:.2f}" ) print(f'Val Loss: {val_epoch_loss:.4f}, Val Acc: {val_epoch_accuracy:.2f}') # accuracy plots plt.figure(figsize=(10, 7)) plt.plot(train_accuracy, color='green', label='train accuracy')
from utils import import_data, data_wrangling, features, fit, print_stats, create_submission_file if __name__ == "__main__": train_data, test_data = import_data() train_data, test_data = data_wrangling(train_data, test_data) X, y, X_test = features(train_data, test_data) model = fit(X, y) print_stats(X, y, model) #without crossvalidation create_submission_file(model, X_test, test_data)
('plot_score', PLOT_SCORE), ('image_score', IMAGE_SCORE), ('music_score', MUSIC_SCORE), ('actors_score', ACTORS_SCORE), ('name0', None)], skip_header=True) train, val = get_dataset(union_toloka_result_proc_path).split() golden_train = get_dataset(union_golden_proc_path2) TEXT.build_vocab(train, max_size=30000) model_path = "./models/model" rnn_model = MultiModel(model=BiLSTMClassifier(300, len(TEXT.vocab.stoi), 256, 2).to(device)) # rnn_model.load_state_dict(torch.load(model_path)) batch_size = 32 train_iter, val_iter = data.BucketIterator.splits( (train, val), sort_key=lambda x: len(x.text), batch_sizes=(batch_size, batch_size), device=device) golden_iter = data.BucketIterator(golden_train, sort_key=lambda x: len(x.text), batch_size=batch_size, device=device) criterion_cls = nn.BCEWithLogitsLoss().to(device) criterion_scores = nn.MSELoss(reduction='none').to(device) criterion_scores_l1 = nn.L1Loss(reduction='none').to(device) rnn_model = MultiModel(model=BiLSTMClassifier(300, len(TEXT.vocab.stoi), 256, 2).to(device)) optimizer = optim.Adam([param for param in rnn_model.model.parameters() if param.requires_grad]) fit(rnn_model, criterion_cls, criterion_scores, optimizer, train_iter, epochs_count=30, val_data=val_iter) torch.save(rnn_model.model.state_dict(), model_path) do_eval_epoch(rnn_model, None, criterion_scores_l1, val_iter)
def train(dataset_name, model_name, metric_name, path_history, path_model, path_opt, path_best_model, reset=False): # SET SEED np.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed_all(1) # Train datasets transformer = ut.ComposeJoint( [ut.RandomHorizontalFlipJoint(), [transforms.ToTensor(), None], [transforms.Normalize(*ut.mean_std), None], [None, ut.ToLong() ] ]) train_set = dataset_dict[dataset_name](split="train", transform_function=transformer) trainloader = torch.utils.data.DataLoader(train_set, batch_size=1, num_workers=0, drop_last=False, sampler=ut.RandomSampler(train_set)) # Val datasets transformer = ut.ComposeJoint( [ [transforms.ToTensor(), None], [transforms.Normalize(*ut.mean_std), None], [None, ut.ToLong() ] ]) val_set = dataset_dict[dataset_name](split="val", transform_function=transformer) test_set = dataset_dict[dataset_name](split="test", transform_function=transformer) # Model model = model_dict[model_name](train_set.n_classes).cuda() opt = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-5, weight_decay=0.0005) # Train if os.path.exists(path_history) and not reset: history = ut.load_json(path_history) model.load_state_dict(torch.load(path_model)) opt.load_state_dict(torch.load(path_opt)) s_epoch = history["train"][-1]["epoch"] print("Resuming epoch...{}".format(s_epoch)) else: history = {"train":[], "val":[], "test":[], "model_name":model_name, "dataset_name":dataset_name, "path_model":path_model, "path_opt":path_opt, "path_best_model":path_best_model, "best_val_epoch":-1, "best_val_mae":np.inf} s_epoch = 0 print("Starting from scratch...") for epoch in range(s_epoch + 1, 1000): train_dict = ut.fit(model, trainloader, opt, loss_function=losses.lc_loss, epoch=epoch) # Update history history["trained_images"] = list(model.trained_images) history["train"] += [train_dict] # Save model, opt and history torch.save(model.state_dict(), path_model) torch.save(opt.state_dict(), path_opt) ut.save_json(path_history, history) # %%%%%%%%%%% 2. VALIDATION PHASE %%%%%%%%%%%%" with torch.no_grad(): val_dict = ut.val(model=model, dataset=val_set, epoch=epoch, metric_name=metric_name) # Update history history["val"] += [val_dict] # Lower is better if val_dict[metric_name] <= history["best_val_mae"]: history["best_val_epoch"] = epoch history["best_val_mae"] = val_dict[metric_name] torch.save(model.state_dict(), path_best_model) # Test Model if not (dataset_name == "penguins" and epoch < 50): testDict = ut.val(model=model, dataset=test_set, epoch=epoch, metric_name=metric_name) history["test"] += [testDict] ut.save_json(path_history, history)
def main(): print("Starting DFC2021 baseline training script at %s" % (str(datetime.datetime.now()))) #------------------- # Setup #------------------- assert os.path.exists(args.input_fn) if os.path.isfile(args.output_dir): print("A file was passed as `--output_dir`, please pass a directory!") return if os.path.exists(args.output_dir) and len(os.listdir(args.output_dir)): if args.overwrite: print("WARNING! The output directory, %s, already exists, we might overwrite data in it!" % (args.output_dir)) else: print("The output directory, %s, already exists and isn't empty. We don't want to overwrite and existing results, exiting..." % (args.output_dir)) return else: print("The output directory doesn't exist or is empty.") os.makedirs(args.output_dir, exist_ok=True) if torch.cuda.is_available(): device = torch.device("cuda:%d" % args.gpu) else: print("WARNING! Torch is reporting that CUDA isn't available, exiting...") return np.random.seed(args.seed) torch.manual_seed(args.seed) #------------------- # Load input data #------------------- input_dataframe = pd.read_csv(args.input_fn) image_fns = input_dataframe["image_fn"].values label_fns = input_dataframe["label_fn"].values groups = input_dataframe["group"].values dataset = StreamingGeospatialDataset( imagery_fns=image_fns, label_fns=label_fns, groups=groups, chip_size=CHIP_SIZE, num_chips_per_tile=NUM_CHIPS_PER_TILE, windowed_sampling=False, verbose=False, image_transform=image_transforms, label_transform=label_transforms, nodata_check=nodata_check ) dataloader = torch.utils.data.DataLoader( dataset, batch_size=args.batch_size, num_workers=NUM_WORKERS, pin_memory=True, ) num_training_batches_per_epoch = int(len(image_fns) * NUM_CHIPS_PER_TILE / args.batch_size) print("We will be training with %d batches per epoch" % (num_training_batches_per_epoch)) #------------------- # Setup training #------------------- if args.model == "unet": model = models.get_unet() elif args.model == "fcn": model = models.get_fcn() else: raise ValueError("Invalid model") model = model.to(device) optimizer = optim.AdamW(model.parameters(), lr=0.001, amsgrad=True) criterion = nn.CrossEntropyLoss() scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, "min") print("Model has %d parameters" % (utils.count_parameters(model))) #------------------- # Model training #------------------- training_task_losses = [] num_times_lr_dropped = 0 model_checkpoints = [] temp_model_fn = os.path.join(args.output_dir, "most_recent_model.pt") for epoch in range(args.num_epochs): lr = utils.get_lr(optimizer) training_losses = utils.fit( model, device, dataloader, num_training_batches_per_epoch, optimizer, criterion, epoch, ) scheduler.step(training_losses[0]) model_checkpoints.append(copy.deepcopy(model.state_dict())) if args.save_most_recent: torch.save(model.state_dict(), temp_model_fn) if utils.get_lr(optimizer) < lr: num_times_lr_dropped += 1 print("") print("Learning rate dropped") print("") training_task_losses.append(training_losses[0]) if num_times_lr_dropped == 4: break #------------------- # Save everything #------------------- save_obj = { 'args': args, 'training_task_losses': training_task_losses, "checkpoints": model_checkpoints } save_obj_fn = "results.pt" with open(os.path.join(args.output_dir, save_obj_fn), 'wb') as f: torch.save(save_obj, f)
def bayesianOptimization(func_objective, func_cost, func_acq, func_policy, bounds, depth_cost, givenCost, initial_n=1, initpoint=None, n_sample=10, decay_rate=1, length_scale=0.3, ARD_Flag=False): """ depth_h: num of nest N: num of iter """ assert depth_cost <= givenCost, "Error: depth_cost > givenCost" #assert initial_n <= N, "Error: initial_n > N" _length_scale = length_scale * (bounds[0][1] - bounds[0][0]) # load/generate init points if initial_n > 0: queries = generate_init(bounds, initial_n) else: queries = initpoint initial_cost = np.sum(func_cost(queries)) remainCost = givenCost - initial_cost values = func_objective(queries) count = 0 while remainCost > 1: # assume that min(func_cost) is 1 count = count + 1 print(count) kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) _remainC = min({depth_cost, remainCost}) GP_model = fit(queries, values, kernel) # define acquisition function if func_acq == ei: facq = lambda x: -1 * ei(x, bounds, GP_model) elif func_acq == ei_per_cost: facq = lambda x: -1 * ei_per_cost(x, func_cost, bounds, GP_model) else: facq = lambda x: -1 * func_acq(x, bounds=bounds, func_policy=func_policy, func_cost=func_cost, depth_c=_remainC, _queries=queries, _values=values, n_sample=n_sample, decay_rate=decay_rate, ARD_Flag=ARD_Flag, length_scale=_length_scale) # compute threthold(lastpoint) muquery = policy_mu(GP_model, bounds) threthold = _remainC - func_cost(muquery) # select next query given that threthold X = minimize_with_threthold2d(facq, func_cost, bounds, threthold) Y = func_objective(X) # subtract cost from remain cost remainCost = remainCost - func_cost(X) queries = np.concatenate([queries, X]) values = np.concatenate([values, Y]) return queries, values
def main(): parser = argparse.ArgumentParser(description='reaction rates regression') parser.add_argument( '-p', '--process', type=str, choices=['DR', 'VT', 'VV', 'VV2', 'ZR'], default='DR,VT,VV,VV2,ZR', help='Comma-separated names of properties whose regression is performed' ) parser.add_argument('-a', '--algorithm', type=str, choices=[ 'DT', 'RF', 'ET', 'GP', 'KN', 'SVM', 'KR', 'GB', 'HGB', 'MLP' ], default='DT', help='regression algorithm') args = parser.parse_args() process = args.process.split(',') directory = process[0] + '/data/processes' path = directory + "/*.csv" print("Process: ", colored(process[0], 'green')) algorithm = args.algorithm.split(',') print("Algorithm: ", colored(algorithm[0], 'blue')) parent_dir = "." print("PWD: ", colored(parent_dir, 'yellow')) n_jobs = 2 for f in glob.glob(path): #print("{bcolors.OKGREEN}f{bcolors.ENDC}") print(colored(f, 'red')) dataset_k = pd.read_csv(f, delimiter=",").to_numpy() dataset_T = pd.read_csv(parent_dir + "/" + process[0] + "/data/Temperatures.csv").to_numpy() x = dataset_T.reshape(-1, 1) y = dataset_k print("### Phase 1: PRE_PROCESSING ###") ######################################## ''' https://stackoverflow.com/questions/50565937/how-to-normalize-the-train-and-test-data-using-minmaxscaler-sklearn https://towardsdatascience.com/6-amateur-mistakes-ive-made-working-with-train-test-splits-916fabb421bb https://www.analyticsvidhya.com/blog/2020/04/feature-scaling-machine-learning-normalization-standardization/ https://towardsdatascience.com/scale-standardize-or-normalize-with-scikit-learn-6ccc7d176a02 You should fit the MinMaxScaler using the training data and then apply the scaler on the testing data before the prediction. In summary: Step 1: fit the scaler on the TRAINING data Step 2: use the scaler to transform the TRAINING data Step 3: use the transformed training data to fit the predictive model Step 4: use the scaler to transform the TEST data Step 5: predict using the trained model (step 3) and the transformed TEST data (step 4). data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0) scaler = MinMaxScaler() X_train_scaled = scaler.fit_transform(X_train) model = SVC() model.fit(X_train_scaled, y_train) X_test_scaled = scaler.transform(X_test) y_pred = model.predict(X_test_scaled) ''' data, dir, proc, model, scaler, figure, outfile = utils.mk_tree( f, parent_dir, process[0], algorithm[0]) # Train/test split dataset x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.75, test_size=0.25, random_state=69) # Define scalers: they can be modified to investigate the effect of scalers ############################################################################## input_scaler = None #MinMaxScaler(feature_range=(-1,1)) output_scaler = None #StandardScaler() ############################################################################## # Scale None and/or inputs and/or outputs x_train, x_test, y_train, y_test = utils.scale_dataset( x_train, x_test, y_train, y_test, input_scaler, output_scaler) print('Training Features Shape:', x_train.shape) print('Training Labels Shape:', y_train.shape) print('Testing Features Shape:', x_test.shape) print('Testing Labels Shape:', y_test.shape) # Save scalers (they may be useful) dump(input_scaler, open(scaler + "/scaler_x_MO_" + data + '.pkl', 'wb')) dump(output_scaler, open(scaler + "/scaler_y_MO_" + data + '.pkl', 'wb')) if (algorithm[0] == 'DT'): est, hyper_params = estimators.est_DT() elif (algorithm[0] == 'ET'): est, hyper_params = estimators.est_ET() elif (algorithm[0] == 'SVM'): est, hyper_params = estimators.est_SVM() elif (algorithm[0] == 'KR'): est, hyper_params = estimators.est_KR() elif (algorithm[0] == 'KN'): est, hyper_params = estimators.est_KN() elif (algorithm[0] == 'MLP'): est, hyper_params = estimators.est_MLP() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() elif (algorithm[0] == 'HGB'): est, hyper_params = estimators.est_HGB() elif (algorithm[0] == 'RF'): est, hyper_params = estimators.est_RF() elif (algorithm[0] == 'GB'): est, hyper_params = estimators.est_GB() else: print("Algorithm not implemented ...") # https://github.com/ray-project/tune-sklearn # https://docs.ray.io/en/latest/tune/api_docs/sklearn.html#tune-sklearn-docs # class ray.tune.sklearn.TuneGridSearchCV(estimator, param_grid, early_stopping=None, scoring=None, # n_jobs=None, cv=5, refit=True, verbose=0, error_score='raise', return_train_score=False, # local_dir='~/ray_results', max_iters=1, use_gpu=False, loggers=None, pipeline_auto_early_stop=True, # stopper=None, time_budget_s=None, sk_n_jobs=None) #scheduler = MedianStoppingRule(grace_period=10.0) #gs = TuneGridSearchCV(est, cv=10, param_grid=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', # refit=True, error_score=np.nan, return_train_score=True) #tune_search = TuneSearchCV(clf, parameter_grid, search_optimization="hyperopt", n_trials=3, early_stopping=scheduler, max_iters=10) #tune_search.fit(x_train, y_train) # Exhaustive search over specified parameter values for the estimator # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html gs = GridSearchCV(est, cv=5, param_grid=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # Randomized search on hyper parameters # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV # class sklearn.model_selection.RandomizedSearchCV(estimator, param_distributions, *, n_iter=10, scoring=None, n_jobs=None, refit=True, # cv=None, verbose=0, pre_dispatch='2*n_jobs', random_state=None, error_score=nan, # return_train_score=False) #gs = RandomizedSearchCV(est, cv=10, n_iter=10, param_distributions=hyper_params, verbose=2, n_jobs=n_jobs, scoring='r2', # refit=True, pre_dispatch='n_jobs', error_score=np.nan, return_train_score=True) # Training utils.fit(x_train, y_train, gs, outfile) results = pd.DataFrame(gs.cv_results_) # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html #compression_opts = dict(method='zip', archive_name='GridSearchCV_results.csv') #results.to_csv('GridSearchCV_results.zip', index=False, compression=compression_opts) results.to_csv(model + "/../" + "GridSearchCV_results.csv", index=False, sep='\t', encoding='utf-8') #plt.figure(figsize=(12, 4)) #for score in ['mean_test_recall', 'mean_test_precision', 'mean_test_min_both']: # plt.plot([_[1] for _ in results['param_class_weight']], results[score], label=score) #plt.legend(); #plt.figure(figsize=(12, 4)) #for score in ['mean_train_recall', 'mean_train_precision', 'mean_test_min_both']: # plt.scatter(x=[_[1] for _ in results['param_class_weight']], y=results[score.replace('test', 'train')], label=score) #plt.legend(); # summarize results print("Best: %f using %s" % (gs.best_score_, gs.best_params_)) means = gs.cv_results_['mean_test_score'] stds = gs.cv_results_['std_test_score'] params = gs.cv_results_['params'] for mean, stdev, param in zip(means, stds, params): print("%f (%f) with: %r" % (mean, stdev, param)) # Perform prediction y_regr = utils.predict(x_test, gs, outfile) # Compute the scores utils.scores(input_scaler, output_scaler, x_train, y_train, x_test, y_test, model, gs, outfile) # Transform back x_train, x_test, y_train, y_test, y_regr = utils.scale_back_dataset( x_train, x_test, y_train, y_test, y_regr, input_scaler, output_scaler) # Make figures utils.draw_plot(x_test, y_test, y_regr, figure, data) # save the model to disk dump(gs, model + "/model_MO_" + data + '.sav')
def main(): # print("Starting DFC2021 baseline training script at %s" % (str(datetime.datetime.now()))) #------------------- # Setup #------------------- assert os.path.exists(args.train_fn) assert os.path.exists(args.valid_fn) now_time = datetime.datetime.now() time_str = datetime.datetime.strftime(now_time, '%m-%d_%H-%M-%S') # output path # output_dir = Path(args.output_dir).parent / time_str / Path(args.output_dir).stem output_dir = Path(args.output_dir) output_dir.mkdir(exist_ok=True, parents=True) logger = utils.init_logger(output_dir / 'info.log') # if os.path.isfile(args.output_dir): # print("A file was passed as `--output_dir`, please pass a directory!") # return # # if os.path.exists(args.output_dir) and len(os.listdir(args.output_dir)): # if args.overwrite: # print("WARNING! The output directory, %s, already exists, we might overwrite data in it!" % (args.output_dir)) # else: # print("The output directory, %s, already exists and isn't empty. We don't want to overwrite and existing results, exiting..." % (args.output_dir)) # return # else: # print("The output directory doesn't exist or is empty.") # os.makedirs(args.output_dir, exist_ok=True) if args.gpu is not None: os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu n_gpu = torch.cuda.device_count() device = torch.device('cuda:0' if n_gpu > 0 else 'cpu') device_ids = list(range(n_gpu)) np.random.seed(args.seed) torch.manual_seed(args.seed) #------------------- # Load input data #------------------- train_dataframe = pd.read_csv(args.train_fn) train_image_fns = train_dataframe["image_fn"].values train_label_fns = train_dataframe["label_fn"].values train_groups = train_dataframe["group"].values train_dataset = StreamingGeospatialDataset( imagery_fns=train_image_fns, label_fns=train_label_fns, groups=train_groups, chip_size=CHIP_SIZE, num_chips_per_tile=NUM_CHIPS_PER_TILE, transform=transform, nodata_check=nodata_check ) valid_dataframe = pd.read_csv(args.valid_fn) valid_image_fns = valid_dataframe["image_fn"].values valid_label_fns = valid_dataframe["label_fn"].values valid_groups = valid_dataframe["group"].values valid_dataset = StreamingValidationDataset( imagery_fns=valid_image_fns, label_fns=valid_label_fns, groups=valid_groups, chip_size=CHIP_SIZE, stride=CHIP_SIZE, transform=transform, nodata_check=nodata_check ) train_dataloader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, num_workers=NUM_WORKERS, pin_memory=True, ) valid_dataloader = torch.utils.data.DataLoader( valid_dataset, batch_size=args.batch_size, num_workers=NUM_WORKERS, pin_memory=True, ) num_training_images_per_epoch = int(len(train_image_fns) * NUM_CHIPS_PER_TILE) # print("We will be training with %d batches per epoch" % (num_training_batches_per_epoch)) #------------------- # Setup training #------------------- # if args.model == "unet": # model = models.get_unet() # elif args.model == "fcn": # model = models.get_fcn() # else: # raise ValueError("Invalid model") model = models.isCNN(args.backbone) weights_init(model, seed=args.seed) model = model.to(device) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids) trainable_params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = optim.AdamW(trainable_params, lr=INIT_LR, amsgrad=True, weight_decay=5e-4) lr_criterion = nn.CrossEntropyLoss(ignore_index=0) # todo hr_criterion = hr_loss # criterion = balanced_ce_loss scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, "min", factor=0.5, patience=3, min_lr=0.0000001) # factor=0.5, patience=3, min_lr=0.0000001 logger.info("Trainable parameters: {}".format(utils.count_parameters(model))) #------------------- # Model training #------------------- train_loss_total_epochs, valid_loss_total_epochs, epoch_lr = [], [], [] best_loss = 1e50 num_times_lr_dropped = 0 # model_checkpoints = [] # temp_model_fn = os.path.join(output_dir, "most_recent_model.pt") for epoch in range(args.num_epochs): lr = utils.get_lr(optimizer) train_loss_epoch, valid_loss_epoch = utils.fit( model, device, train_dataloader, valid_dataloader, num_training_images_per_epoch, optimizer, lr_criterion, hr_criterion, epoch, logger) scheduler.step(valid_loss_epoch) if epoch % config.SAVE_PERIOD == 0 and epoch != 0: temp_model_fn = output_dir / 'checkpoint-epoch{}.pth'.format(epoch+1) torch.save(model.state_dict(), temp_model_fn) if valid_loss_epoch < best_loss: logger.info("Saving model_best.pth...") temp_model_fn = output_dir / 'model_best.pth' torch.save(model.state_dict(), temp_model_fn) best_loss = valid_loss_epoch if utils.get_lr(optimizer) < lr: num_times_lr_dropped += 1 print("") print("Learning rate dropped") print("") train_loss_total_epochs.append(train_loss_epoch) valid_loss_total_epochs.append(valid_loss_epoch) epoch_lr.append(lr)
def bayesianOptimization(func_objective, func_acq, func_policy, bounds, depth_h, N, initial_n=1, initpoint=None, N_q=3, n_sample=10, decay_rate=1, ARD_Flag=False, length_scale=None): """ depth_h: num of nest N: num of iter """ assert depth_h <= N, "Error: depth_h > N" assert initial_n <= N, "Error: initial_n > N" _N = N - initial_n if initial_n > 0: queries = generate_init(bounds, initial_n) else: queries = initpoint values = func_objective(queries) length_scale = (bounds[0][1] - bounds[0][0]) / 10. for i in range(_N): print(i) kernel = GPy.kern.RBF(len(bounds), ARD=ARD_Flag, lengthscale=length_scale) #gp_model = fit(queries, values) _h = min({depth_h, _N - i}) #_count_depth = 0 #_gp_list = {} #_queries_list = {} #_values_list = {} #_trajectory = [] #_U = 0 #_idlist = [] if func_acq == ei: GP_model = fit(queries, values, kernel) facq = lambda x: -1 * ei(x, bounds, GP_model) else: facq = lambda x: -1 * func_acq(x, bounds=bounds, func_policy=func_policy, depth_h=_h, _queries=queries, _values=values, N_q=N_q, n_sample=10, decay_rate=decay_rate, ARD_Flag=ARD_Flag, length_scale=length_scale) X = minimize(facq, bounds) Y = func_objective(X) queries = np.concatenate([queries, X]) values = np.concatenate([values, Y]) return queries, values