def __lt__(self, other): my_attrs = utils.flatten_dict(self.attrs) other_attrs = utils.flatten_dict(other.attrs) if my_attrs != other_attrs: return my_attrs < other_attrs else: return self.actions < other.actions
def __lt__(self, other): my_attrs = utils.flatten_dict(self.attrs) other_attrs = utils.flatten_dict(other.attrs) if my_attrs != other_attrs: return my_attrs < other_attrs else: return self.actions < other.actions
def update_service(self, host, service, data): """ This function updates/inserts a service It used by arbiter in hook_late_configuration to put the configuration in the database Return * query_result: None * error: bool """ # Prepare mongo Filter mongo_filter = {"host": host, "service": service} # Flatten dict serv data = flatten_dict(data) # Save in mongo try: mongo_res = getattr(self.db_conn, self.db_name).services.update(mongo_filter, {"$set": data}, upsert=True) except Exception as exp: logger.error("[SnmpBooster] [code 1204] [%s, %s] " "%s" % (host, service, str(exp))) return (None, True) return (None, self.handle_error(mongo_res, mongo_filter))
def csv_fieldnames(self): if hasattr(self, "_csv_fieldnames"): return self._csv_fieldnames result_structure = { "metrics": { "metatrain": { "roc_auc": None, "precision": None, "recall": None, "f1": None }, "metaval": { "roc_auc": None, "precision": None, "recall": None, "f1": None }, "metatest": { "roc_auc": None, "precision": None, "recall": None, "f1": None } }, "deviation_after_best": None, "training_iteration": None } fieldnames = list(flatten_dict(result_structure)) self._csv_fieldnames = fieldnames return self._csv_fieldnames
def _serialize_v1(self, request_params): parameters = flatten_dict(request_params) key_values = [] for key in sorted(parameters.iterkeys()): value = parameters[key] quoted_value = get_quoted_value(value, quote=True) key_values.append(key + '=' + quoted_value) return '&'.join(key_values)
def unique_ids(components, flags): for x in components['section']: this_df = pd.DataFrame(utils.flatten_dict_detailed( utils.flatten_dict(x)).items(), columns=['tag', 'value']) flags.append(check_duplicates_ids(this_df)) return 0
def __init__(self, params, normal_weight_init=True): self.adv_bce_loss = nn.BCELoss() self.aux_nll_loss = nn.NLLLoss() self.discriminator = AuxDiscriminator( params["n_classes"], params["disc"]["n_feature"], params["disc"]["n_channel"], n_conv_block=params["n_conv_block"]).to(var.device) self.generator = self.init_generator(params["generator_type"], params) self.disc_optim = torch.optim.Adam( self.discriminator.parameters(), lr=params['disc']['lr'], betas=tuple(params['disc']['betas'].values())) """ self.disc_optim = torch.optim.SGD(self.discriminator.parameters(), lr=params['disc']['lr']) """ self.gen_optim = torch.optim.Adam(self.generator.parameters(), lr=params['gen']['lr'], betas=tuple( params['gen']['betas'].values())) self.z_dim = params['z_dim'] self.n_classes = params["n_classes"] self.label_smoothing = params["label_smoothing"] self.use_inception = params["use_inception"] if self.use_inception: self.inception = InceptionV3() self.writer = None self.h_params_added = False self.step = 0 self.epoch = 0 self.adv_fake_acc = list() self.adv_real_acc = list() self.aux_fake_acc = list() self.aux_real_acc = list() self.fake_aux_entropy = list() self.real_aux_entropy = list() if normal_weight_init: self.discriminator.apply(ut.weights_init) self.generator.apply(ut.weights_init) params['GAN_type'] = self.__class__.__name__ params['gen_optim_type'] = self.gen_optim.__class__.__name__ params['disc_optim_type'] = self.disc_optim.__class__.__name__ self.params = ut.flatten_dict(params)
def process_response(self, request, response): if not self.start: return response try: path = smart_unicode(request.path) proctime = round(time.time() - self.start, 3) get = flatten_dict(request.GET) post = flatten_dict(request.POST) ip = request.META.get('REMOTE_ADDR', u'') status_code = response.status_code self._logger.info( u'%u %s exec_time=%.3fs GET=%s POST=%s IP=%s RESP=%s' % (status_code, path, proctime, get, post, ip, '')) except Exception as e: self._logger(e) return response
def __init__(self, params, normal_weight_init=True): self.aux_nll_loss = nn.NLLLoss() self.critic = Critic(params["n_classes"], params["critic"]["n_feature"], params["critic"]["n_channel"], params["n_conv_block"]).to(var.device) self.generator = self.init_generator(params["generator_type"], params) self.critic_optim = torch.optim.Adam( self.critic.parameters(), lr=params['critic']['lr'], betas=tuple(params['critic']['betas'].values())) self.gen_optim = torch.optim.Adam(self.generator.parameters(), lr=params['gen']['lr'], betas=tuple( params['gen']['betas'].values())) self.z_dim = params['z_dim'] self.n_classes = params["n_classes"] self.gradient_penalty_factor = params['gradient_penalty_factor'] self.stability_noise_std = params['stability_noise_std'] self.use_inception = params["use_inception"] if self.use_inception: self.inception = InceptionV3() self.writer = None self.h_params_added = False self.step = 0 self.epoch = 0 self.aux_fake_acc = list() self.aux_real_acc = list() self.fake_aux_entropy = list() self.real_aux_entropy = list() if normal_weight_init: self.critic.apply(ut.weights_init) self.generator.apply(ut.weights_init) params['GAN_type'] = self.__class__.__name__ params['gen_optim_type'] = self.gen_optim.__class__.__name__ params['disc_optim_type'] = self.critic_optim.__class__.__name__ self.params = ut.flatten_dict(params)
def parse_available_button(list_action): """ Assert actions are possible and create all possible combinations of available actions """ all_actions = get_subkey(ALL_AVAILABLE_ACTIONS) reversed_dict = flatten_dict(ALL_AVAILABLE_ACTIONS) a_action = [] new_dict_act = defaultdict(list) for action in list_action: if action in all_actions: a_action.append(action) new_dict_act[reversed_dict[action]].append(action) else: logger.warning("{} is not an action available!".format(action)) # check if at least one action was correct assert a_action return a_action, new_dict_act
def update_service(self, host, service, data): """ This function updates/inserts a service It used by arbiter in hook_late_configuration to put the configuration in the database Return * query_result: None * error: bool """ # Prepare mongo Filter mongo_filter = {"host": host, "service": service} # Flatten dict serv data = flatten_dict(data) # Save in mongo try: mongo_res = getattr(self.db_conn, self.db_name).services.update(mongo_filter, {"$set": data}, upsert=True) except Exception as exp: logger.error("[SnmpBooster] [code 1204] [%s, %s] " "%s" % (host, service, str(exp))) return (None, True) return (None, self.handle_error(mongo_res, mongo_filter))
def get_best_lr_for_linear_bert_sentiment(): datasets = ['mr', 'subj', 'mpqa', 'sst'] lrs = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001, 0.0000001] metric = "best_valid_err" best_lr = {} for dataset in datasets: all_json_regex = "../../results/predictions/dimensionality_2019-07-07/{}/nbit_32/*/final_results.json".format( dataset) results = utils.gather_results(all_json_regex) results = [utils.flatten_dict(result) for result in results] best_err = 1.0 for lr in lrs: keys = {"lr": [lr]} subset_results = utils.extract_result_subset(results, keys) assert len(subset_results) == 3 ave, std = utils.stats_on_subset_json(subset_results, metric) print(dataset, " lr ", lr, " val err ave/std", ave, std) if ave < best_err: best_err = ave best_lr[dataset] = lr print(best_lr) return best_lr
def __init__(self, result, compare_method, compare_metric): super(Compare, self).__init__() """ Compare Args: result (dict): result dict compare_method (fn): function to compare two metrics compare_metric (fn): function to specify metrics to be compared """ self.result = result self.compare_metric = compare_metric self.compare_method = compare_method self.four_metrics = {} self.compare_result = {} for error_type in config.error_types: self.compare_result[error_type['name']], self.four_metrics[ error_type['name']] = self.compare_error(error_type['name']) # key order: error/clean_method/dataset/models/scenario/ [compare_keys...] self.compare_result = utils.flatten_dict(self.compare_result) # rearrange key order: error/dataset/clean_method/models/scenario/ [compare_keys...] self.compare_result = utils.rearrange_dict(self.compare_result, [0, 2, 1, 3, 4])
def label_predict(_config, _seed, _run): """Train a model with configurations.""" if ('complete' in _run.info) and (_run.info['complete']): print("The task is already finished") return None log_dir = os.path.join( "logs", _config['db_name'], datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')) _run.info['log_dir'] = log_dir random.seed(_seed) np.random.seed(_seed) torch.manual_seed(_seed) torch.cuda.manual_seed(_seed) # torch.cuda.set_device(_config['gpu']) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False writer = SummaryWriter(log_dir) # load datasets and build model datasets = get_dataset(**_config['dataset']) train_dataset_joint, valid_dataset_joint, train_dataset_marginal, valid_dataset_marginal, test_dataset = datasets if _config['classifier']['label_size'] < 1.0: # train_unsup_dataset_joint = train_dataset_joint train_dataset_joint, _ = get_split_datasets( train_dataset_joint, split_size=_config['classifier']['label_size']) train_dataset_marginal, _ = get_split_datasets( train_dataset_marginal, split_size=_config['classifier']['label_size']) if _config['method']['sampler_mode'] == 'random': print("Sample mode: Random") train_loader_joint = data.DataLoader( train_dataset_joint, batch_size=_config['optim']['batch_size'], shuffle=True) train_loader_marginal = data.DataLoader( train_dataset_marginal, batch_size=_config['optim']['batch_size'], shuffle=True) elif _config['method']['sampler_mode'] == 'diff': joint_sampler = get_split_samplers(train_dataset_joint, [0, 1, 2]) joint_batch_sampler = SplitBatchSampler(joint_sampler, _config['optim']['batch_size'], True) train_loader_joint = data.DataLoader(train_dataset_joint, batch_sampler=joint_batch_sampler) marginal_sampler = get_split_samplers(train_dataset_marginal, [1, 2, 0]) marginal_batch_sampler = SplitBatchSampler(marginal_sampler, _config['optim']['batch_size'], True) train_loader_marginal = data.DataLoader(train_dataset_marginal, batch_sampler=marginal_batch_sampler) elif _config['method']['sampler_mode'] == 'same': joint_sampler = get_split_samplers(train_dataset_joint, [0, 1, 2]) joint_batch_sampler = SplitBatchSampler(joint_sampler, _config['optim']['batch_size'], True) train_loader_joint = data.DataLoader(train_dataset_joint, batch_sampler=joint_batch_sampler) marginal_sampler = get_split_samplers(train_dataset_marginal, [0, 1, 2]) marginal_batch_sampler = SplitBatchSampler(marginal_sampler, _config['optim']['batch_size'], True) train_loader_marginal = data.DataLoader(train_dataset_marginal, batch_sampler=marginal_batch_sampler) print("Dataset: {} train, {} valid, {} test".format( len(train_dataset_joint), len(valid_dataset_joint), len(test_dataset))) model = get_model( input_shape=train_dataset_joint.get('input_shape'), K=_config['dataset']['K'], **_config['method']) if _config['classifier']['pretrain']: query = deepcopy(_config) del query['classifier'] del query['classifier_optim'] del query['db_name'] del query['gpu'] del query['unsup_db_name'] query = flatten_dict(query) extractor = MongoExtractor(None, _config['unsup_db_name']) result = list(extractor.find(query, ['config', 'info'], False, 'COMPLETED')) assert len(result) == 1, "There are too many or no results. Please check the query {}".format(query) result = result[0] if _config['method']['name'] == 'CPC': path = os.path.join(result['info']['log_dir'], 'model_{}.pth'.format(_config['optim']['num_batch'])) model = model.cpu() model.load_state_dict(torch.load(path, map_location='cpu')) model = model.cuda() elif _config['method']['name'] == 'VAE': path = os.path.join(result['info']['log_dir'], 'q_{}.pth'.format(_config['optim']['num_batch'])) # TODO: Need refactor from run_sacred import Encoder, CPCModel from vae import Inference g_enc_size = _config['method']['hidden'] g_enc = Encoder(input_shape=train_dataset_joint.get('input_shape'), hidden_size=None).cuda() c_enc = model.c_enc predictor = model.predictor q = Inference(g_enc, network_output=g_enc.output_shape()[1], z_size=g_enc_size) q.load_state_dict(torch.load(path, map_location='cpu')) q = q.cuda() g_enc = nn.Sequential(q.network, q.network_mu, nn.ReLU(True), nn.Dropout(0.5)) g_enc.output_shape = lambda: (None, g_enc_size) # dummy function, may be there exists a better way model = CPCModel(g_enc, c_enc, predictor).cuda() classifier = get_classifier(model, train_dataset_joint.get('num_classes'), **_config['classifier']) print(classifier) # TODO: Select valid poarameter from dictionary # MEMO: It can be donw with inspect module. optimizer = optim.Adam(classifier.parameters(), lr=_config['classifier_optim']['lr']) criterion = nn.NLLLoss() # Training and evaluation L = _config['dataset']['L'] monitor_per = _config['classifier_optim']['monitor_per'] divergence_criterion = CMD(n_moments=5) get_g_of = get_feature_of(model.g_enc, None, _config['dataset']['L']) get_c_of = get_feature_of(model.g_enc, model.c_enc, _config['dataset']['L']) # early stopping metric_names = ['test-accuracy', 'valid-accuracy'] best_values = dict(zip(metric_names, [0] * len(metric_names))) patient = 10 counter = 0 start_time = time.time() for num_iter in range(_config['classifier_optim']['num_batch']): optimizer.zero_grad() X, Y = train_loader_joint.__iter__().__next__() y = Y[:, 0, L-1].long().cuda() pred_y = classifier(X[..., :L].float().cuda()) loss = criterion(pred_y, y) if _config['classifier']['auxiliary'] != 0.0: assert _config['method']['name'] == 'CPC' aux_criterion = nn.BCELoss() # X_m, _ = train_loader_marginal.__iter__().__next__() num_negative = _config['method']['num_negative'] num_spy_mask = _config['method']['num_spy_mask'] X_m = [None] * num_negative for i in range(num_negative): _X, _ = train_loader_marginal.__iter__().__next__() X_m[i] = _X.float().cuda() K = X_m[0].shape[-1] L = X.shape[-1] - K X = X.float().cuda() score_j_list, score_m_list = model(X, X_m, L, K) score_j_list, score_m_list, predictions = model(X, X_m, L, K, return_predictions=True) _loss = 0 if num_spy_mask != 0: masks = model._get_masks(predictions[0], p=model.p, num_mask=num_spy_mask) spy_score_j_list = model.get_score_of(X[..., L:], K, predictions, masks) spy_score_m_list = [None] * len(X_m) for i, X in enumerate(X_m): score_m = model.get_score_of(X, K, predictions, masks) spy_score_m_list[i] = score_m for k in range(K): score_j = score_j_list[k] loss += aux_criterion(torch.sigmoid(score_j), torch.ones((len(score_j), 1)).cuda()) for m in range(num_negative): score_m = score_m_list[m][k] if num_spy_mask != 0: spy_score_j = spy_score_j_list[k] spy_score_m = spy_score_m_list[m][k] spy_mask = (spy_score_j > spy_score_m).float().detach() score_m = score_m * spy_mask _loss += aux_criterion(torch.sigmoid(score_m), torch.zeros((len(score_j), 1)).cuda()) _loss = _loss / (2*K) loss += _config['classifier']['auxiliary'] * _loss loss.backward() optimizer.step() if ((num_iter + 1) % monitor_per) != 0: continue train_result = validate_label_prediction(classifier, train_dataset_joint, L=L, nb_batch=None) valid_result = validate_label_prediction(classifier, valid_dataset_joint, L=L, nb_batch=None) test_result = validate_label_prediction(classifier, test_dataset, L=L, nb_batch=None) elapsed_time_before_cmd = time.time() - start_time valid_result['cmdg'] = pairwise_divergence( valid_dataset_joint.datasets, get_g_of, divergence_criterion, num_batch=None, batch_size=128 ) valid_result['cmdc'] = pairwise_divergence( valid_dataset_joint.datasets, get_c_of, divergence_criterion, num_batch=None, batch_size=128 ) test_result['cmdg'] = pairwise_divergence( [valid_dataset_joint, test_dataset], get_g_of, divergence_criterion, num_batch=None, batch_size=128 ) test_result['cmdc'] = pairwise_divergence( [valid_dataset_joint, test_dataset], get_c_of, divergence_criterion, num_batch=None, batch_size=128 ) # model_path = '{}/model_{}.pth'.format(log_dir, num_iter+1) # torch.save(model.state_dict(), model_path) writer.add_scalars('train', train_result, num_iter+1) writer.add_scalars('valid', valid_result, num_iter+1) writer.add_scalars('test', test_result, num_iter+1) elapsed_time = time.time() - start_time print("-------- #iter: {}, elapad time: {} ({}) --------".format( num_iter+1, elapsed_time, elapsed_time_before_cmd)) print('train', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(train_result)])) print('valid', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(valid_result)])) print('test', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(test_result)])) start_time = time.time() # early stopping is_better = False for metric_name in metric_names: db_name, metric = metric_name.split('-') if db_name == 'valid': current_value = valid_result[metric] elif db_name == 'test': current_value = test_result[metric] if current_value >= best_values[metric_name]: counter = 0 best_values[metric_name] = current_value is_better = True if is_better: print("Update the best results", best_values) else: counter += 1 print("Not update the best results (counter={})".format(counter), best_values) if counter == patient: break # Post process result = writer.scalar_dict for k, v in iteritems(result): ks = k.split('/') _run.info['{}-{}'.format(ks[-2], ks[-1])] = v
def entry_train(cfg, record_file=""): loader_train, _, _ = create_dataloader(split='train', **cfg.DATALOADER) loader_valid, _, _ = create_dataloader('valid', **cfg.DATALOADER) loader_test, _, _ = create_dataloader('test', **cfg.DATALOADER) model = get_model(cfg) model.to(DEVICE) print(model) optimizer = torch.optim.Adam(model.parameters(), lr=cfg.TRAIN.learning_rate, weight_decay=cfg.TRAIN.l2) scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=10, verbose=True) best_acc_rel_avg_valid = -1 best_epoch_rel_avg_valid = 0 best_acc_rel_avg_test = -1 log_dir = f"./runs/{cfg.EXP.EXP_ID}" if not os.path.exists(log_dir): os.makedirs(log_dir) tb = TensorboardManager(log_dir) for epoch in range(cfg.TRAIN.num_epochs): print('\nEpoch #%d' % epoch) print('Training..') (acc_train, pre_train, rec_train, f1_train, acc_rel_train, acc_rel_avg_train) = train(loader_train, model, optimizer, DEVICE, cfg.TRAIN.weighted_loss) print(f'Train, acc avg: {acc_rel_avg_train} acc: {acc_train},' f' pre: {pre_train}, rec: {rec_train}, f1: {f1_train}') print({x: round(y, 3) for x, y in acc_rel_train.items()}) tb.update('train', epoch, {'acc': acc_train}) print('\nValidating..') (acc_valid, pre_valid, rec_valid, f1_valid, acc_rel_valid, acc_rel_avg_valid) = validate(loader_valid, model, DEVICE) print(f'Valid, acc avg: {acc_rel_avg_valid} acc: {acc_valid},' f' pre: {pre_valid}, rec: {rec_valid}, f1: {f1_valid}') print({x: round(y, 3) for x, y in acc_rel_valid.items()}) tb.update('val', epoch, {'acc': acc_valid}) print('\nTesting..') (acc_test, pre_test, rec_test, f1_test, acc_rel_test, acc_rel_avg_test) = validate(loader_test, model, DEVICE) print(f'Test, acc avg: {acc_rel_avg_test} acc: {acc_test},' f' pre: {pre_test}, rec: {rec_test}, f1: {f1_test}') print({x: round(y, 3) for x, y in acc_rel_test.items()}) if acc_rel_avg_valid > best_acc_rel_avg_valid: print('Accuracy has improved') best_acc_rel_avg_valid = acc_rel_avg_valid best_epoch_rel_avg_valid = epoch save_checkpoint(epoch, model, optimizer, acc_rel_avg_valid, cfg) if acc_rel_avg_test > best_acc_rel_avg_test: best_acc_rel_avg_test = acc_rel_avg_test if (epoch - best_epoch_rel_avg_valid) > cfg.TRAIN.early_stop: print(f"Early stopping at {epoch} as val acc did not improve" f" for {cfg.TRAIN.early_stop} epochs.") break scheduler.step(acc_train) print('\nTesting..') load_best_checkpoint(model, cfg) (acc_test, pre_test, rec_test, f1_test, acc_rel_test, acc_rel_avg_test) = validate(loader_test, model, DEVICE) print(f'Best valid, acc: {best_acc_rel_avg_valid}') print(f'Best test, acc: {best_acc_rel_avg_test}') print(f'Test at best valid, acc avg: {acc_rel_avg_test}, acc: {acc_test},' f' pre: {pre_test}, rec: {rec_test}, f1: {f1_test}') print({x: round(y, 3) for x, y in acc_rel_test.items()}) if record_file != "": exp = RecordExp(record_file) exp.record_param(flatten_dict(dict(cfg))) exp.record_result({ "final_train": acc_rel_avg_train, "best_val": best_acc_rel_avg_valid, "best_test": best_acc_rel_avg_test, "final_test": acc_rel_avg_test })
def train(train_data_dir: str, hparams: HyperParameters, train_config: TrainConfig) -> TrainingResults: print("Hyperparameters:", hparams) print("Train_config:", train_config) start_time = time.time() # save the hyperparameter config to a file. with open(os.path.join(train_config.log_dir, "hyperparameters.json"), "w") as f: json.dump(asdict(hparams), f, indent=4) with open(os.path.join(train_config.log_dir, "train_config.json"), "w") as f: json.dump(asdict(train_config), f, indent=4) model = get_model(hparams) model.summary() train_dataset, valid_dataset = train_input_pipeline( train_data_dir, hparams, train_config) if DEBUG: train_dataset = train_dataset.repeat(100) if valid_dataset: valid_dataset = valid_dataset.repeat(100) using_validation_set = valid_dataset is not None hparams_dict = asdict(hparams) import pprint pprint.pprint(hparams_dict, indent=4) flattened_dict = utils.flatten_dict(hparams_dict) training_callbacks = [ tf.keras.callbacks.TensorBoard(log_dir=train_config.log_dir, profile_batch=0), hp.KerasCallback(train_config.log_dir, flattened_dict), tf.keras.callbacks.TerminateOnNaN(), utils.EarlyStoppingWhenValueExplodes(monitor="loss", check_every_batch=True), tf.keras.callbacks.ModelCheckpoint( filepath=os.path.join(train_config.log_dir, "model.h5"), monitor="val_loss" if using_validation_set else "loss", verbose=1, save_best_only=True, mode='auto'), tf.keras.callbacks.EarlyStopping( patience=train_config.early_stopping_patience, monitor='val_loss' if using_validation_set else "loss"), ] history = None try: history = model.fit( train_dataset, validation_data=valid_dataset if using_validation_set else None, epochs=train_config.epochs, callbacks=training_callbacks, # steps_per_epoch=int(train_samples / hparams.batch_size), ) end_time = time.time() training_time_secs = end_time - start_time loss_metric = "val_loss" if using_validation_set else "loss" best_loss_value = min(history.history[loss_metric]) num_epochs = len(history.history[loss_metric]) print( f"BEST {'VALIDATION' if using_validation_set else 'TRAIN'} LOSS:", best_loss_value) results = TrainingResults() results.total_epochs = num_epochs results.model_param_count = model.count_params() results.training_time_secs = training_time_secs if using_validation_set: metrics = model.evaluate(valid_dataset) results.metrics_dict = OrderedDict( zip(model.metrics_names, metrics)) else: results.metrics_dict = { metric_name: values[-1] for metric_name, values in history.history.items() } return results except Exception as e: print(f"\n\n {e} \n\n") return TrainingResults()
def forward(self, x_0, x, a, prior_sample=True, return_x=False): keys = set(locals().keys()) loss = 0. # for backprop s_losss = [0.] * self.num_states x_losss_p = [0.] * self.num_states x_losss_q = [0.] * self.num_states s01_losss = [0.] * self.num_states if self.args.posterior_s_0: s_0_s_loss = None s_0_x_loss = None if self.debug: s_abss_p = [0.] * self.num_states s_stds_p = [0.] * self.num_states s_abss_q = [0.] * self.num_states s_stds_q = [0.] * self.num_states keys = set(locals().keys()) - keys - {"keys"} x = x.transpose(0, 1).to(self.device) # T,B,3,28,28 a = a.transpose(0, 1).to(self.device) # T,B,4 x_0 = x_0.to(self.device) _T, _B = x.size(0), x.size(1) _x_p = [] _x_q = [] if self.args.posterior_s_0: s_prevs, s_0_s_loss, s_0_x_loss = self.sample_s_0(x_0) else: s_prevs = self.sample_s_0(x_0) for t in range(_T): x_t, a_t = x[t], a[t] s_ts = [] # store all hierarchy's state for i in range(self.num_states): h_t = self.encoders[i](x_t) s_prev = s_prevs[i] a_list = [a_t] + s_ts[-1:] # use upper state q = Normal(*self.posteriors[i](s_prev, h_t, a_list)) p = Normal(*self.priors[i](s_prev, a_list)) s_losss[i] += torch.sum(kl_divergence(q, p), dim=[1]).mean() s_t_p = p.mean if prior_sample else p.rsample() s_t_q = q.rsample() s_t = s_t_p if prior_sample else s_t_q s_ts.append(s_t) s01_losss[i] += torch.sum(kl_divergence(q, Normal(0., 1.)), dim=[1]).mean() if self.debug: s_abss_p[i] += p.mean.abs().mean() s_stds_p[i] += p.stddev.mean() s_abss_q[i] += q.mean.abs().mean() s_stds_q[i] += q.stddev.mean() decoder_dist_p = Normal(*self.decoders[i](s_t_p)) x_losss_p[i] += -torch.sum(decoder_dist_p.log_prob(x_t), dim=[1, 2, 3]).mean() decoder_dist_q = Normal(*self.decoders[i](s_t_q)) x_losss_q[i] += -torch.sum(decoder_dist_q.log_prob(x_t), dim=[1, 2, 3]).mean() if return_x: _x_p.append(decoder_dist_p.mean) _x_q.append(decoder_dist_q.mean) s_prevs = s_ts if return_x: _x_p = torch.stack(_x_p).transpose(0, 1) _x_p = torch.clamp(_x_p, 0, 1) _x_q = torch.stack(_x_q).transpose(0, 1) _x_q = torch.clamp(_x_q, 0, 1) return _x_p, _x_q else: for i in range(self.num_states): if i not in self.args.static_hierarchy: loss += s_losss[i] + x_losss_q[i] if self.args.beta_x_p: loss += self.args.beta_x_p * x_losss_p[i] if self.args.beta_s01: loss += self.args.beta_s01 * s01_losss[i] if self.args.posterior_s_0: loss += s_0_s_loss + s_0_x_loss _locals = locals() info = flatten_dict({key: _locals[key] for key in keys}) info = dict(sorted(info.items())) return loss, info
def parse_loaded_data(data_dict): effective_score_labels = [ "arrival_order", "frame_no", "horse_no", # "horse_name", "horse_sex", "horse_age", "horse_weight", "horse_weight_diff", # "horse_b", # "arrival_diff", "time", "last3f_time", # "passing_order_1st", # "passing_order_2nd", # "passing_order_3rd", # "passing_order_4th", # "jockey_name", "jockey_weight", "odds", "popularity", # "trainer_name", ] effective_racehead_labels = [ "race_no", # "date", "week", "kai", # "lacation", "nichi", # "start_time", "weather", "condition", ] # ネスト化された辞書を平坦化 score_dicts = [] racehead_dict = {} for k, v in data_dict.items(): if k == 'scores': for s in v: flattened_dict = flatten_dict(s) score_dict = { k_: v_ for k_, v_ in flattened_dict.items() if k_ in effective_score_labels } score_dicts.append(score_dict) elif k == 'racehead': flattened_dict = flatten_dict(v) racehead_dict = { k_: v_ for k_, v_ in flattened_dict.items() if k_ in effective_racehead_labels } # データを整形 score_and_racehead = [] # racehead racehead = [] for k, v in racehead_dict.items(): if k == 'week': racehead.append(parse_week(v)) # elif k == 'start_time': # racehead.append(parse_start_time(v)) elif k == 'weather': racehead.append(parse_weather(v)) elif k == 'condition': racehead.append(parse_condition(v)) else: racehead.append(v) arrival_orders = [] for score_dict in score_dicts: # score score = [] for k, v in score_dict.items(): if k == 'arrival_order': arrival_orders.append(v) elif k == 'horse_sex': score.append(parse_horse_sex(v)) elif k == 'horse_weight_diff': score.append(parse_horse_weight_diff(v)) elif k == 'jockey_weight': score.append(parse_jockey_weight(v)) else: score.append(v) score_and_racehead.append(score + racehead) return score_and_racehead, arrival_orders
def format_record(recid, record=None): if not record: record = ALL[str(recid)] if len(record): user_id = get_id(record) # uid = record.get('UID', '') # guid = record.get('GUID', '') # pa = record.get('PA', '') # user_id = uid # if len(user_id) != 6: # user_id = guid # if len(user_id) != 6: # user_id = pa # if len(user_id) != 6: # user_id = uid # if len(user_id) != 7: # user_id = guid # if len(user_id) != 7: # user_id = pa # if len(user_id) != 7: # user_id = uid # print 'recno: %s\tuid: %s\tguid: %s\tpa: %s' % (recid, uid, guid, pa) template = {'tag': '999', 'ind1': ' ', 'ind2': ' ', 'subs': {'a' : user_id}} tags = [] tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs']))) for (field, value) in record.items(): if field in mapping: subfields = mapping[field](value) for template in subfields: tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs']))) tags.sort() res = Record() for tag in tags: res.add_ordered_field(tag) return merge_subfields(res, ['101', '104', '110', '130', '400'])
def format_record(recid, record=None): if not record: record = ALL[str(recid)] if len(record): tags = [] for (field, value) in record.items(): if field in mapping: subfields = mapping[field](value) for template in subfields: tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs']))) tags.sort() res = Record() for tag in tags: res.add_ordered_field(tag) return merge_subfields(res, [])
def norm(params, p=2.): sqnorm = jnp.sum(flatten_dict(lambda x: (x**p).sum(), params)) sqnorm = (sqnorm**(1. / p)) return sqnorm
def main(): X_train = pd.read_pickle('input/train.pkl').drop( 'building_id', axis=1).sample(frac=0.1, random_state=42) y_train = np.log1p(X_train.pop('meter_reading')) X_test = pd.read_pickle('input/test.pkl').drop(['row_id', 'building_id'], axis=1) process_date(X_train) process_date(X_test) label_encoding(X_train, X_test) models = [] metrics = {'train': [], 'valid': []} feature_importances_split = np.zeros(X_train.shape[1]) feature_importances_gain = np.zeros(X_train.shape[1]) fold = KFold(**PARAMS['fold']) mlflow.set_experiment('ashrae-energy-prediction') with mlflow.start_run() as run: mlflow.log_params(flatten_dict(PARAMS)) for fold_idx, (idx_tr, idx_val) in enumerate(fold.split(X_train)): print_divider(f'Fold {fold_idx}') X_tr, X_val = X_train.iloc[idx_tr], X_train.iloc[idx_val] y_tr, y_val = y_train.iloc[idx_tr], y_train.iloc[idx_val] model = lgbm.LGBMRegressor(**PARAMS['model']) model.fit(X_tr, y_tr, eval_set=[(X_tr, y_tr), (X_val, y_val)], eval_names=list(metrics.keys()), **PARAMS['fit']) feature_importances_split += devide_by_sum( model.booster_.feature_importance( importance_type='split')) / fold.n_splits feature_importances_gain += devide_by_sum( model.booster_.feature_importance( importance_type='gain')) / fold.n_splits for key in metrics.keys(): metrics[key].append({ 'name': 'rmse', 'values': model.evals_result_[key]['rmse'], 'best_iteration': model.best_iteration_ }) models.append(model) del idx_tr, idx_val, X_tr, X_val, y_tr, y_val for key in metrics.keys(): log_plot(metrics[key], pf.metric_history, f'metric_history_{key}.png') features = np.array(model.booster_.feature_name()) log_plot((features, feature_importances_split, 'split'), pf.feature_importance, 'feature_importance_split.png') log_plot((features, feature_importances_gain, 'gain'), pf.feature_importance, 'feature_importance_gain.png') # prediction (split the data into chunks because it's too large) print_divider('prediction') i = 0 pred = [] chunk_size = 500000 for j in tqdm(range(int(np.ceil(X_test.shape[0] / chunk_size)))): pred.append( np.expm1( sum([ model.predict(X_test.iloc[i:i + chunk_size]) for model in models ]) / fold.n_splits)) i += chunk_size # make submission submission = pd.read_csv('input/sample_submission.csv') submission['meter_reading'] = pred submission.loc[submission['meter_reading'] < 0, 'meter_reading'] = 0 submission_path = f'submission/{timestamp()}.csv' submission.to_csv(submission_path, index=False) mlflow.log_param('submission_path', submission_path) # run the MLflow UI to check the training result print_divider('MLflow UI') url_base = 'http://127.0.0.1:5000/#/experiments/{0}/runs/{1}' print('Run URL:', url_base.format(run.info.experiment_id, run.info.run_uuid)) mlflow_ui()
def flatten(self, filename): return utils.flatten(utils.flatten_dict(utils.load_file(filename)))
def train_and_predict(X_train, y_train, X_test, params, experiment_path): # explicitly setting up mlflow experiment try: mlflow.create_experiment(experiment_path) except (mlflow.exceptions.RestException, mlflow.exceptions.MlflowException): print(f'The specified experiment ({experiment_path}) already exists.') skf = StratifiedKFold(**params['fold']) proba_oof = np.zeros(X_train.shape[0]) pred_oof = np.zeros(X_train.shape[0]) proba_test = np.zeros(X_test.shape[0]) metric_history = [] avg_scores = defaultdict(int) feature_importances_split = np.zeros(X_train.shape[1]) feature_importances_gain = np.zeros(X_train.shape[1]) with mlflow.start_run() as run: for fold, (trn_idx, val_idx) in enumerate(skf.split(X_train, y_train)): print_divider(f'Fold: {fold}') X_trn, y_trn = X_train.iloc[trn_idx, :], y_train.iloc[trn_idx] X_val, y_val = X_train.iloc[val_idx, :], y_train.iloc[val_idx] model = lgbm.LGBMClassifier(**params['model']) model.fit(X_trn, y_trn, eval_set=[(X_val, y_val)], eval_names=['valid'], **params['fit']) metric_history.append({ 'name': model.metric, 'values': model.evals_result_['valid'][model.metric], 'best_iteration': model.best_iteration_ }) feature_importances_split += devide_by_sum( model.booster_.feature_importance( importance_type='split')) / skf.n_splits feature_importances_gain += devide_by_sum( model.booster_.feature_importance( importance_type='gain')) / skf.n_splits proba_val = model.predict_proba( X_val, num_iteration=model.best_iteration_)[:, 1] pred_val = (proba_val > 0.5).astype(np.int8) proba_oof[val_idx] = proba_val pred_oof[val_idx] = pred_val scores_val = get_scores(y_val, pred_val) mlflow.log_metrics( { **scores_val, 'best_iteration': model.best_iteration_, }, step=fold) for k, v in scores_val.items(): avg_scores[k] += v / skf.n_splits proba_test += model.predict_proba(X_test)[:, 1] / skf.n_splits del trn_idx, X_trn, y_trn, val_idx, X_val, y_val, proba_val, pred_val gc.collect() mlflow.log_params({ **flatten_dict(params), 'fold.strategy': skf.__class__.__name__, 'model.type': model.__class__.__name__ }) print_divider('Saving plots') # scores log_plot(avg_scores, pf.scores, 'scores.png') # feature importance (only top 30) features = np.array(model.booster_.feature_name()) log_plot((features, feature_importances_split, 'split', 30), pf.feature_importance, 'feature_importance_split.png') log_plot((features, feature_importances_gain, 'gain', 30), pf.feature_importance, 'feature_importance_gain.png') # metric history log_plot(metric_history, pf.metric_history, 'metric_history.png') # confusion matrix cm = confusion_matrix(y_train, pred_oof) log_plot(cm, pf.confusion_matrix, 'confusion_matrix.png') # roc curve fpr, tpr, _ = roc_curve(y_train, pred_oof) roc_auc = roc_auc_score(y_train, pred_oof) log_plot((fpr, tpr, roc_auc), pf.roc_curve, 'roc_curve.png') # precision-recall curve pre, rec, _ = precision_recall_curve(y_train, pred_oof) pr_auc = average_precision_score(y_train, pred_oof) log_plot((pre, rec, pr_auc), pf.pr_curve, 'pr_curve.png') return proba_test, run.info.experiment_id, run.info.run_uuid
def save_four_metrics(self, save_dir): for error_type in config.error_types: save_path = os.path.join( save_dir, "{}_four_metrics.xlsx".format(error_type['name'])) utils.dfs_to_xls(self.four_metrics[error_type['name']], save_path) flat_metrics = utils.flatten_dict(self.four_metrics)
def flatten_dict(self, dct): return flatten_dict(dct)
def format_record(recid, record=None): if not record: record = ALL[str(recid)] if len(record): tags = [] try: tags.append(Field(tag='999', indicators=[' ', ' '], subfields=['a', match_serial(record)])) except: with codecs.open(log_dir + 'unmatched_serials.txt', 'a', encoding='utf8') as f: f.write(recid + '\r\n') f.close() for (field, value) in record.items(): if field in mapping: try: subfields = mapping[field](value) for template in subfields: tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs']))) except: with codecs.open(log_dir + 'log_serials.txt', 'a', encoding='utf8') as f: f.write(("==================\n%s\n%s:\t%s\n\n" % (recid, field, record[field])).replace('\n', '\r\n')) f.close() tags.sort() res = Record() for tag in tags: res.add_ordered_field(tag) return merge_subfields(res, [])