def prepare_hyper_search(cfg_kwargs: dict, reporthook=None, final_reporthook=None, primary_key=None, max_key=True, reporter_cls=None, with_keys: (list, str, None) = None, final_keys: (list, str, None) = None, dump=False, disable=False): """ Updated in v1.3.18 从 nni package 中获取超参,更新配置文件参数。当 nni 不可用或不是 nni 搜索模式时,参数将不会改变。 .. code-block :: python cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search( cfg_kwargs, Configuration, reporthook, final_reporthook, primary_key="macro_avg:f1" ) _cfg = Configuration(**cfg_kwargs) model = Model(_cfg) ... for epoch in range(_cfg.begin_epoch, _cfg.end_epoch): for batch_data in dataset: train_model(batch_data) data = evaluate_model() reporthook(data) final_reporthook() Parameters ---------- cfg_kwargs: dict 待传入cfg的参数 reporthook final_reporthook primary_key: 评估模型用的主键, ``nni.report_intermediate_result`` 和 ``nni.report_final_result`` 中 ``metric`` 的 ``default`` max_key: bool 主键是越大越好 reporter_cls with_keys: list or str 其它要存储的 metric,final report时默认为 primary_key 最优时指标 final_keys: list or str with_keys 中使用最后一个 report result 而不是 primary_key 最优时指标 dump: bool 为 True 时,会修改 配置文件 中 workspace 参数为 ``workspace/nni.get_experiment_id()/nni.get_trial_id()`` 使得 nni 的中间结果会被存储下来。 disable Returns ------- cfg_kwargs: dict 插入了nni超参后的配置文件参数 reporthook: function 每个iteration结束后的回调函数,用来报告中间结果。 默认 ``nni.report_intermediate_result``。 final_reporthook: 所有iteration结束后的回调函数,用来报告最终结果。 默认 ``nni.report_final_result`` dump: bool 和传入参数保持一致 Examples -------- .. code-block :: python class CFG(Configuration): hyper_params = {"hidden_num": 100} learning_rate = 0.001 workspace = "" cfg_kwargs, reporthook, final_reporthook, dump = prepare_hyper_search( {"learning_rate": 0.1}, CFG, primary_key="macro_avg:f1", with_keys="accuracy" ) # cfg_kwargs: {'learning_rate': 0.1} when nni start (e.g., using ``nni create --config _config.yml``), suppose in ``_config.yml``: .. code-block: yml searchSpacePath: _search_space.json and in ``_search_space.json`` .. code-block :: json { "hidden_num": {"_type": "choice", "_value": [500, 600, 700, 835, 900]}, } one of the return cfg_kwargs is ``{'hyper_params': {'hidden_num': 50}, 'learning_rate': 0.1}`` """ if disable: return cfg_kwargs, None, None, None try: import nni from nni import get_next_parameter, report_intermediate_result, report_final_result assert primary_key is not None def _as_key_list(_keys: (list, str, None)): if isinstance(_keys, str): if ";" in _keys: _keys = _keys.split(";") else: _keys = [_keys] elif isinstance(_keys, list): pass elif _keys is None: _keys = [] return _keys with_keys = _as_key_list(with_keys) final_keys = _as_key_list(final_keys) class Reporter(BaseReporter): def __init__(self): self.datas = [] def intermediate(self, data): feed_dict = { 'default': float(get_by_key(data, key_parser(primary_key))), primary_key: get_by_key(data, key_parser(primary_key)) } for key in with_keys: feed_dict[key] = get_by_key(data, key_parser(key)) report_intermediate_result(feed_dict) self.datas.append(data) def final(self): best_fn = get_min if max_key is False else get_max _with_keys = (with_keys if with_keys else []) + [primary_key] _final_keys = set(final_keys if final_keys else []) final_result = best_fn( self.datas, primary_key, with_keys=";".join(_with_keys), merge=False ) feed_dict = { 'default': float(final_result[0][primary_key]) } appendix_dict = dict(final_result[1][primary_key]) for key in _with_keys: if key in _final_keys: feed_dict[key] = get_by_key(self.datas[-1], key_parser(key)) else: feed_dict[key] = appendix_dict[key] report_final_result(feed_dict) rc = Reporter() if reporter_cls is None else reporter_cls reporthook = reporthook if reporthook is not None else rc.intermediate final_reporthook = final_reporthook if final_reporthook is not None else rc.final cfg_cls_params = get_params(get_next_parameter()) using_nni_tag = True if cfg_cls_params else False nested_update(cfg_kwargs, cfg_cls_params) if using_nni_tag is True and dump is True: # pragma: no cover cfg_kwargs["workspace"] = cfg_kwargs.get("workspace", "") + path_append( nni.get_experiment_id(), nni.get_trial_id(), to_str=True ) return cfg_kwargs, reporthook, final_reporthook, dump except ModuleNotFoundError: # pragma: no cover warnings.warn("nni package not found, skip") return cfg_kwargs, reporthook, final_reporthook, dump
dev_file = os.path.expanduser(args.dev_file) max_epoch = args.max_epoch cfg = GAGConfig() cfg.batch_size = args.batch_size cfg.learning_rate = float(args.learning_rate) cfg.dropout = args.dropout_rate cfg.rnn_units = args.rnn_units cfg.labelsmoothing = args.labelsmoothing cfg.num_heads = args.num_heads timer = Timer() qp_pairs, dev_qp_pairs = load_data() logger.debug('Init finish.') original_params = nni.get_next_parameter() ''' with open('data.json') as f: original_params = json.load(f) ''' p_graph = graph.graph_loads(original_params['graph']) save_path = original_params['save_dir'] os.makedirs(save_path) restore_path = original_params['restore_dir'] restore_shared = [ hash_id + '/' for hash_id in original_params['shared_id'] ] if original_params['shared_id'] is not None else [] + [ 'word_embed', 'char_embed', 'char_encoding/' ] train_loss, best_acc = train_with_graph(p_graph, qp_pairs, dev_qp_pairs)
import nni parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training') parser.add_argument('--bs', default=128, type=int, help='batch size') parser.add_argument('--lr', default=0.1, type=float, help='learning rate') parser.add_argument('--epochs', default=30, type=int, help='number of epochs') parser.add_argument('--model', default='ResNet18', type=str, help='model') parser.add_argument('--autoscale-bsz', dest='autoscale_bsz', default=True, action='store_true', help='autoscale batchsize') args = parser.parse_args() # load the parameters from nni RCV_CONFIG = nni.get_next_parameter() args.lr = RCV_CONFIG["lr"] device = 'cuda' if torch.cuda.is_available() else 'cpu' # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(),
def generate_default_params(): ''' Generate default hyper parameters ''' return { 'optimizer': 'Adam', 'learning_rate': 0.001 } if __name__ == '__main__': PARSER = argparse.ArgumentParser() PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) PARSER.add_argument("--num_train", type=int, default=60000, help="Number of train samples to be used, maximum 60000", required=False) PARSER.add_argument("--num_test", type=int, default=10000, help="Number of test samples to be used, maximum 10000", required=False) ARGS, UNKNOWN = PARSER.parse_known_args() try: # get parameters from tuner # RECEIVED_PARAMS = {"optimizer": "Adam", "learning_rate": 0.00001} RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) # train train(ARGS, PARAMS) except Exception as e: LOG.exception(e) raise
example_start_time = time.time() net = None args = get_args() try: experiment_path = os.environ[ "HOME"] + "/mountdir/nni/experiments/" + str( nni.get_experiment_id()) lock = multiprocessing.Lock() context = zmq.Context() socket = context.socket(zmq.REQ) tmpstr = 'tcp://' + args.ip + ':800081' socket.connect(tmpstr) os.makedirs(experiment_path + "/trials/" + str(nni.get_trial_id())) get_next_parameter_start = time.time() nni.get_next_parameter(socket) get_next_parameter_end = time.time() while True: lock.acquire() with open(experiment_path + "/graph.txt", "a+") as f: f.seek(0) lines = f.readlines() lock.release() if lines: break if len(lines) > args.slave: x = random.randint(1, args.slave) json_and_id_str = lines[-x].replace("\n", "") else:
def test_get_current_parameter(self): nni.get_next_parameter() self.assertEqual(nni.get_current_parameter('x'), 123)
def main(): hp_dict = nni.get_next_parameter() # hp_dict is automatically suggested _, test_acc = tunable_train(hp_dict) nni.report_final_result(test_acc) # report final accuracy
train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=compute_metrics, tb_writer=tb_writer) # Training if training_args.do_train: trainer.train(model_path=model_args.model_name_or_path if os.path. isdir(model_args.model_name_or_path) else None) # final eval train_time = (datetime.today() + timedelta(hours=9)).strftime("%Y-%m-%d") backbone = model_args.model_name_or_path.split('/')[-1] hyperparams = ",".join([f"{k}={param[k]}" for k in param]) app_key = f"{train_time}/{backbone}/{hyperparams}" df = trainer.get_best_model(metric='token-f1', app_key=app_key, app_name='sentiment-extractor') # hard-coded # nni nni.report_final_result(df['eval_token-f1'].max()) # best case if __name__ == '__main__': try: param_trials = nni.get_next_parameter() logger.debug(param_trials) main(param_trials) except Exception as exception: logger.exception(exception) raise
def main(argv): # basepath base_path = FLAGS.basepath if not os.path.exists(base_path): os.mkdir(base_path) # model path save_path = os.path.join(base_path, '%s_model' % str(time.time())) os.mkdir(save_path) params = nni.get_next_parameter() source_embedding_dim = params['encoder_hidden_dim'] * 2 encoder_hidden_dim = params['encoder_hidden_dim'] encoder_input_dropout = params['encoder_input_dropout'] encoder_output_dropout = params['encoder_output_dropout'] decoder_hidden_dim = params['encoder_hidden_dim'] * 2 decoder_num_layers = params['decoder_num_layers'] rule_embedding_dim = params['rule_embedding_dim'] nonterminal_embedding_dim = params['nonterminal_embedding_dim'] dropout = params['decoder_dropout'] batch_size = params['batch_size'] lr = params['lr'] command = '''CUDA_VISIBLE_DEVICES=%d python run_parser.py --do_train=True \ --source_embedding_dim=%d \ --encoder_hidden_dim=%d \ --encoder_bidirectional=True \ --encoder_input_dropout=%f \ --encoder_output_dropout=%f \ --decoder_hidden_dim=%d \ --decoder_num_layers=%d \ --rule_embedding_dim=%d \ --nonterminal_embedding_dim=%d \ --max_decode_length=200 \ --serialization_dir=%s \ --seed=1 \ --dropout=%f \ --task=geo \ --language=%s \ --train_data=%s \ --test_data=%s \ --batch_size=%d \ --lr=%f \ --patient=30 \ --optimizer=adam \ --epoch=50 \ --model_save_interval=1 \ --gradient_clip=5 \ ''' % ( FLAGS.cuda_device, source_embedding_dim, encoder_hidden_dim, encoder_input_dropout, encoder_output_dropout, decoder_hidden_dim, decoder_num_layers, rule_embedding_dim, nonterminal_embedding_dim, save_path, dropout, FLAGS.language, FLAGS.train_data, FLAGS.test_data, batch_size, lr) command = re.sub('\s+', ' ', command).strip() subprocess.call(command, shell=True) last_epoch = 0 for f in os.listdir(save_path): if f.startswith('metrics_epoch_'): match = PATTERN.match(f) if match: epoch = int(match.group(1)) if epoch > last_epoch: last_epoch = epoch file_path = os.path.join(save_path, 'metrics_epoch_%d.json' % last_epoch) with open(file_path, 'r') as f: metrics = json.load(f) accuracy = metrics['best_validation_accuracy'] nni.report_final_result(accuracy)
params = {'CT': 12, 'PT': 14, 'TT': 1, 'max_depth': 7, 'num_boost_round': 182} parser = argparse.ArgumentParser(description="Argument Parser") # data source parser = argparse.ArgumentParser(description="Argument Parser") parser.add_argument('--dataset', default='Metro', type=str) parser.add_argument('--city', default='Chongqing', type=str) parser.add_argument('--MergeIndex', default=3) parser.add_argument('--DataRange', default="all") parser.add_argument('--TrainDays', default="all") #use params and args to show its difference args = vars(parser.parse_args()) params.update(nni.get_next_parameter()) data_loader = NodeTrafficLoader( dataset=args["dataset"], city=args['city'], closeness_len=int(params['CT']), period_len=int(params['PT']), trend_len=int(params['TT']), data_range=args['DataRange'], train_data_length=args['TrainDays'], test_ratio=0.1, with_lm=False, normalize=False, MergeIndex=args['MergeIndex'], MergeWay="max" if args["dataset"] == "ChargeStation" else "sum")
'dropout_rate': 0.5, 'channel_1_num': 32, 'channel_2_num': 64, 'conv_size': 5, 'pool_size': 2, 'hidden_size': 1024, 'learning_rate': 1e-4, 'batch_size': 32 } return params if __name__ == '__main__': try: # get parameters form tuner RCV_PARAMS = nni.get_next_parameter() logger.debug(RCV_PARAMS) # run params = generate_default_params() params.update(RCV_PARAMS) ''' If you use Hyperband, among the hyperparameters (i.e., key-value pairs) received by a trial, there is one more key called `STEPS` besides the hyperparameters defined by user. By using this `STEPS`, the trial can control how long it runs. ''' params['batch_num'] = RCV_PARAMS['STEPS'] * 10 main(params) except Exception as exception: logger.exception(exception) raise
return params def parse_init_json(data): params = {} for key in data: value = data[key] if value == 'Empty': params[key] = ['Empty'] else: params[key] = [value[0], value[1], value[1]] return params if __name__ == '__main__': try: # get parameters form tuner data = nni.get_next_parameter() logger.debug(data) RCV_PARAMS = parse_init_json(data) logger.debug(RCV_PARAMS) params = generate_defualt_params() params.update(RCV_PARAMS) print(RCV_PARAMS) main(params) except Exception as exception: logger.exception(exception) raise
if __name__ == "__main__": datasets_mapping = { 'cifar10': datasets.CIFAR10, 'cifar100': datasets.CIFAR100, 'svhn': datasets.SVHN, } args = parse_args() set_seeds(args.seed) # -- # IO print('Making dataloaders ...', file=sys.stderr) transform_train = transforms.Compose([ Policy(nni.get_next_parameter()), transforms.ToTensor(), btransforms.NormalizeDataset(dataset=args.dataset), ]) transform_eval = transforms.Compose([ transforms.ToTensor(), btransforms.NormalizeDataset(dataset=args.dataset), ]) try: if args.dataset in ['cifar10', 'cifar100']: trainset = datasets_mapping[args.dataset]( root='./data', train=True, download=args.download,
def main(): # Training settings RCV_CONFIG = nni.get_next_parameter() _logger.debug(RCV_CONFIG) ''' parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') ''' parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=RCV_CONFIG['batch-size'], metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=RCV_CONFIG['epochs'], metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=RCV_CONFIG['lr'], metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) model = Net().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) nni.report_final_result(best_acc) if (args.save_model): torch.save(model.state_dict(),"mnist_cnn.pt")
choices=['sgd', 'rmsprop', 'adam']) parser.add_argument('--momentum', default=0.9, type=int, help='optimizer momentum (ignored in adam)') parser.add_argument('--num_workers', default=2, type=int, help='number of workers to preprocess data') parser.add_argument('--model', default='resnet18', choices=available_models, help='the model to use') parser.add_argument('--grad_clip', default=0., type=float, help='gradient clip (use 0 to disable)') parser.add_argument('--log_frequency', default=20, type=int, help='number of mini-batches between logging') parser.add_argument('--seed', default=42, type=int, help='global initial seed') args = parser.parse_args() nni.utils.merge_parameter(args, nni.get_next_parameter()) main(args)
parser.add_argument( "--eval_window_strict", type=bool, default=False, help= "True: expect 0 output for all segs except response; False: expect 0 for Cue only.", required=False) parser.add_argument( "--cl_platform_vendor", type=str, default='NVIDIA', help= "Use the first CL platform matching this vendor (case insensitive)", required=False) args, unknown = parser.parse_known_args() try: params = {**generate_default_params(), **nni.get_next_parameter()} logger.debug(params) run_trial(params, ens_seed=args.ens_seed, train_trials_seed=args.train_trials_seed, test_trials_seed=args.test_trials_seed, train_trials_per_cond=args.train_trials_per_cond, test_trials_per_cond=args.test_trials_per_cond, eval_strict=args.eval_window_strict, cl_platform_vendor=args.cl_platform_vendor) except Exception as e: logger.exception(e) raise
def run_nn_bacteria_network(tax, data_set_name): """ Complex models - feature importance calculation- NN is complex model which function as a black box that cannot be clearly deduced from the contribution of each feature. A different approach for inferring the importance of each bacterium in these models is required. Initially we trained the model to predict the change in each of the bacteria, we used the original data as an input. Now, although the equivalent to ‘coefficencts‘ is not visible to us, every bacterium has an effect, between none and extreme on the prediction. Therefore, we could estimate this effect by introducing modified input into the model and examining its effects on prediction. We chose to examine the relationship between each bacterial pair by using the existing fixed model that was trained for the ‘first’ bacterial prediction, and forward it an input that was modified only for the ‘second’ bacterium. U test can be used to investigate whether two independent samples were selected from populations having the same distribution. That is, the test can tell whether the distribution of predictions has changed significantly in light of the change in input - indicating interacting. Comparing the original prediction and the modified data’s prediction distributions, if the change between the two is significant according to U test, we conclude that there is interaction between the bacterial pair. The type of interaction will be determined by the obtained change- increasing or decreasing the count of the bacterium at a fixed size, and its effect, increase or decrease in the prediction of the count of bacteria. The change will be by the constant 'CHANGE', you can alter it :param tax: (string) main dataset folder "DATASET/tax=x" :return: doesn't return an object, create a csv file with all the results csv name = "interaction_network_" + params.__str__().replace(" ", "").replace("'", "") + "_df.csv" To determine which bacteria have interactions and create a visual graph use "built_network_form_file.py" sent the results csv path """ # sub folder path for network results, folder = os.path.join(tax, "interaction_network") if not os.path.exists(folder): os.mkdir(folder) # ------------------------------------ decide on mission ------------------------------------ nni_ = False # check model or run nni for real GPU = True if nni_ else False report_loss = True report_correlation = not report_loss single_bacteria = True # learn the change in single bacteria or all in once k_fold = False # run k fold p_value = 0.001 test_size = 0.3 if nni_: params = nni.get_next_parameter() else: params = { "STRUCTURE": "001L200H", "TRAIN_TEST_SPLIT": 0.7, "EPOCHS": 70, "LEARNING_RATE": 1e-3, "OPTIMIZER": "Adam", "REGULARIZATION": 0.01, "DROPOUT": 0.1 } with open(os.path.join(tax, "bacteria.txt"), "r") as b_file: bacteria = b_file.readlines() bacteria = [b.rstrip() for b in bacteria] bacteria_number_list = range(len(bacteria)) # ------------------------------------ data loading ------------------------------------ # run a prediction of a single bacteria at a time # consider the average loss and correlation of all runs as the performance measurement df_title = os.path.join( folder, "interaction_network_" + params.__str__().replace(" ", "").replace("'", "") + "_df.csv") df = pd.DataFrame(columns=["BACTERIA", "CHANGED_BACTERIA", "CHANGE", "Y"]) df.to_csv(df_title, index=False) # create a df that saves a binary value 1/0 => interaction/no interaction according to the train set train_binary_significant_df = pd.DataFrame(columns=bacteria) # create a df that saves the continuous b value of each bacteria according to the test set test_b_df = pd.DataFrame(columns=bacteria) for b_i, bacteria_num in enumerate( bacteria_number_list): # for each bacteria df = pd.read_csv(df_title) path = "X_y_for_bacteria_number_" + str(b_i) + ".csv" X_trains, X_tests, y_trains, y_tests, name = \ get_adapted_X_y_for_wanted_learning_task(tax, path, "regular", k_fold=1) X_train, X_test, y_train, y_test = X_trains[0], X_tests[0], y_trains[ 0], y_tests[0] NUMBER_OF_SAMPLES = X_train.shape[0] NUMBER_OF_BACTERIA = X_train.shape[1] NUMBER_OF_TIME_POINTS = None missing_values = np.array([1 for j in range(NUMBER_OF_SAMPLES)]) # split to train and test """ split_list = [1 - test_size, test_size] split_list = np.multiply(np.cumsum(split_list), len(X)).astype("int").tolist() # list of shuffled indices to sample randomly shuffled_idx = [] shuffle(person_indexes) for arr in person_indexes: for val in arr: shuffled_idx.append(val) # split the data itself X_train = X[shuffled_idx[:split_list[0]]] y_train = y[shuffled_idx[:split_list[0]]] X_test = X[shuffled_idx[split_list[0]:split_list[1]]] y_test = y[shuffled_idx[split_list[0]:split_list[1]]] """ train_binary_significant_for_b_i = [] test_1_u_score_for_b_i = [] """ path = "time_serie_X_y_for_bacteria_number_" + str(bacteria_num) + ".csv" X, y, missing_values, name = get_adapted_X_y_for_wanted_learning_task(tax, path, "time_serie") NUMBER_OF_SAMPLES = X.shape[0] NUMBER_OF_TIME_POINTS = X.shape[1] NUMBER_OF_BACTERIA = X.shape[2] flat_time_points_values_num = NUMBER_OF_SAMPLES * NUMBER_OF_TIME_POINTS X = X.reshape(flat_time_points_values_num, NUMBER_OF_BACTERIA) y = y.reshape(flat_time_points_values_num) missing_values = missing_values.reshape(flat_time_points_values_num) person_indexes = np.linspace(0, flat_time_points_values_num - 1, flat_time_points_values_num). \ reshape(NUMBER_OF_SAMPLES, NUMBER_OF_TIME_POINTS).astype(int).tolist() """ # TRAIN # run the model one time with no change, then save it res_map = run_NN(X_train, y_train, missing_values, params, name, folder, NUMBER_OF_SAMPLES, NUMBER_OF_TIME_POINTS, NUMBER_OF_BACTERIA, save_model=True, GPU_flag=GPU, k_fold=k_fold, task_id="base_" + str(b_i) + "_model", person_indexes=None) model_path = os.path.join( folder, "trained_models", params.__str__().replace(" ", "").replace("'", "") + "_base_" + str(b_i) + "_model_model") out_dim = 1 if len(y_train.shape) == 1 else y_train.shape[ 1] # else NUMBER_OF_BACTERIA structure = params["STRUCTURE"] layer_num = int(structure[0:3]) hid_dim_1 = int(structure[4:7]) hid_dim_2 = int(structure[8:11]) if len(structure) > 10 else None clf_params = { "NN_input_dim": X_train.shape[1], "NN_hidden_dim_1": hid_dim_1, "NN_output_dim": out_dim } clf = bacteria_network_clf(clf_params) clf.load(model_path) y_pred_no_change = clf.predict(torch.FloatTensor(X_train)) y_str = "" for val in y_pred_no_change: y_str += str(val.detach().numpy()[0]) + " " df.loc[len(df)] = [int(bacteria_num), int(-1), "no change", y_str] # ------------------------------------ send to network ------------------------------------ for bacteria_to_change_num in bacteria_number_list: # change each bacteria # change X, y for only bacteria_to_change_num X_positive_change = copy.deepcopy(X_train) for s_i, sample in enumerate( X_positive_change ): # 0.9459053900000001 -0.05409460999999999 X_positive_change[s_i][bacteria_to_change_num] += CHANGE y_pred_pos_change = clf.predict( torch.FloatTensor(X_positive_change)) y_str = "" for val in y_pred_pos_change: y_str += str(val.detach().numpy()[0]) + " " df.loc[len(df)] = [ int(bacteria_num), int(bacteria_to_change_num), "plus " + str(CHANGE), y_str ] X_negative_change = copy.deepcopy(X_train) for s_i, sample in enumerate(X_negative_change): X_negative_change[s_i][bacteria_to_change_num] -= CHANGE y_pred_neg_change = clf.predict( torch.FloatTensor(X_negative_change)) y_str = "" for val in y_pred_neg_change: y_str += str(val.detach().numpy()[0]) + " " df.loc[len(df)] = [ int(bacteria_num), int(bacteria_to_change_num), "minus " + str(CHANGE), y_str ] pos_u, pos_u_test_p_val = mannwhitneyu( y_pred_no_change.detach().numpy(), y_pred_pos_change.detach().numpy()) neg_u, neg_u_test_p_val = mannwhitneyu( y_pred_no_change.detach().numpy(), y_pred_neg_change.detach().numpy()) if pos_u_test_p_val < p_value and neg_u_test_p_val < p_value: train_binary_significant_for_b_i.append(1) else: train_binary_significant_for_b_i.append(0) # TEST # run the model one time with no change, then save it NUMBER_OF_SAMPLES = X_test.shape[0] NUMBER_OF_BACTERIA = X_test.shape[1] NUMBER_OF_TIME_POINTS = None missing_values = np.array([1 for j in range(NUMBER_OF_SAMPLES)]) res_map = run_NN(X_test, y_test, missing_values, params, name, folder, NUMBER_OF_SAMPLES, NUMBER_OF_TIME_POINTS, NUMBER_OF_BACTERIA, save_model=True, GPU_flag=GPU, k_fold=k_fold, task_id="base_" + str(b_i) + "_model", person_indexes=None) model_path = os.path.join( folder, "trained_models", params.__str__().replace(" ", "").replace("'", "") + "_base_" + str(b_i) + "_model_model") out_dim = 1 if len( y_test.shape) == 1 else y_test.shape[1] # else NUMBER_OF_BACTERIA structure = params["STRUCTURE"] layer_num = int(structure[0:3]) hid_dim_1 = int(structure[4:7]) hid_dim_2 = int(structure[8:11]) if len(structure) > 10 else None clf_params = { "NN_input_dim": X_test.shape[1], "NN_hidden_dim_1": hid_dim_1, "NN_output_dim": out_dim } clf = bacteria_network_clf(clf_params) clf.load(model_path) y_pred_no_change = clf.predict(torch.FloatTensor(X_test)) # ------------------------------------ send to network ------------------------------------ for bacteria_to_change_num in bacteria_number_list: # change each bacteria # change X, y for only bacteria_to_change_num X_positive_change = copy.deepcopy(X_test) for s_i, sample in enumerate(X_positive_change): X_positive_change[s_i][bacteria_to_change_num] += CHANGE y_pred_pos_change = clf.predict( torch.FloatTensor(X_positive_change)) X_negative_change = copy.deepcopy(X_test) for s_i, sample in enumerate(X_negative_change): X_negative_change[s_i][bacteria_to_change_num] -= CHANGE y_pred_neg_change = clf.predict( torch.FloatTensor(X_negative_change)) pos_u, pos_u_test_p_val = mannwhitneyu( y_pred_no_change.detach().numpy(), y_pred_pos_change.detach().numpy()) neg_u, neg_u_test_p_val = mannwhitneyu( y_pred_no_change.detach().numpy(), y_pred_neg_change.detach().numpy()) test_1_u_score_for_b_i.append((1 / pos_u, 1 / neg_u)) # save bacteria b_i results df.to_csv(df_title, index=False) train_binary_significant_df.loc[len( train_binary_significant_df)] = train_binary_significant_for_b_i test_b_df.loc[len(test_b_df)] = test_1_u_score_for_b_i # calculate AUC on the flatten data frame # positive change tuple[0] pos_b = [] neg_b = [] for row in test_b_df.values: for val in row: pos_b.append(float(val[0])) neg_b.append(float(val[1])) pos_b = np.array(pos_b) neg_b = np.array(neg_b) train_binary_significant_values = [] for val in np.array(train_binary_significant_df.values).flatten(): train_binary_significant_values.append(val) train_binary_significant_values = np.array(train_binary_significant_values) try: pos_auc = roc_auc_score(train_binary_significant_values, pos_b) neg_auc = roc_auc_score(train_binary_significant_values, neg_b) Networks_AUC_df = pd.read_csv("all_Networks_AUC.csv") Networks_AUC_df.loc[len(Networks_AUC_df)] = [ "positive change", "neural network " + params.__str__(), data_set_name, test_size, k_fold, pos_auc, datetime.utcnow().strftime("%d/%m/%Y %H:%M:%S") ] Networks_AUC_df.loc[len(Networks_AUC_df)] = [ "negative change", "neural network " + params.__str__(), data_set_name, test_size, k_fold, neg_auc, datetime.utcnow().strftime("%d/%m/%Y %H:%M:%S") ] Networks_AUC_df.to_csv("all_Networks_AUC.csv", index=False) except: print( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ) print(len(train_binary_significant_values)) print(set(train_binary_significant_values)) print(len(pos_b)) print(len(set(pos_b))) print(len(neg_b)) print(len(set(neg_b))) print( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" )
Run main.py to start. This script is modified from PyTorch quickstart: https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html """ import nni import torch from torch import nn from torch.utils.data import DataLoader from torchvision import datasets from torchvision.transforms import ToTensor # Get optimized hyperparameters params = {'features': 512, 'lr': 0.001, 'momentum': 0} optimized_params = nni.get_next_parameter() params.update(optimized_params) # Load dataset training_data = datasets.FashionMNIST(root='data', train=True, download=True, transform=ToTensor()) test_data = datasets.FashionMNIST(root='data', train=False, download=True, transform=ToTensor()) train_dataloader = DataLoader(training_data, batch_size=64) test_dataloader = DataLoader(test_data, batch_size=64) # Build model
import torch import torch.nn as nn import torch.nn.functional as F from operations import ReLUConvBN, OPS, FactorizedReduce from torch.autograd import Variable # from genotypes import PRIMITIVES import nni from genotypes import Genotype TUNER_PARAMS = nni.get_next_parameter() PRIMITIVES = TUNER_PARAMS["primitives"] class MixedOp(nn.Module): def __init__(self, C, stride): super(MixedOp, self).__init__() self._ops = nn.ModuleList() for primitive in PRIMITIVES: op = OPS[primitive](C, stride, False) if 'pool' in primitive: op = nn.Sequential(op, nn.BatchNorm2d(C, affine=False)) self._ops.append(op) def forward(self, x, weights): return sum(w * op(x) for w, op in zip(weights, self._ops)) class Cell(nn.Module): def __init__(self, steps, multiplier, C_prev_prev, C_prev, C, reduction, reduction_prev): super(Cell, self).__init__()
def test_get_next_parameter(self): self.assertEqual(nni.get_next_parameter(), self._trial_params)
def main_compute(code_only=False): tvm_target = 'cuda' tvm.register_func('tvm_callback_cuda_compile', compile_source, override=True) logging.getLogger('autotvm').setLevel(logging.DEBUG) logging.getLogger('autotvm').addHandler(logging.StreamHandler(sys.stdout)) default_tune_op = importlib.import_module('templates.' + ( os.environ['OP'] if 'OP' in os.environ else 'auto.generic')) print(' >> Backend = %s, Python PID = %s, Task = %s;' % (backend, os.getpid(), default_tune_op.__name__)) task = autotvm.task.create("template_op", args=(), target=tvm_target) def json_to_config(json_dict, index=-1, code_hash=None): if not isinstance(json_dict, list): json_list = [] for key in json_dict: json_list.append([ key, 'ot' if type(json_dict[key]) is not list else ('sp' if json_dict[key][0:1] == [-1] else 're'), json_dict[key] ]) json_dict = json_list config = ConfigEntity.from_json_dict({ "index": index, "time": "", "code_hash": code_hash, "entity": json_dict }) # config = ConfigEntity.from_json_dict({"i": index, "t": "", "c": code_hash, "e": json_dict}) return config def config_to_json(config): if config is None: return {} if isinstance(config, str): return json.loads(config) jobj = config.to_json_dict()['entity'] # jobj = config.to_json_dict()['e'] json_dict = dict() for i in range(len(jobj)): assert (jobj[i][1] in ['sp', 'ot', 're']) json_dict[jobj[i][0]] = jobj[i][2] return json_dict num_trials = int(os.environ['STEP']) if 'STEP' in os.environ else 0 config = os.environ.get('CONFIG', '').strip() if config != '': if config[0] != '[': params_given = json.loads(config) print("====>> [Current Config Option]", config) best_config = json_to_config(params_given) else: best_config = config elif 'NNI_TRIAL_JOB_ID' in os.environ: if os.environ['NNI_TRIAL_JOB_ID'] == '@': search_space = get_search_space(task.config_space) json_space = json.dumps(search_space) dump_to_file = './search_space.json' print("\n>> Writing Search Space to '%s', Search Space = %s;" % (dump_to_file, json_space)) with open("search_space.json", "w") as fp: fp.write(json_space) sys.exit(0) try: import nni params_given = nni.get_next_parameter() if params_given is None: raise local_dir_id = os.environ['NNI_TRIAL_JOB_ID'] except: params_given = default_tune_op.get_choice_example() local_dir_id = '_' t = run_config_entity(params_given, local_dir_id) gflops = compute_gflops(task.flop, t) print('[Antares-engine] Final entity result is: %g' % gflops) try: nni.report_final_result(gflops) except: print('[Antares-engine] (not reporting final result to NNI.)') exit(0) elif num_trials > 0: dev_num = platform_config.get_execution_parallism() if dev_num <= 0: raise Exception("No valid device found for backend: %s." % backend) batch_size = int(os.environ.get('BATCH', '16')) from concurrent.futures import ThreadPoolExecutor try: if platform_config.allow_concurrent_compile_execution(): raise Exception() worker_size = 1 except: worker_size = batch_size thread_pool = ThreadPoolExecutor(max_workers=worker_size) task.antares_helper = Mock() task.antares_helper.json_to_config = json_to_config task.antares_helper.config_to_json = config_to_json task.antares_helper.to_json_search_space = get_search_space tuner_type = os.environ.get('TUNER', 'XGBoost') print(' >> MAKE_PARA = %d/%d, EXEC_PARA = %d, TUNER = %s' % (worker_size, batch_size, dev_num, tuner_type)) auto_commit = os.environ.get('COMMIT', '') if auto_commit: saved_code = codehub_db(os.environ['COMPUTE_V1']) if saved_code is not None and auto_commit != 'force': raise Exception( "Saved code has existed in codehub. Please try COMMIT=force to overide it." ) os.environ.pop('COMMIT') try: tuner = importlib.import_module('tuner.%s.main' % tuner_type) tuner = tuner.MainTuner(task) except: raise Exception('>> Cannot import Antares Tuner: %s' % tuner_type) if tuner is not None: def measure_batch(inputs): results, futures = [], [] best_slot = -1 expected_timecost = tuner.task.best.timecost for i in range(len(inputs)): futures.append( thread_pool.submit(run_config_entity, config_to_json(inputs[i].config), i, expected_timecost, i % dev_num)) for i in range(len(inputs)): t = futures[i].result() if t < tuner.task.best.timecost: best_slot = i tuner.task.best.timecost = t tuner.task.best.config = inputs[i].config tuner.task.best.occur = tuner.task.best.curr_step + i + 1 results.append( autotvm.measure.MeasureResult(costs=(t, ), error_no=0, all_cost=i, timestamp=time.time())) tuner.task.best.curr_step += len(results) print( '\nSTEP[%d / %d] Current Best Config = %s, Perf = %g Gflops, Occur Step = %d;' % (tuner.task.best.curr_step, num_trials, json.dumps(config_to_json(tuner.task.best.config)), compute_gflops(tuner.task.flop, tuner.task.best.timecost), tuner.task.best.occur)) if auto_commit and best_slot >= 0: with open(local_get_dir_file('my_kernel.cc', best_slot), 'r') as fp: device_source = fp.read() with open(local_get_dir_file('result.txt', best_slot), 'r') as fp: t = float(fp.read().split()[0]) kernel_path = codehub_db( os.environ['COMPUTE_V1'], source_code=device_source + '\n// Saved Perf = %g sec / run; Step Produced = %d;' % (t, tuner.task.best.curr_step)) print(' >> Update current code to codehub: %s' % kernel_path) return results tuner.task.best = Mock() tuner.task.best.timecost = float('inf') tuner.task.best.config = None tuner.task.best.occur = -1 tuner.task.best.curr_step = 0 tuner.measure_batch = measure_batch callbacks = [] history_log_for_transfer_learning = os.environ.get('RECORD', '') if history_log_for_transfer_learning: callbacks.append( autotvm.callback.log_to_file( history_log_for_transfer_learning)) # Enable Transfer Learning for Incremental Task if os.path.exists(history_log_for_transfer_learning): print( ' >> Loading incremental history from log file: %s ..' % history_log_for_transfer_learning) tuner.load_history( autotvm.record.load_from_file( history_log_for_transfer_learning)) tuner.tune(n_trial=num_trials, measure_option=autotvm.measure_option( builder=autotvm.LocalBuilder(n_parallel=batch_size), runner=autotvm.LocalRunner(repeat=3, min_repeat_ms=100, timeout=4)), callbacks=callbacks) assert not math.isinf( tuner.task.best.timecost ), "Not valid config found in the whole tuning." best_config = tuner.task.best.config print( "\n[Best Config] CONFIG='%s' ==> Performance is up to %f Gflops, occurred at step %d / %d; time per run = %g sec." % (json.dumps(config_to_json(best_config)), compute_gflops(tuner.task.flop, tuner.task.best.timecost), tuner.task.best.occur, num_trials, tuner.task.best.timecost)) if hasattr(tuner, 'cleanup'): tuner.cleanup() else: raise Exception('Unrecognized tuner type: `%s`' % tuner_type) exit(0) else: if os.environ['OP'] == 'auto.generic': saved_code = codehub_db(os.environ['COMPUTE_V1']) if saved_code is not None: print(" >> Using Saved Code from Codehub:") print("===========================") print(saved_code) print("===========================") exit(0) best_config = task.config_space if isinstance(best_config, str): from tvm import auto_scheduler origin_cfg = json.loads(best_config) origin_cfg = { "i": [[ '["main_compute.<locals>.auto_template"]', 'cuda -keys=cuda,gpu -max_num_threads=%d -thread_warp_size=%d' % (device_properties().max_threads_per_block, device_properties().warp_size) ], origin_cfg], "r": [[0], 0, 0, 0], "v": "v0.2", } origin_cfg_file = local_get_dir_file('my_kernel.cfg') with open(origin_cfg_file, 'w') as fp: fp.write(json.dumps(origin_cfg)) origin_cfg = tvm.auto_scheduler.measure_record.load_records( origin_cfg_file) @auto_scheduler.register_workload def auto_template(): _, arg_bufs = default_tune_op.get_template_op() return arg_bufs target = tvm.target.Target("cuda") auto_task = auto_scheduler.create_task(auto_template, (), target) for inp, res in origin_cfg: s, arg_bufs = auto_task.compute_dag.apply_steps_from_state( inp.state) break else: with ApplyConfig(best_config): with tvm.target.Target(tvm_target): s, arg_bufs = default_tune_op.get_template_op() if s is not None: lower_source = str(tvm.lower(s, arg_bufs, simple_mode=True)) lower_file = local_get_dir_file('my_kernel.lower') with open(lower_file, 'w') as fp: fp.write(lower_source) # Verify Lower Code Code if len(('\n' + lower_source).split('\nprimfn(')) != 2: raise Exception('[Not Support Multi Unfuse-able kernels]\n\n' + lower_source) max_threads_per_block = device_properties().max_threads_per_block max_shared_memory_per_block = device_properties( ).max_shared_memory_per_block assert max_threads_per_block > 0 and max_shared_memory_per_block >= 0, '[Error] Invalid device properties, maybe device is not detected correctly.' lower_lines = lower_source.split('\n') thread_extents, allocate_shared = [], [] for ll in lower_lines: if ll.strip().startswith( 'attr [IterVar(') and ll.find(' "thread_extent" = ') >= 0: thread_name = ll.split('attr [IterVar(')[-1].split(':')[0] thread_val = int( ll.split(' "thread_extent" = ')[-1].split(';') [0].strip().split(' ')[0]) thread_extents.append((thread_name, thread_val)) elif ll.strip().startswith('allocate(') and ll.find( '.shared, ') >= 0 and ll.endswith(");"): parts = ll[:-2].split(', ')[1:] allocate_type = parts[0] allocate_val = int(np.product(eval(parts[1]))) allocate_shared.append((allocate_type, allocate_val)) reserved_axes = dict() for thread_name, thread_val in thread_extents: if thread_name in reserved_axes: assert reserved_axes[ thread_name] == thread_val, "Invalid code: Multiple hints for thread extent conflict with each other: %d v.s. %d" % ( reserved_axes[thread_name], thread_val) else: reserved_axes[thread_name] = thread_val num_threads = 1 for thread_name in ['threadIdx.x', 'threadIdx.y', 'threadIdx.z']: num_threads *= reserved_axes.get(thread_name, 1) assert num_threads <= max_threads_per_block, "Invalid kernel code: using num_threads(%d) > max_threads_per_block(%d)" % ( num_threads, max_threads_per_block) shared_memory_in_bytes = 0 for allocate_type, allocate_size in allocate_shared: if allocate_type.startswith('custom['): type_name = allocate_type[7:].split(']')[0] shared_memory_inc = int( custom_dtypes[type_name][-1].split('@')[-1]) else: shared_memory_inc = 8 * np.dtype(allocate_type).itemsize assert shared_memory_inc % 8 == 0, "The bits of shared_memory is not aligned with 8-bit bytes." shared_memory_in_bytes += shared_memory_inc // 8 * allocate_size if shared_memory_in_bytes > max_shared_memory_per_block: raise Exception( "Invalid kernel code: using shared_memory_in_bytes %d > max_shared_memory_per_block %d" % (shared_memory_in_bytes, max_shared_memory_per_block)) # Compile Source Code def build_template(): return tvm.build(s, arg_bufs, tvm_target, name='template_op') func = wait_for(build_template, 30) assert (len(func.imported_modules) == 1) device_source = translate_code(func.imported_modules[0].get_source()) if code_only: return device_source print("====================================") print(device_source) print("====================================") print() try: eval_client = importlib.import_module('platforms.%s.evaluator.client' % backend) except ModuleNotFoundError: print('>> Evaluator for backend %s not found, skipping evaluation.' % backend) exit(0) except: traceback.print_exc() exit(1) def handle_result(result): print('\n[EvalAgent] Results =', json.dumps(result)) if 'RESULT' in os.environ: if abs(float(os.environ['RESULT']) / result['K/0'] - 1.0) > 1e-6: result['TPR'] = None t = result.get('TPR', None) if t is None: print("\n[Antares] Incorrect compute kernel from evaluator.") else: gflops = compute_gflops(task.flop, t) print("\n[Antares] Average time cost / run = %g sec, %g gflops." % (t, gflops)) with open(local_get_dir_file('result.txt'), 'w') as fp: fp.write(str(t) + '\n') if 'K/0' in result: fp.write(str(result['K/0']) + '\n') if os.environ['OP'] == 'auto.generic' and os.environ.get('COMMIT', ''): kernel_path = codehub_db(os.environ['COMPUTE_V1'], source_code=device_source + '\n// Saved Perf = %g sec / run' % t) print(' >> Update current code to codehub: %s' % kernel_path) tune_slot_id = int(os.environ.get(unified_slot_key, '0')) exec_fd, _ = system_lock([tune_slot_id]) try: expected_timeout = None if 'EXPECTED_TIMEOUT' in os.environ and not math.isinf( float(os.environ['EXPECTED_TIMEOUT'])): expected_timeout = float(os.environ['EXPECTED_TIMEOUT']) expected_timeout = max(expected_timeout * 1.1, expected_timeout + 0.1) results = eval_client.eval( kernel_path=local_get_dir_file('my_kernel.cc'), expected_timeout=expected_timeout, func=func, ) except: traceback.print_exc() exit(1) handle_result(results) exec_fd() exit(0)
def main(): global NNI _params = { "print_every_num_of_epochs": 100, "data_folder_name": "dnc", "data_name": "dnc_candidate_two", "graphs_cutoff_number": 2, "l1_lambda": 0, "epochs": 500, "gcn_dropout_rate": 0.7, "lstm_dropout_rate": 0, "gcn_hidden_sizes": [10, 10, 10, 10, 10, 10, 10, 10, 10], "learning_rate": 0.001, "weight_decay": 0, "gcn_latent_dim": 5, "lstm_hidden_size": 10, "lstm_num_layers": 1, "learned_label": DEFAULT_LABEL_TO_LEARN, } l1_lambda = [0, 1e-7] epochs = [500] gcn_dropout_rate = [0.3, 0.5] gcn_hidden_sizes = [[100, 100], [200, 200]] learning_rate = [1e-3, 1e-2, 3e-2] weight_decay = [5e-2, 1e-2] gcn_latent_dim = [50, 100] lstm_hidden_size = [50, 100] results = [] argparser = argparse.ArgumentParser() argparser.add_argument("--nni", action='store_true') args = argparser.parse_args() NNI = args.nni if NNI: p = nni.get_next_parameter() p["gcn_hidden_sizes"] = ast.literal_eval(p["gcn_hidden_sizes"]) _params.update(p) (model_loss, model_accuracy, model_tot_accuracy, model_correlation, model_tot_correlation, zero_model_tot_loss, zero_model_tot_accuracy, zero_model_tot_correlation, first_order_tot_loss, first_order_tot_accuracy, first_order_tot_correlation, zero_model_diff_loss, zero_model_diff_accuracy, zero_model_diff_correlation, first_order_diff_loss, first_order_diff_accuracy, first_order_diff_correlation) = run_trial(_params) if NNI: nni.report_final_result(model_tot_accuracy[0] - zero_model_tot_accuracy[-1]) else: print( f"Final result (accruacy): model tot accuracy: {model_tot_accuracy}, zero_model_tot_accuracy: {zero_model_tot_accuracy}, " f"first order tot accuracy: {first_order_tot_accuracy}, zero model diff accuracy: {zero_model_diff_accuracy}, " f"first order diff accuracy: {first_order_diff_accuracy}") print( f"Final result (correlation): model tot correlation: {model_tot_correlation}, zero_model_tot_correlation: {zero_model_tot_correlation}, " f"first order tot correlation: {first_order_tot_correlation}, zero model diff correlation: {zero_model_diff_correlation}, " f"first order diff correlation: {first_order_diff_correlation}") return (model_tot_accuracy, model_tot_correlation, zero_model_tot_accuracy, zero_model_tot_correlation, first_order_tot_accuracy, first_order_tot_correlation, zero_model_diff_accuracy, zero_model_diff_correlation, first_order_diff_accuracy, first_order_diff_correlation)
beta_2=params["beta_2"], decay=params["decay"]) elif params["optimizer"] == 'Nadam': optimizer = Nadam(lr=params["lr"], beta_1=params["beta_1"], beta_2=params["beta_2"], schedule_decay=params["schedule_decay"]) model.compile(optimizer=optimizer, loss=params['loss'], metrics=['accuracy']) hist = model.fit(x_train, y_train, batch_size=params['batch_size'], epochs=params['epochs'], validation_split=0.2, shuffle=True, callbacks=[SendMetrics()]) score = model.evaluate(x_test, y_test, batch_size=params['batch_size']) model.save('m.h5') nni.report_final_result(score[1]) return hist.history, score if __name__ == '__main__': received_params = nni.get_next_parameter() params = generate_default_params() params.update(received_params) hist, score = train() print('Test loss:', score[0]) print('Test accuracy:', score[1])
def main_compute(code_only=False): tvm.register_func('tvm_callback_cuda_compile', compile_source, override=True) logging.getLogger('autotvm').setLevel(logging.DEBUG) logging.getLogger('autotvm').addHandler(logging.StreamHandler(sys.stdout)) default_tune_op = importlib.import_module('templates.' + (os.environ['OP'] if 'OP' in os.environ else 'auto.generic')) if verbose: print(' >> Backend = %s, Python PID = %s, Task = %s;' % (backend, os.getpid(), default_tune_op.__name__)) task = autotvm.task.create("template_op", args=(), target=tvm_target) def json_to_config(json_dict, index=-1, code_hash=None): if not isinstance(json_dict, list): json_list = [] for key in json_dict: json_list.append([key, 'ot' if type(json_dict[key]) is not list else ('sp' if json_dict[key][0:1] == [-1] else 're'), json_dict[key]]) json_dict = json_list config = ConfigEntity.from_json_dict({"index": index, "time": "", "code_hash": code_hash, "entity": json_dict}) # config = ConfigEntity.from_json_dict({"i": index, "t": "", "c": code_hash, "e": json_dict}) return config def config_to_json(config): if config is None: return {} if isinstance(config, str): return json.loads(config) jobj = config.to_json_dict()['entity'] # jobj = config.to_json_dict()['e'] json_dict = dict() for i in range(len(jobj)): assert(jobj[i][1] in ['sp', 'ot', 're']) json_dict[jobj[i][0]] = jobj[i][2] return json_dict num_trials = int(os.environ['STEP']) if 'STEP' in os.environ else 0 config = os.environ.get('CONFIG', '').strip() if config != '': best_config = config elif 'NNI_TRIAL_JOB_ID' in os.environ: if os.environ['NNI_TRIAL_JOB_ID'] == '@': search_space = get_search_space(task.config_space) json_space = json.dumps(search_space) dump_to_file='./search_space.json' print("\n>> Writing Search Space to '%s', Search Space = %s;" % (dump_to_file, json_space)) with open("search_space.json", "w") as fp: fp.write(json_space) sys.exit(0) try: import nni params_given = nni.get_next_parameter() if params_given is None: raise local_dir_id = os.environ['NNI_TRIAL_JOB_ID'] except: params_given = default_tune_op.get_choice_example() local_dir_id = '_' t = run_config_entity(params_given, local_dir_id) gflops = compute_gflops(task.flop, t) print('[Antares-engine] Final entity result is: %g' % gflops) try: nni.report_final_result(gflops) except: print('[Antares-engine] (not reporting final result to NNI.)') exit(0) elif num_trials > 0: dev_num = platform_config.get_execution_parallism() if dev_num <= 0: raise Exception("No valid device found for backend: %s." % backend) batch_size = int(os.environ.get('BATCH', '16')) from concurrent.futures import ThreadPoolExecutor try: if platform_config.allow_concurrent_compile_execution(): raise Exception() worker_size = 1 except: worker_size = batch_size thread_pool = ThreadPoolExecutor(max_workers=worker_size) task.antares_helper = Mock() task.antares_helper.json_to_config = json_to_config task.antares_helper.config_to_json = config_to_json task.antares_helper.to_json_search_space = get_search_space tuner_type = os.environ.get('TUNER', '') if not tuner_type: comp = os.environ['COMPUTE_V1'] if '=!' in comp and 'plan/' not in comp[comp.find(' ##') + 1:] and ';' not in comp and backend in ['c-rocm', 'c-cuda', 'c-hlsl', 'c-ocl']: tuner_type = 'AutoTVM2' else: tuner_type = 'XGBoost' print(' >> MAKE_PARA = %d/%d, EXEC_PARA = %d, TUNER = %s' % (worker_size, batch_size, dev_num, tuner_type)) auto_commit = os.environ.get('COMMIT', '') if auto_commit: saved_code = codehub_db(os.environ['COMPUTE_V1']) if saved_code is not None and auto_commit != 'force': raise Exception("Saved code has existed in codehub. Please try COMMIT=force to override it.") os.environ.pop('COMMIT') try: tuner = importlib.import_module('tuner.%s.main' % tuner_type) tuner = tuner.MainTuner(task) except: raise Exception('>> Cannot import Antares Tuner: %s' % tuner_type) if tuner is not None: AntaresGlobal.current_step = 0 def measure_batch(inputs): results, futures = [], [] best_slot = -1 expected_timecost = tuner.task.best.timecost for i in range(len(inputs)): futures.append(thread_pool.submit(run_config_entity, config_to_json(inputs[i].config), AntaresGlobal.current_step + i + 1, expected_timecost, i % dev_num)) for i in range(len(inputs)): t = futures[i].result() if t < tuner.task.best.timecost: best_slot = AntaresGlobal.current_step + i + 1 tuner.task.best.timecost = t tuner.task.best.config = inputs[i].config tuner.task.best.occur = best_slot results.append(autotvm.measure.MeasureResult(costs=(t,), error_no=0, all_cost=i, timestamp=time.time())) AntaresGlobal.current_step += len(results) print('\nSTEP[%d / %d] Current Best Config = %s, Perf = %g Gflops, MemRatio = %g %%, Occur Step = %d;' % ( AntaresGlobal.current_step, num_trials, json.dumps(config_to_json(tuner.task.best.config)), compute_gflops(tuner.task.flop, tuner.task.best.timecost), compute_mem_ratio(tuner.task.best.timecost), tuner.task.best.occur)) if auto_commit and best_slot >= 0: with open(local_get_dir_file('my_kernel.cc', best_slot), 'r') as fp: device_source = fp.read() with open(local_get_dir_file('result.txt', best_slot), 'r') as fp: t = float(fp.read().split()[0]) kernel_path = codehub_db(os.environ['COMPUTE_V1'], source_code=device_source + '\n// Saved Perf = %g sec / run; Step Produced = %d;' % (t, best_slot)) print(' >> Update current code to codehub: %s' % kernel_path) return results tuner.task.best = Mock() tuner.task.best.timecost = float('inf') tuner.task.best.config = None tuner.task.best.occur = -1 tuner.measure_batch = measure_batch tuner.measure_batch.n_parallel = batch_size callbacks = [] history_log_for_transfer_learning = os.environ.get('RECORD', '') if history_log_for_transfer_learning: callbacks.append(autotvm.callback.log_to_file(history_log_for_transfer_learning)) # Enable Transfer Learning for Incremental Task if os.path.exists(history_log_for_transfer_learning): print(' >> Loading incremental history from log file: %s ..' % history_log_for_transfer_learning) tuner.load_history(autotvm.record.load_from_file(history_log_for_transfer_learning)) tuner.tune(n_trial=num_trials, measure_option=autotvm.measure_option( builder=autotvm.LocalBuilder(n_parallel=batch_size), runner=autotvm.LocalRunner(repeat=3, min_repeat_ms=100, timeout=4) ), callbacks=callbacks) assert not math.isinf(tuner.task.best.timecost), "Not valid config found in the whole tuning." best_config = json.dumps(config_to_json(tuner.task.best.config)) if auto_commit: device_source = codehub_db(os.environ['COMPUTE_V1']) codehub_db(os.environ['COMPUTE_V1'], source_code=device_source + '\n// Antares Tuning Completed in %d steps.' % AntaresGlobal.current_step) print("\n[Best Config] CONFIG='%s' ==> Performance is up to %f Gflops, occurred at step %d / %d; time per run = %g sec." % ( best_config, compute_gflops(tuner.task.flop, tuner.task.best.timecost), tuner.task.best.occur, num_trials, tuner.task.best.timecost)) if hasattr(tuner, 'cleanup'): tuner.cleanup() else: raise Exception('Unrecognized tuner type: `%s`' % tuner_type) exit(0) else: if os.environ['OP'] == 'auto.generic': saved_code = codehub_db(os.environ['COMPUTE_V1']) if saved_code is not None: print(" >> Using Saved Code from Codehub:") print("===========================") print(saved_code) print("===========================") exit(0) best_config = '' assert isinstance(best_config, str) if verbose: print("====>> [Current Config Option]", best_config) if best_config.startswith('['): from tvm import auto_scheduler origin_cfg = json.loads(best_config) origin_cfg = { "i": [['["main_compute.<locals>.auto_template"]', 'cuda -keys=cuda,gpu -max_num_threads=%d -thread_warp_size=%d' % ( device_properties().max_threads_per_block, device_properties().warp_size )], origin_cfg], "r": [[0], 0, 0, 0], "v": "v0.2", } origin_cfg_file = local_get_dir_file('my_kernel.cfg') with open(origin_cfg_file, 'w') as fp: fp.write(json.dumps(origin_cfg)) origin_cfg = tvm.auto_scheduler.measure_record.load_records(origin_cfg_file) @auto_scheduler.register_workload def auto_template(): _, arg_bufs = default_tune_op.get_template_op() return arg_bufs target = tvm.target.Target("cuda") auto_task = auto_scheduler.create_task(auto_template, (), target) for inp, res in origin_cfg: s, arg_bufs = auto_task.compute_dag.apply_steps_from_state(inp.state) break else: config = json_to_config(json.loads(best_config)) if best_config else task.config_space with ApplyConfig(config): with tvm.target.Target(tvm_target): s, arg_bufs = default_tune_op.get_template_op() device_source, kernel_path = get_target_source(s, arg_bufs) if code_only: return device_source if verbose: print("====================================") print(device_source) print("====================================\n") dev_id = int(os.environ.get('DEV_KEY', '0')) result = evaluate_perf(kernel_path, task.flop, dev_id) exit(0 if result is not None else 1)
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '3' try: RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) args = get_args() print(PARAMS['learning_rate']) config = process_config_UtsClassification_bayes_optimization( args.config, PARAMS['learning_rate'], PARAMS['model_name'], PARAMS['batch_size']) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([ config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir ]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() nni.report_final_result(evaluater.f1) print('done') except Exception as e: LOG.exception(e) raise
import nni import time if __name__ == '__main__': nni.get_next_parameter() time.sleep(3) nni.report_final_result(0.5)
parser.add_argument('-f', '--load', dest='load', type=str, default=False, help='Load model from a .pth file') return parser.parse_args() if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') args = get_args() #使用nni的get_next_parameter()获取一组超参组合 nni_args = nni.get_next_parameter() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logging.info(f'Using device {device}') #初始化网络,使用nni_args中获取的参数:其中的dropout_rate与hidden_size用于网络参数调整 net = Net(dropout_rate=nni_args["dropout_rate"],hidden_size=nni_args["hidden_size"]) if args.load: net.load_state_dict(torch.load(args.load, map_location=device)) logging.info(f'Model loaded from {args.load}') net.to(device=device) #重写训练网络部分,在nni框架支持下,使用nni_args中获取的batch_size与lr try: train_net(net=net, epochs=args.epochs,
import time import nni if __name__ == '__main__': print('trial start') params = nni.get_next_parameter() print('params:', params) epochs = 2 for i in range(epochs): nni.report_intermediate_result(0.1 * (i + 1)) time.sleep(1) nni.report_final_result(0.8) print('trial done')
"""#Training Arguments""" datadir = './data' data_name = 'cifar10' fraction = float(0.1) num_epochs = int(300) select_every = int(20) feature = 'dss' num_runs = 1 # number of random runs model_name = 'ResNet164' device = "cuda" if torch.cuda.is_available() else "cpu" strategy = "GLISTER" print("Using Device:", device) config = nni.get_next_parameter() """#Training Runs""" for run in range(num_runs): train_model(num_epochs, data_name, datadir, feature, config['model'], fraction, select_every, config['lr'], run, device, 'GradMatch-Explore') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, # 'GradMatchPB-Explore') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Random') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GLISTER') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GradMatch') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GradMatchPB') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Random-Online') # train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Full')
parser.add_argument('--reg_qi', type=float, dest='reg_qi', default=None) args = parser.parse_args() return args def main(params): logger.debug("Args: %s", str(params)) logger.debug('Number of epochs %d', params["n_epochs"]) svd = svd_training(params) # Save SVD model to the output directory for later use output_dir = os.environ.get('NNI_OUTPUT_DIR') surprise.dump.dump(os.path.join(output_dir, 'model.dump'), algo=svd) if __name__ == "__main__": try: tuner_params = nni.get_next_parameter() logger.debug("Hyperparameters: %s", tuner_params) params = vars(get_params()) # in the case of Hyperband, use STEPS to allocate the number of epochs SVD will run for if 'STEPS' in tuner_params: steps_param = tuner_params['STEPS'] params['n_epochs'] = int(np.rint(steps_param)) params.update(tuner_params) main(params) except Exception as exception: logger.exception(exception) raise
import yaml parser = argparse.ArgumentParser(description='Launch PENN training') parser.add_argument('--config', default='PedSM/PENN/training/default_config.yaml', help='Configuration File Location') # TODO: these parsers below should do something but at the moment they don't parser.add_argument('--load-checkpoint', default=False, help='Start from a saved checkpoint') parser.add_argument('--verbose', '-v', action='count', default=0) args = parser.parse_args() config = yaml.safe_load(open(args.config)) try: torch.manual_seed(42) # Always set this as we want to reproduce values updated_params = nni.get_next_parameter() params = generate_default_params() params.update(updated_params) main(config=config, restore_old_checkpoint=args.load_checkpoint, verbosity=args.verbose, params=params) except Exception as exc: logging.debug('Failed to Complete Search') logging.debug(exc) raise