def prepare_hyper_search(cfg_kwargs: dict,
reporthook=None, final_reporthook=None,
primary_key=None, max_key=True, reporter_cls=None, with_keys: (list, str, None) = None,
final_keys: (list, str, None) = None,
dump=False, disable=False):
"""
Updated in v1.3.18
从 nni package 中获取超参，更新配置文件参数。当 nni 不可用或不是 nni 搜索模式时，参数将不会改变。
.. code-block :: python
cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search(
cfg_kwargs, Configuration, reporthook, final_reporthook, primary_key="macro_avg:f1"
)
_cfg = Configuration(**cfg_kwargs)
model = Model(_cfg)
...
for epoch in range(_cfg.begin_epoch, _cfg.end_epoch):
for batch_data in dataset:
train_model(batch_data)
data = evaluate_model()
reporthook(data)
final_reporthook()
Parameters
----------
cfg_kwargs: dict
待传入cfg的参数
reporthook
final_reporthook
primary_key:
评估模型用的主键,
``nni.report_intermediate_result`` 和 ``nni.report_final_result`` 中 ``metric`` 的 ``default``
max_key: bool
主键是越大越好
reporter_cls
with_keys: list or str
其它要存储的 metric，final report时默认为 primary_key 最优时指标
final_keys: list or str
with_keys 中使用最后一个 report result 而不是 primary_key 最优时指标
dump: bool
为 True 时，会修改 配置文件 中 workspace 参数为 ``workspace/nni.get_experiment_id()/nni.get_trial_id()``
使得 nni 的中间结果会被存储下来。
disable
Returns
-------
cfg_kwargs: dict
插入了nni超参后的配置文件参数
reporthook: function
每个iteration结束后的回调函数，用来报告中间结果。
默认 ``nni.report_intermediate_result``。
final_reporthook:
所有iteration结束后的回调函数，用来报告最终结果。
默认 ``nni.report_final_result``
dump: bool
和传入参数保持一致
Examples
--------
.. code-block :: python
class CFG(Configuration):
hyper_params = {"hidden_num": 100}
learning_rate = 0.001
workspace = ""
cfg_kwargs, reporthook, final_reporthook, dump = prepare_hyper_search(
{"learning_rate": 0.1}, CFG, primary_key="macro_avg:f1", with_keys="accuracy"
)
# cfg_kwargs: {'learning_rate': 0.1}
when nni start (e.g., using ``nni create --config _config.yml``),
suppose in ``_config.yml``:
.. code-block: yml
searchSpacePath: _search_space.json
and in ``_search_space.json``
.. code-block :: json
{
"hidden_num": {"_type": "choice", "_value": [500, 600, 700, 835, 900]},
}
one of the return cfg_kwargs is ``{'hyper_params': {'hidden_num': 50}, 'learning_rate': 0.1}``
"""
if disable:
return cfg_kwargs, None, None, None
try:
import nni
from nni import get_next_parameter, report_intermediate_result, report_final_result
assert primary_key is not None
def _as_key_list(_keys: (list, str, None)):
if isinstance(_keys, str):
if ";" in _keys:
_keys = _keys.split(";")
else:
_keys = [_keys]
elif isinstance(_keys, list):
pass
elif _keys is None:
_keys = []
return _keys
with_keys = _as_key_list(with_keys)
final_keys = _as_key_list(final_keys)
class Reporter(BaseReporter):
def __init__(self):
self.datas = []
def intermediate(self, data):
feed_dict = {
'default': float(get_by_key(data, key_parser(primary_key))),
primary_key: get_by_key(data, key_parser(primary_key))
}
for key in with_keys:
feed_dict[key] = get_by_key(data, key_parser(key))
report_intermediate_result(feed_dict)
self.datas.append(data)
def final(self):
best_fn = get_min if max_key is False else get_max
_with_keys = (with_keys if with_keys else []) + [primary_key]
_final_keys = set(final_keys if final_keys else [])
final_result = best_fn(
self.datas, primary_key, with_keys=";".join(_with_keys), merge=False
)
feed_dict = {
'default': float(final_result[0][primary_key])
}
appendix_dict = dict(final_result[1][primary_key])
for key in _with_keys:
if key in _final_keys:
feed_dict[key] = get_by_key(self.datas[-1], key_parser(key))
else:
feed_dict[key] = appendix_dict[key]
report_final_result(feed_dict)
rc = Reporter() if reporter_cls is None else reporter_cls
reporthook = reporthook if reporthook is not None else rc.intermediate
final_reporthook = final_reporthook if final_reporthook is not None else rc.final
cfg_cls_params = get_params(get_next_parameter())
using_nni_tag = True if cfg_cls_params else False
nested_update(cfg_kwargs, cfg_cls_params)
if using_nni_tag is True and dump is True: # pragma: no cover
cfg_kwargs["workspace"] = cfg_kwargs.get("workspace", "") + path_append(
nni.get_experiment_id(), nni.get_trial_id(), to_str=True
)
return cfg_kwargs, reporthook, final_reporthook, dump
except ModuleNotFoundError: # pragma: no cover
warnings.warn("nni package not found, skip")
return cfg_kwargs, reporthook, final_reporthook, dump
dev_file = os.path.expanduser(args.dev_file)
max_epoch = args.max_epoch
cfg = GAGConfig()
cfg.batch_size = args.batch_size
cfg.learning_rate = float(args.learning_rate)
cfg.dropout = args.dropout_rate
cfg.rnn_units = args.rnn_units
cfg.labelsmoothing = args.labelsmoothing
cfg.num_heads = args.num_heads
timer = Timer()
qp_pairs, dev_qp_pairs = load_data()
logger.debug('Init finish.')
original_params = nni.get_next_parameter()
'''
with open('data.json') as f:
original_params = json.load(f)
'''
p_graph = graph.graph_loads(original_params['graph'])
save_path = original_params['save_dir']
os.makedirs(save_path)
restore_path = original_params['restore_dir']
restore_shared = [
hash_id + '/' for hash_id in original_params['shared_id']
] if original_params['shared_id'] is not None else [] + [
'word_embed', 'char_embed', 'char_encoding/'
]
train_loss, best_acc = train_with_graph(p_graph, qp_pairs,
dev_qp_pairs)
import nni
parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
parser.add_argument('--bs', default=128, type=int, help='batch size')
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
parser.add_argument('--epochs', default=30, type=int, help='number of epochs')
parser.add_argument('--model', default='ResNet18', type=str, help='model')
parser.add_argument('--autoscale-bsz',
dest='autoscale_bsz',
default=True,
action='store_true',
help='autoscale batchsize')
args = parser.parse_args()
# load the parameters from nni
RCV_CONFIG = nni.get_next_parameter()
args.lr = RCV_CONFIG["lr"]
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
def generate_default_params():
'''
Generate default hyper parameters
'''
return {
'optimizer': 'Adam',
'learning_rate': 0.001
}
if __name__ == '__main__':
PARSER = argparse.ArgumentParser()
PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False)
PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False)
PARSER.add_argument("--num_train", type=int, default=60000, help="Number of train samples to be used, maximum 60000", required=False)
PARSER.add_argument("--num_test", type=int, default=10000, help="Number of test samples to be used, maximum 10000", required=False)
ARGS, UNKNOWN = PARSER.parse_known_args()
try:
# get parameters from tuner
# RECEIVED_PARAMS = {"optimizer": "Adam", "learning_rate": 0.00001}
RECEIVED_PARAMS = nni.get_next_parameter()
LOG.debug(RECEIVED_PARAMS)
PARAMS = generate_default_params()
PARAMS.update(RECEIVED_PARAMS)
# train
train(ARGS, PARAMS)
except Exception as e:
LOG.exception(e)
raise
example_start_time = time.time()
net = None
args = get_args()
try:
experiment_path = os.environ[
"HOME"] + "/mountdir/nni/experiments/" + str(
nni.get_experiment_id())
lock = multiprocessing.Lock()
context = zmq.Context()
socket = context.socket(zmq.REQ)
tmpstr = 'tcp://' + args.ip + ':800081'
socket.connect(tmpstr)
os.makedirs(experiment_path + "/trials/" + str(nni.get_trial_id()))
get_next_parameter_start = time.time()
nni.get_next_parameter(socket)
get_next_parameter_end = time.time()
while True:
lock.acquire()
with open(experiment_path + "/graph.txt", "a+") as f:
f.seek(0)
lines = f.readlines()
lock.release()
if lines:
break
if len(lines) > args.slave:
x = random.randint(1, args.slave)
json_and_id_str = lines[-x].replace("\n", "")
else:
def test_get_current_parameter(self):
nni.get_next_parameter()
self.assertEqual(nni.get_current_parameter('x'), 123)
def main():
hp_dict = nni.get_next_parameter() # hp_dict is automatically suggested
_, test_acc = tunable_train(hp_dict)
nni.report_final_result(test_acc) # report final accuracy
train_dataset=train_dataset,
eval_dataset=eval_dataset,
compute_metrics=compute_metrics,
tb_writer=tb_writer)
# Training
if training_args.do_train:
trainer.train(model_path=model_args.model_name_or_path if os.path.
isdir(model_args.model_name_or_path) else None)
# final eval
train_time = (datetime.today() + timedelta(hours=9)).strftime("%Y-%m-%d")
backbone = model_args.model_name_or_path.split('/')[-1]
hyperparams = ",".join([f"{k}={param[k]}" for k in param])
app_key = f"{train_time}/{backbone}/{hyperparams}"
df = trainer.get_best_model(metric='token-f1',
app_key=app_key,
app_name='sentiment-extractor') # hard-coded
# nni
nni.report_final_result(df['eval_token-f1'].max()) # best case
if __name__ == '__main__':
try:
param_trials = nni.get_next_parameter()
logger.debug(param_trials)
main(param_trials)
except Exception as exception:
logger.exception(exception)
raise
def main(argv):
# basepath
base_path = FLAGS.basepath
if not os.path.exists(base_path):
os.mkdir(base_path)
# model path
save_path = os.path.join(base_path, '%s_model' % str(time.time()))
os.mkdir(save_path)
params = nni.get_next_parameter()
source_embedding_dim = params['encoder_hidden_dim'] * 2
encoder_hidden_dim = params['encoder_hidden_dim']
encoder_input_dropout = params['encoder_input_dropout']
encoder_output_dropout = params['encoder_output_dropout']
decoder_hidden_dim = params['encoder_hidden_dim'] * 2
decoder_num_layers = params['decoder_num_layers']
rule_embedding_dim = params['rule_embedding_dim']
nonterminal_embedding_dim = params['nonterminal_embedding_dim']
dropout = params['decoder_dropout']
batch_size = params['batch_size']
lr = params['lr']
command = '''CUDA_VISIBLE_DEVICES=%d python run_parser.py --do_train=True \
--source_embedding_dim=%d \
--encoder_hidden_dim=%d \
--encoder_bidirectional=True \
--encoder_input_dropout=%f \
--encoder_output_dropout=%f \
--decoder_hidden_dim=%d \
--decoder_num_layers=%d \
--rule_embedding_dim=%d \
--nonterminal_embedding_dim=%d \
--max_decode_length=200 \
--serialization_dir=%s \
--seed=1 \
--dropout=%f \
--task=geo \
--language=%s \
--train_data=%s \
--test_data=%s \
--batch_size=%d \
--lr=%f \
--patient=30 \
--optimizer=adam \
--epoch=50 \
--model_save_interval=1 \
--gradient_clip=5 \ ''' % (
FLAGS.cuda_device, source_embedding_dim, encoder_hidden_dim,
encoder_input_dropout, encoder_output_dropout, decoder_hidden_dim,
decoder_num_layers, rule_embedding_dim, nonterminal_embedding_dim,
save_path, dropout, FLAGS.language, FLAGS.train_data, FLAGS.test_data,
batch_size, lr)
command = re.sub('\s+', ' ', command).strip()
subprocess.call(command, shell=True)
last_epoch = 0
for f in os.listdir(save_path):
if f.startswith('metrics_epoch_'):
match = PATTERN.match(f)
if match:
epoch = int(match.group(1))
if epoch > last_epoch:
last_epoch = epoch
file_path = os.path.join(save_path, 'metrics_epoch_%d.json' % last_epoch)
with open(file_path, 'r') as f:
metrics = json.load(f)
accuracy = metrics['best_validation_accuracy']
nni.report_final_result(accuracy)
params = {'CT': 12, 'PT': 14, 'TT': 1, 'max_depth': 7, 'num_boost_round': 182}
parser = argparse.ArgumentParser(description="Argument Parser")
# data source
parser = argparse.ArgumentParser(description="Argument Parser")
parser.add_argument('--dataset', default='Metro', type=str)
parser.add_argument('--city', default='Chongqing', type=str)
parser.add_argument('--MergeIndex', default=3)
parser.add_argument('--DataRange', default="all")
parser.add_argument('--TrainDays', default="all")
#use params and args to show its difference
args = vars(parser.parse_args())
params.update(nni.get_next_parameter())
data_loader = NodeTrafficLoader(
dataset=args["dataset"],
city=args['city'],
closeness_len=int(params['CT']),
period_len=int(params['PT']),
trend_len=int(params['TT']),
data_range=args['DataRange'],
train_data_length=args['TrainDays'],
test_ratio=0.1,
with_lm=False,
normalize=False,
MergeIndex=args['MergeIndex'],
MergeWay="max" if args["dataset"] == "ChargeStation" else "sum")
'dropout_rate': 0.5,
'channel_1_num': 32,
'channel_2_num': 64,
'conv_size': 5,
'pool_size': 2,
'hidden_size': 1024,
'learning_rate': 1e-4,
'batch_size': 32
}
return params
if __name__ == '__main__':
try:
# get parameters form tuner
RCV_PARAMS = nni.get_next_parameter()
logger.debug(RCV_PARAMS)
# run
params = generate_default_params()
params.update(RCV_PARAMS)
'''
If you use Hyperband, among the hyperparameters (i.e., key-value pairs) received by a trial,
there is one more key called `STEPS` besides the hyperparameters defined by user.
By using this `STEPS`, the trial can control how long it runs.
'''
params['batch_num'] = RCV_PARAMS['STEPS'] * 10
main(params)
except Exception as exception:
logger.exception(exception)
raise
return params
def parse_init_json(data):
params = {}
for key in data:
value = data[key]
if value == 'Empty':
params[key] = ['Empty']
else:
params[key] = [value[0], value[1], value[1]]
return params
if __name__ == '__main__':
try:
# get parameters form tuner
data = nni.get_next_parameter()
logger.debug(data)
RCV_PARAMS = parse_init_json(data)
logger.debug(RCV_PARAMS)
params = generate_defualt_params()
params.update(RCV_PARAMS)
print(RCV_PARAMS)
main(params)
except Exception as exception:
logger.exception(exception)
raise
if __name__ == "__main__":
datasets_mapping = {
'cifar10': datasets.CIFAR10,
'cifar100': datasets.CIFAR100,
'svhn': datasets.SVHN,
}
args = parse_args()
set_seeds(args.seed)
# --
# IO
print('Making dataloaders ...', file=sys.stderr)
transform_train = transforms.Compose([
Policy(nni.get_next_parameter()),
transforms.ToTensor(),
btransforms.NormalizeDataset(dataset=args.dataset),
])
transform_eval = transforms.Compose([
transforms.ToTensor(),
btransforms.NormalizeDataset(dataset=args.dataset),
])
try:
if args.dataset in ['cifar10', 'cifar100']:
trainset = datasets_mapping[args.dataset](
root='./data',
train=True,
download=args.download,
def main():
# Training settings
RCV_CONFIG = nni.get_next_parameter()
_logger.debug(RCV_CONFIG)
'''
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size', type=int, default=64, metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=10, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model', action='store_true', default=False,
help='For Saving the current Model')
'''
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size', type=int, default=RCV_CONFIG['batch-size'], metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=RCV_CONFIG['epochs'], metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=RCV_CONFIG['lr'], metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model', action='store_true', default=False,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.test_batch_size, shuffle=True, **kwargs)
model = Net().to(device)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader)
nni.report_final_result(best_acc)
if (args.save_model):
torch.save(model.state_dict(),"mnist_cnn.pt")
choices=['sgd', 'rmsprop', 'adam'])
parser.add_argument('--momentum',
default=0.9,
type=int,
help='optimizer momentum (ignored in adam)')
parser.add_argument('--num_workers',
default=2,
type=int,
help='number of workers to preprocess data')
parser.add_argument('--model',
default='resnet18',
choices=available_models,
help='the model to use')
parser.add_argument('--grad_clip',
default=0.,
type=float,
help='gradient clip (use 0 to disable)')
parser.add_argument('--log_frequency',
default=20,
type=int,
help='number of mini-batches between logging')
parser.add_argument('--seed',
default=42,
type=int,
help='global initial seed')
args = parser.parse_args()
nni.utils.merge_parameter(args, nni.get_next_parameter())
main(args)
parser.add_argument(
"--eval_window_strict",
type=bool,
default=False,
help=
"True: expect 0 output for all segs except response; False: expect 0 for Cue only.",
required=False)
parser.add_argument(
"--cl_platform_vendor",
type=str,
default='NVIDIA',
help=
"Use the first CL platform matching this vendor (case insensitive)",
required=False)
args, unknown = parser.parse_known_args()
try:
params = {**generate_default_params(), **nni.get_next_parameter()}
logger.debug(params)
run_trial(params,
ens_seed=args.ens_seed,
train_trials_seed=args.train_trials_seed,
test_trials_seed=args.test_trials_seed,
train_trials_per_cond=args.train_trials_per_cond,
test_trials_per_cond=args.test_trials_per_cond,
eval_strict=args.eval_window_strict,
cl_platform_vendor=args.cl_platform_vendor)
except Exception as e:
logger.exception(e)
raise
def run_nn_bacteria_network(tax, data_set_name):
"""
Complex models - feature importance calculation-
NN is complex model which function as a black box that cannot be clearly deduced from the contribution of each
feature.
A different approach for inferring the importance of each bacterium in these models is required.
Initially we trained the model to predict the change in each of the bacteria, we used the original data as an input.
Now, although the equivalent to ‘coefficencts‘ is not visible to us, every bacterium has an effect, between none and
extreme on the prediction.
Therefore, we could estimate this effect by introducing modified input into the model and examining its effects on
prediction.
We chose to examine the relationship between each bacterial pair by using the existing fixed model that was trained
for the ‘first’ bacterial prediction, and forward it an input that was modified only for the ‘second’ bacterium.
U test can be used to investigate whether two independent samples were selected from populations having the same
distribution. That is, the test can tell whether the distribution of predictions has changed significantly in light
of the change in input - indicating interacting.
Comparing the original prediction and the modified data’s prediction distributions, if the change between the two is
significant according to U test, we conclude that there is interaction between the bacterial pair.
The type of interaction will be determined by the obtained change- increasing or decreasing the count of the
bacterium at a fixed size, and its effect, increase or decrease in the prediction of the count of bacteria.
The change will be by the constant 'CHANGE', you can alter it
:param tax: (string) main dataset folder "DATASET/tax=x"
:return: doesn't return an object, create a csv file with all the results
csv name = "interaction_network_" + params.__str__().replace(" ", "").replace("'", "") + "_df.csv"
To determine which bacteria have interactions and create a visual graph use "built_network_form_file.py"
sent the results csv path
"""
# sub folder path for network results,
folder = os.path.join(tax, "interaction_network")
if not os.path.exists(folder):
os.mkdir(folder)
# ------------------------------------ decide on mission ------------------------------------
nni_ = False # check model or run nni for real
GPU = True if nni_ else False
report_loss = True
report_correlation = not report_loss
single_bacteria = True # learn the change in single bacteria or all in once
k_fold = False # run k fold
p_value = 0.001
test_size = 0.3
if nni_:
params = nni.get_next_parameter()
else:
params = {
"STRUCTURE": "001L200H",
"TRAIN_TEST_SPLIT": 0.7,
"EPOCHS": 70,
"LEARNING_RATE": 1e-3,
"OPTIMIZER": "Adam",
"REGULARIZATION": 0.01,
"DROPOUT": 0.1
}
with open(os.path.join(tax, "bacteria.txt"), "r") as b_file:
bacteria = b_file.readlines()
bacteria = [b.rstrip() for b in bacteria]
bacteria_number_list = range(len(bacteria))
# ------------------------------------ data loading ------------------------------------
# run a prediction of a single bacteria at a time
# consider the average loss and correlation of all runs as the performance measurement
df_title = os.path.join(
folder, "interaction_network_" +
params.__str__().replace(" ", "").replace("'", "") + "_df.csv")
df = pd.DataFrame(columns=["BACTERIA", "CHANGED_BACTERIA", "CHANGE", "Y"])
df.to_csv(df_title, index=False)
# create a df that saves a binary value 1/0 => interaction/no interaction according to the train set
train_binary_significant_df = pd.DataFrame(columns=bacteria)
# create a df that saves the continuous b value of each bacteria according to the test set
test_b_df = pd.DataFrame(columns=bacteria)
for b_i, bacteria_num in enumerate(
bacteria_number_list): # for each bacteria
df = pd.read_csv(df_title)
path = "X_y_for_bacteria_number_" + str(b_i) + ".csv"
X_trains, X_tests, y_trains, y_tests, name = \
get_adapted_X_y_for_wanted_learning_task(tax, path, "regular", k_fold=1)
X_train, X_test, y_train, y_test = X_trains[0], X_tests[0], y_trains[
0], y_tests[0]
NUMBER_OF_SAMPLES = X_train.shape[0]
NUMBER_OF_BACTERIA = X_train.shape[1]
NUMBER_OF_TIME_POINTS = None
missing_values = np.array([1 for j in range(NUMBER_OF_SAMPLES)])
# split to train and test
"""
split_list = [1 - test_size, test_size]
split_list = np.multiply(np.cumsum(split_list), len(X)).astype("int").tolist()
# list of shuffled indices to sample randomly
shuffled_idx = []
shuffle(person_indexes)
for arr in person_indexes:
for val in arr:
shuffled_idx.append(val)
# split the data itself
X_train = X[shuffled_idx[:split_list[0]]]
y_train = y[shuffled_idx[:split_list[0]]]
X_test = X[shuffled_idx[split_list[0]:split_list[1]]]
y_test = y[shuffled_idx[split_list[0]:split_list[1]]]
"""
train_binary_significant_for_b_i = []
test_1_u_score_for_b_i = []
"""
path = "time_serie_X_y_for_bacteria_number_" + str(bacteria_num) + ".csv"
X, y, missing_values, name = get_adapted_X_y_for_wanted_learning_task(tax, path, "time_serie")
NUMBER_OF_SAMPLES = X.shape[0]
NUMBER_OF_TIME_POINTS = X.shape[1]
NUMBER_OF_BACTERIA = X.shape[2]
flat_time_points_values_num = NUMBER_OF_SAMPLES * NUMBER_OF_TIME_POINTS
X = X.reshape(flat_time_points_values_num, NUMBER_OF_BACTERIA)
y = y.reshape(flat_time_points_values_num)
missing_values = missing_values.reshape(flat_time_points_values_num)
person_indexes = np.linspace(0, flat_time_points_values_num - 1, flat_time_points_values_num). \
reshape(NUMBER_OF_SAMPLES, NUMBER_OF_TIME_POINTS).astype(int).tolist()
"""
# TRAIN
# run the model one time with no change, then save it
res_map = run_NN(X_train,
y_train,
missing_values,
params,
name,
folder,
NUMBER_OF_SAMPLES,
NUMBER_OF_TIME_POINTS,
NUMBER_OF_BACTERIA,
save_model=True,
GPU_flag=GPU,
k_fold=k_fold,
task_id="base_" + str(b_i) + "_model",
person_indexes=None)
model_path = os.path.join(
folder, "trained_models",
params.__str__().replace(" ", "").replace("'", "") + "_base_" +
str(b_i) + "_model_model")
out_dim = 1 if len(y_train.shape) == 1 else y_train.shape[
1] # else NUMBER_OF_BACTERIA
structure = params["STRUCTURE"]
layer_num = int(structure[0:3])
hid_dim_1 = int(structure[4:7])
hid_dim_2 = int(structure[8:11]) if len(structure) > 10 else None
clf_params = {
"NN_input_dim": X_train.shape[1],
"NN_hidden_dim_1": hid_dim_1,
"NN_output_dim": out_dim
}
clf = bacteria_network_clf(clf_params)
clf.load(model_path)
y_pred_no_change = clf.predict(torch.FloatTensor(X_train))
y_str = ""
for val in y_pred_no_change:
y_str += str(val.detach().numpy()[0]) + " "
df.loc[len(df)] = [int(bacteria_num), int(-1), "no change", y_str]
# ------------------------------------ send to network ------------------------------------
for bacteria_to_change_num in bacteria_number_list: # change each bacteria
# change X, y for only bacteria_to_change_num
X_positive_change = copy.deepcopy(X_train)
for s_i, sample in enumerate(
X_positive_change
): # 0.9459053900000001 -0.05409460999999999
X_positive_change[s_i][bacteria_to_change_num] += CHANGE
y_pred_pos_change = clf.predict(
torch.FloatTensor(X_positive_change))
y_str = ""
for val in y_pred_pos_change:
y_str += str(val.detach().numpy()[0]) + " "
df.loc[len(df)] = [
int(bacteria_num),
int(bacteria_to_change_num), "plus " + str(CHANGE), y_str
]
X_negative_change = copy.deepcopy(X_train)
for s_i, sample in enumerate(X_negative_change):
X_negative_change[s_i][bacteria_to_change_num] -= CHANGE
y_pred_neg_change = clf.predict(
torch.FloatTensor(X_negative_change))
y_str = ""
for val in y_pred_neg_change:
y_str += str(val.detach().numpy()[0]) + " "
df.loc[len(df)] = [
int(bacteria_num),
int(bacteria_to_change_num), "minus " + str(CHANGE), y_str
]
pos_u, pos_u_test_p_val = mannwhitneyu(
y_pred_no_change.detach().numpy(),
y_pred_pos_change.detach().numpy())
neg_u, neg_u_test_p_val = mannwhitneyu(
y_pred_no_change.detach().numpy(),
y_pred_neg_change.detach().numpy())
if pos_u_test_p_val < p_value and neg_u_test_p_val < p_value:
train_binary_significant_for_b_i.append(1)
else:
train_binary_significant_for_b_i.append(0)
# TEST
# run the model one time with no change, then save it
NUMBER_OF_SAMPLES = X_test.shape[0]
NUMBER_OF_BACTERIA = X_test.shape[1]
NUMBER_OF_TIME_POINTS = None
missing_values = np.array([1 for j in range(NUMBER_OF_SAMPLES)])
res_map = run_NN(X_test,
y_test,
missing_values,
params,
name,
folder,
NUMBER_OF_SAMPLES,
NUMBER_OF_TIME_POINTS,
NUMBER_OF_BACTERIA,
save_model=True,
GPU_flag=GPU,
k_fold=k_fold,
task_id="base_" + str(b_i) + "_model",
person_indexes=None)
model_path = os.path.join(
folder, "trained_models",
params.__str__().replace(" ", "").replace("'", "") + "_base_" +
str(b_i) + "_model_model")
out_dim = 1 if len(
y_test.shape) == 1 else y_test.shape[1] # else NUMBER_OF_BACTERIA
structure = params["STRUCTURE"]
layer_num = int(structure[0:3])
hid_dim_1 = int(structure[4:7])
hid_dim_2 = int(structure[8:11]) if len(structure) > 10 else None
clf_params = {
"NN_input_dim": X_test.shape[1],
"NN_hidden_dim_1": hid_dim_1,
"NN_output_dim": out_dim
}
clf = bacteria_network_clf(clf_params)
clf.load(model_path)
y_pred_no_change = clf.predict(torch.FloatTensor(X_test))
# ------------------------------------ send to network ------------------------------------
for bacteria_to_change_num in bacteria_number_list: # change each bacteria
# change X, y for only bacteria_to_change_num
X_positive_change = copy.deepcopy(X_test)
for s_i, sample in enumerate(X_positive_change):
X_positive_change[s_i][bacteria_to_change_num] += CHANGE
y_pred_pos_change = clf.predict(
torch.FloatTensor(X_positive_change))
X_negative_change = copy.deepcopy(X_test)
for s_i, sample in enumerate(X_negative_change):
X_negative_change[s_i][bacteria_to_change_num] -= CHANGE
y_pred_neg_change = clf.predict(
torch.FloatTensor(X_negative_change))
pos_u, pos_u_test_p_val = mannwhitneyu(
y_pred_no_change.detach().numpy(),
y_pred_pos_change.detach().numpy())
neg_u, neg_u_test_p_val = mannwhitneyu(
y_pred_no_change.detach().numpy(),
y_pred_neg_change.detach().numpy())
test_1_u_score_for_b_i.append((1 / pos_u, 1 / neg_u))
# save bacteria b_i results
df.to_csv(df_title, index=False)
train_binary_significant_df.loc[len(
train_binary_significant_df)] = train_binary_significant_for_b_i
test_b_df.loc[len(test_b_df)] = test_1_u_score_for_b_i
# calculate AUC on the flatten data frame
# positive change tuple[0]
pos_b = []
neg_b = []
for row in test_b_df.values:
for val in row:
pos_b.append(float(val[0]))
neg_b.append(float(val[1]))
pos_b = np.array(pos_b)
neg_b = np.array(neg_b)
train_binary_significant_values = []
for val in np.array(train_binary_significant_df.values).flatten():
train_binary_significant_values.append(val)
train_binary_significant_values = np.array(train_binary_significant_values)
try:
pos_auc = roc_auc_score(train_binary_significant_values, pos_b)
neg_auc = roc_auc_score(train_binary_significant_values, neg_b)
Networks_AUC_df = pd.read_csv("all_Networks_AUC.csv")
Networks_AUC_df.loc[len(Networks_AUC_df)] = [
"positive change", "neural network " + params.__str__(),
data_set_name, test_size, k_fold, pos_auc,
datetime.utcnow().strftime("%d/%m/%Y %H:%M:%S")
]
Networks_AUC_df.loc[len(Networks_AUC_df)] = [
"negative change", "neural network " + params.__str__(),
data_set_name, test_size, k_fold, neg_auc,
datetime.utcnow().strftime("%d/%m/%Y %H:%M:%S")
]
Networks_AUC_df.to_csv("all_Networks_AUC.csv", index=False)
except:
print(
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
print(len(train_binary_significant_values))
print(set(train_binary_significant_values))
print(len(pos_b))
print(len(set(pos_b)))
print(len(neg_b))
print(len(set(neg_b)))
print(
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
Run main.py to start.
This script is modified from PyTorch quickstart:
https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html
"""
import nni
import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
# Get optimized hyperparameters
params = {'features': 512, 'lr': 0.001, 'momentum': 0}
optimized_params = nni.get_next_parameter()
params.update(optimized_params)
# Load dataset
training_data = datasets.FashionMNIST(root='data',
train=True,
download=True,
transform=ToTensor())
test_data = datasets.FashionMNIST(root='data',
train=False,
download=True,
transform=ToTensor())
train_dataloader = DataLoader(training_data, batch_size=64)
test_dataloader = DataLoader(test_data, batch_size=64)
# Build model
import torch
import torch.nn as nn
import torch.nn.functional as F
from operations import ReLUConvBN, OPS, FactorizedReduce
from torch.autograd import Variable
# from genotypes import PRIMITIVES
import nni
from genotypes import Genotype
TUNER_PARAMS = nni.get_next_parameter()
PRIMITIVES = TUNER_PARAMS["primitives"]
class MixedOp(nn.Module):
def __init__(self, C, stride):
super(MixedOp, self).__init__()
self._ops = nn.ModuleList()
for primitive in PRIMITIVES:
op = OPS[primitive](C, stride, False)
if 'pool' in primitive:
op = nn.Sequential(op, nn.BatchNorm2d(C, affine=False))
self._ops.append(op)
def forward(self, x, weights):
return sum(w * op(x) for w, op in zip(weights, self._ops))
class Cell(nn.Module):
def __init__(self, steps, multiplier, C_prev_prev, C_prev, C, reduction,
reduction_prev):
super(Cell, self).__init__()
def test_get_next_parameter(self):
self.assertEqual(nni.get_next_parameter(), self._trial_params)
def main_compute(code_only=False):
tvm_target = 'cuda'
tvm.register_func('tvm_callback_cuda_compile',
compile_source,
override=True)
logging.getLogger('autotvm').setLevel(logging.DEBUG)
logging.getLogger('autotvm').addHandler(logging.StreamHandler(sys.stdout))
default_tune_op = importlib.import_module('templates.' + (
os.environ['OP'] if 'OP' in os.environ else 'auto.generic'))
print(' >> Backend = %s, Python PID = %s, Task = %s;' %
(backend, os.getpid(), default_tune_op.__name__))
task = autotvm.task.create("template_op", args=(), target=tvm_target)
def json_to_config(json_dict, index=-1, code_hash=None):
if not isinstance(json_dict, list):
json_list = []
for key in json_dict:
json_list.append([
key, 'ot' if type(json_dict[key]) is not list else
('sp' if json_dict[key][0:1] == [-1] else 're'),
json_dict[key]
])
json_dict = json_list
config = ConfigEntity.from_json_dict({
"index": index,
"time": "",
"code_hash": code_hash,
"entity": json_dict
})
# config = ConfigEntity.from_json_dict({"i": index, "t": "", "c": code_hash, "e": json_dict})
return config
def config_to_json(config):
if config is None:
return {}
if isinstance(config, str):
return json.loads(config)
jobj = config.to_json_dict()['entity']
# jobj = config.to_json_dict()['e']
json_dict = dict()
for i in range(len(jobj)):
assert (jobj[i][1] in ['sp', 'ot', 're'])
json_dict[jobj[i][0]] = jobj[i][2]
return json_dict
num_trials = int(os.environ['STEP']) if 'STEP' in os.environ else 0
config = os.environ.get('CONFIG', '').strip()
if config != '':
if config[0] != '[':
params_given = json.loads(config)
print("====>> [Current Config Option]", config)
best_config = json_to_config(params_given)
else:
best_config = config
elif 'NNI_TRIAL_JOB_ID' in os.environ:
if os.environ['NNI_TRIAL_JOB_ID'] == '@':
search_space = get_search_space(task.config_space)
json_space = json.dumps(search_space)
dump_to_file = './search_space.json'
print("\n>> Writing Search Space to '%s', Search Space = %s;" %
(dump_to_file, json_space))
with open("search_space.json", "w") as fp:
fp.write(json_space)
sys.exit(0)
try:
import nni
params_given = nni.get_next_parameter()
if params_given is None:
raise
local_dir_id = os.environ['NNI_TRIAL_JOB_ID']
except:
params_given = default_tune_op.get_choice_example()
local_dir_id = '_'
t = run_config_entity(params_given, local_dir_id)
gflops = compute_gflops(task.flop, t)
print('[Antares-engine] Final entity result is: %g' % gflops)
try:
nni.report_final_result(gflops)
except:
print('[Antares-engine] (not reporting final result to NNI.)')
exit(0)
elif num_trials > 0:
dev_num = platform_config.get_execution_parallism()
if dev_num <= 0:
raise Exception("No valid device found for backend: %s." % backend)
batch_size = int(os.environ.get('BATCH', '16'))
from concurrent.futures import ThreadPoolExecutor
try:
if platform_config.allow_concurrent_compile_execution():
raise Exception()
worker_size = 1
except:
worker_size = batch_size
thread_pool = ThreadPoolExecutor(max_workers=worker_size)
task.antares_helper = Mock()
task.antares_helper.json_to_config = json_to_config
task.antares_helper.config_to_json = config_to_json
task.antares_helper.to_json_search_space = get_search_space
tuner_type = os.environ.get('TUNER', 'XGBoost')
print(' >> MAKE_PARA = %d/%d, EXEC_PARA = %d, TUNER = %s' %
(worker_size, batch_size, dev_num, tuner_type))
auto_commit = os.environ.get('COMMIT', '')
if auto_commit:
saved_code = codehub_db(os.environ['COMPUTE_V1'])
if saved_code is not None and auto_commit != 'force':
raise Exception(
"Saved code has existed in codehub. Please try COMMIT=force to overide it."
)
os.environ.pop('COMMIT')
try:
tuner = importlib.import_module('tuner.%s.main' % tuner_type)
tuner = tuner.MainTuner(task)
except:
raise Exception('>> Cannot import Antares Tuner: %s' % tuner_type)
if tuner is not None:
def measure_batch(inputs):
results, futures = [], []
best_slot = -1
expected_timecost = tuner.task.best.timecost
for i in range(len(inputs)):
futures.append(
thread_pool.submit(run_config_entity,
config_to_json(inputs[i].config), i,
expected_timecost, i % dev_num))
for i in range(len(inputs)):
t = futures[i].result()
if t < tuner.task.best.timecost:
best_slot = i
tuner.task.best.timecost = t
tuner.task.best.config = inputs[i].config
tuner.task.best.occur = tuner.task.best.curr_step + i + 1
results.append(
autotvm.measure.MeasureResult(costs=(t, ),
error_no=0,
all_cost=i,
timestamp=time.time()))
tuner.task.best.curr_step += len(results)
print(
'\nSTEP[%d / %d] Current Best Config = %s, Perf = %g Gflops, Occur Step = %d;'
%
(tuner.task.best.curr_step, num_trials,
json.dumps(config_to_json(tuner.task.best.config)),
compute_gflops(tuner.task.flop, tuner.task.best.timecost),
tuner.task.best.occur))
if auto_commit and best_slot >= 0:
with open(local_get_dir_file('my_kernel.cc', best_slot),
'r') as fp:
device_source = fp.read()
with open(local_get_dir_file('result.txt', best_slot),
'r') as fp:
t = float(fp.read().split()[0])
kernel_path = codehub_db(
os.environ['COMPUTE_V1'],
source_code=device_source +
'\n// Saved Perf = %g sec / run; Step Produced = %d;' %
(t, tuner.task.best.curr_step))
print(' >> Update current code to codehub: %s' %
kernel_path)
return results
tuner.task.best = Mock()
tuner.task.best.timecost = float('inf')
tuner.task.best.config = None
tuner.task.best.occur = -1
tuner.task.best.curr_step = 0
tuner.measure_batch = measure_batch
callbacks = []
history_log_for_transfer_learning = os.environ.get('RECORD', '')
if history_log_for_transfer_learning:
callbacks.append(
autotvm.callback.log_to_file(
history_log_for_transfer_learning))
# Enable Transfer Learning for Incremental Task
if os.path.exists(history_log_for_transfer_learning):
print(
' >> Loading incremental history from log file: %s ..'
% history_log_for_transfer_learning)
tuner.load_history(
autotvm.record.load_from_file(
history_log_for_transfer_learning))
tuner.tune(n_trial=num_trials,
measure_option=autotvm.measure_option(
builder=autotvm.LocalBuilder(n_parallel=batch_size),
runner=autotvm.LocalRunner(repeat=3,
min_repeat_ms=100,
timeout=4)),
callbacks=callbacks)
assert not math.isinf(
tuner.task.best.timecost
), "Not valid config found in the whole tuning."
best_config = tuner.task.best.config
print(
"\n[Best Config] CONFIG='%s' ==> Performance is up to %f Gflops, occurred at step %d / %d; time per run = %g sec."
%
(json.dumps(config_to_json(best_config)),
compute_gflops(tuner.task.flop, tuner.task.best.timecost),
tuner.task.best.occur, num_trials, tuner.task.best.timecost))
if hasattr(tuner, 'cleanup'):
tuner.cleanup()
else:
raise Exception('Unrecognized tuner type: `%s`' % tuner_type)
exit(0)
else:
if os.environ['OP'] == 'auto.generic':
saved_code = codehub_db(os.environ['COMPUTE_V1'])
if saved_code is not None:
print(" >> Using Saved Code from Codehub:")
print("===========================")
print(saved_code)
print("===========================")
exit(0)
best_config = task.config_space
if isinstance(best_config, str):
from tvm import auto_scheduler
origin_cfg = json.loads(best_config)
origin_cfg = {
"i": [[
'["main_compute.<locals>.auto_template"]',
'cuda -keys=cuda,gpu -max_num_threads=%d -thread_warp_size=%d'
% (device_properties().max_threads_per_block,
device_properties().warp_size)
], origin_cfg],
"r": [[0], 0, 0, 0],
"v":
"v0.2",
}
origin_cfg_file = local_get_dir_file('my_kernel.cfg')
with open(origin_cfg_file, 'w') as fp:
fp.write(json.dumps(origin_cfg))
origin_cfg = tvm.auto_scheduler.measure_record.load_records(
origin_cfg_file)
@auto_scheduler.register_workload
def auto_template():
_, arg_bufs = default_tune_op.get_template_op()
return arg_bufs
target = tvm.target.Target("cuda")
auto_task = auto_scheduler.create_task(auto_template, (), target)
for inp, res in origin_cfg:
s, arg_bufs = auto_task.compute_dag.apply_steps_from_state(
inp.state)
break
else:
with ApplyConfig(best_config):
with tvm.target.Target(tvm_target):
s, arg_bufs = default_tune_op.get_template_op()
if s is not None:
lower_source = str(tvm.lower(s, arg_bufs, simple_mode=True))
lower_file = local_get_dir_file('my_kernel.lower')
with open(lower_file, 'w') as fp:
fp.write(lower_source)
# Verify Lower Code Code
if len(('\n' + lower_source).split('\nprimfn(')) != 2:
raise Exception('[Not Support Multi Unfuse-able kernels]\n\n' +
lower_source)
max_threads_per_block = device_properties().max_threads_per_block
max_shared_memory_per_block = device_properties(
).max_shared_memory_per_block
assert max_threads_per_block > 0 and max_shared_memory_per_block >= 0, '[Error] Invalid device properties, maybe device is not detected correctly.'
lower_lines = lower_source.split('\n')
thread_extents, allocate_shared = [], []
for ll in lower_lines:
if ll.strip().startswith(
'attr [IterVar(') and ll.find(' "thread_extent" = ') >= 0:
thread_name = ll.split('attr [IterVar(')[-1].split(':')[0]
thread_val = int(
ll.split(' "thread_extent" = ')[-1].split(';')
[0].strip().split(' ')[0])
thread_extents.append((thread_name, thread_val))
elif ll.strip().startswith('allocate(') and ll.find(
'.shared, ') >= 0 and ll.endswith(");"):
parts = ll[:-2].split(', ')[1:]
allocate_type = parts[0]
allocate_val = int(np.product(eval(parts[1])))
allocate_shared.append((allocate_type, allocate_val))
reserved_axes = dict()
for thread_name, thread_val in thread_extents:
if thread_name in reserved_axes:
assert reserved_axes[
thread_name] == thread_val, "Invalid code: Multiple hints for thread extent conflict with each other: %d v.s. %d" % (
reserved_axes[thread_name], thread_val)
else:
reserved_axes[thread_name] = thread_val
num_threads = 1
for thread_name in ['threadIdx.x', 'threadIdx.y', 'threadIdx.z']:
num_threads *= reserved_axes.get(thread_name, 1)
assert num_threads <= max_threads_per_block, "Invalid kernel code: using num_threads(%d) > max_threads_per_block(%d)" % (
num_threads, max_threads_per_block)
shared_memory_in_bytes = 0
for allocate_type, allocate_size in allocate_shared:
if allocate_type.startswith('custom['):
type_name = allocate_type[7:].split(']')[0]
shared_memory_inc = int(
custom_dtypes[type_name][-1].split('@')[-1])
else:
shared_memory_inc = 8 * np.dtype(allocate_type).itemsize
assert shared_memory_inc % 8 == 0, "The bits of shared_memory is not aligned with 8-bit bytes."
shared_memory_in_bytes += shared_memory_inc // 8 * allocate_size
if shared_memory_in_bytes > max_shared_memory_per_block:
raise Exception(
"Invalid kernel code: using shared_memory_in_bytes %d > max_shared_memory_per_block %d"
% (shared_memory_in_bytes, max_shared_memory_per_block))
# Compile Source Code
def build_template():
return tvm.build(s, arg_bufs, tvm_target, name='template_op')
func = wait_for(build_template, 30)
assert (len(func.imported_modules) == 1)
device_source = translate_code(func.imported_modules[0].get_source())
if code_only:
return device_source
print("====================================")
print(device_source)
print("====================================")
print()
try:
eval_client = importlib.import_module('platforms.%s.evaluator.client' %
backend)
except ModuleNotFoundError:
print('>> Evaluator for backend %s not found, skipping evaluation.' %
backend)
exit(0)
except:
traceback.print_exc()
exit(1)
def handle_result(result):
print('\n[EvalAgent] Results =', json.dumps(result))
if 'RESULT' in os.environ:
if abs(float(os.environ['RESULT']) / result['K/0'] - 1.0) > 1e-6:
result['TPR'] = None
t = result.get('TPR', None)
if t is None:
print("\n[Antares] Incorrect compute kernel from evaluator.")
else:
gflops = compute_gflops(task.flop, t)
print("\n[Antares] Average time cost / run = %g sec, %g gflops." %
(t, gflops))
with open(local_get_dir_file('result.txt'), 'w') as fp:
fp.write(str(t) + '\n')
if 'K/0' in result:
fp.write(str(result['K/0']) + '\n')
if os.environ['OP'] == 'auto.generic' and os.environ.get('COMMIT', ''):
kernel_path = codehub_db(os.environ['COMPUTE_V1'],
source_code=device_source +
'\n// Saved Perf = %g sec / run' % t)
print(' >> Update current code to codehub: %s' % kernel_path)
tune_slot_id = int(os.environ.get(unified_slot_key, '0'))
exec_fd, _ = system_lock([tune_slot_id])
try:
expected_timeout = None
if 'EXPECTED_TIMEOUT' in os.environ and not math.isinf(
float(os.environ['EXPECTED_TIMEOUT'])):
expected_timeout = float(os.environ['EXPECTED_TIMEOUT'])
expected_timeout = max(expected_timeout * 1.1,
expected_timeout + 0.1)
results = eval_client.eval(
kernel_path=local_get_dir_file('my_kernel.cc'),
expected_timeout=expected_timeout,
func=func,
)
except:
traceback.print_exc()
exit(1)
handle_result(results)
exec_fd()
exit(0)
def main():
global NNI
_params = {
"print_every_num_of_epochs": 100,
"data_folder_name": "dnc",
"data_name": "dnc_candidate_two",
"graphs_cutoff_number": 2,
"l1_lambda": 0,
"epochs": 500,
"gcn_dropout_rate": 0.7,
"lstm_dropout_rate": 0,
"gcn_hidden_sizes": [10, 10, 10, 10, 10, 10, 10, 10, 10],
"learning_rate": 0.001,
"weight_decay": 0,
"gcn_latent_dim": 5,
"lstm_hidden_size": 10,
"lstm_num_layers": 1,
"learned_label": DEFAULT_LABEL_TO_LEARN,
}
l1_lambda = [0, 1e-7]
epochs = [500]
gcn_dropout_rate = [0.3, 0.5]
gcn_hidden_sizes = [[100, 100], [200, 200]]
learning_rate = [1e-3, 1e-2, 3e-2]
weight_decay = [5e-2, 1e-2]
gcn_latent_dim = [50, 100]
lstm_hidden_size = [50, 100]
results = []
argparser = argparse.ArgumentParser()
argparser.add_argument("--nni", action='store_true')
args = argparser.parse_args()
NNI = args.nni
if NNI:
p = nni.get_next_parameter()
p["gcn_hidden_sizes"] = ast.literal_eval(p["gcn_hidden_sizes"])
_params.update(p)
(model_loss, model_accuracy, model_tot_accuracy, model_correlation,
model_tot_correlation, zero_model_tot_loss, zero_model_tot_accuracy,
zero_model_tot_correlation, first_order_tot_loss,
first_order_tot_accuracy, first_order_tot_correlation,
zero_model_diff_loss, zero_model_diff_accuracy,
zero_model_diff_correlation, first_order_diff_loss,
first_order_diff_accuracy,
first_order_diff_correlation) = run_trial(_params)
if NNI:
nni.report_final_result(model_tot_accuracy[0] -
zero_model_tot_accuracy[-1])
else:
print(
f"Final result (accruacy): model tot accuracy: {model_tot_accuracy}, zero_model_tot_accuracy: {zero_model_tot_accuracy}, "
f"first order tot accuracy: {first_order_tot_accuracy}, zero model diff accuracy: {zero_model_diff_accuracy}, "
f"first order diff accuracy: {first_order_diff_accuracy}")
print(
f"Final result (correlation): model tot correlation: {model_tot_correlation}, zero_model_tot_correlation: {zero_model_tot_correlation}, "
f"first order tot correlation: {first_order_tot_correlation}, zero model diff correlation: {zero_model_diff_correlation}, "
f"first order diff correlation: {first_order_diff_correlation}")
return (model_tot_accuracy, model_tot_correlation, zero_model_tot_accuracy,
zero_model_tot_correlation, first_order_tot_accuracy,
first_order_tot_correlation, zero_model_diff_accuracy,
zero_model_diff_correlation, first_order_diff_accuracy,
first_order_diff_correlation)
beta_2=params["beta_2"],
decay=params["decay"])
elif params["optimizer"] == 'Nadam':
optimizer = Nadam(lr=params["lr"],
beta_1=params["beta_1"],
beta_2=params["beta_2"],
schedule_decay=params["schedule_decay"])
model.compile(optimizer=optimizer,
loss=params['loss'],
metrics=['accuracy'])
hist = model.fit(x_train,
y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_split=0.2,
shuffle=True,
callbacks=[SendMetrics()])
score = model.evaluate(x_test, y_test, batch_size=params['batch_size'])
model.save('m.h5')
nni.report_final_result(score[1])
return hist.history, score
if __name__ == '__main__':
received_params = nni.get_next_parameter()
params = generate_default_params()
params.update(received_params)
hist, score = train()
print('Test loss:', score[0])
print('Test accuracy:', score[1])
def main_compute(code_only=False):
tvm.register_func('tvm_callback_cuda_compile', compile_source, override=True)
logging.getLogger('autotvm').setLevel(logging.DEBUG)
logging.getLogger('autotvm').addHandler(logging.StreamHandler(sys.stdout))
default_tune_op = importlib.import_module('templates.' + (os.environ['OP'] if 'OP' in os.environ else 'auto.generic'))
if verbose:
print(' >> Backend = %s, Python PID = %s, Task = %s;' % (backend, os.getpid(), default_tune_op.__name__))
task = autotvm.task.create("template_op", args=(), target=tvm_target)
def json_to_config(json_dict, index=-1, code_hash=None):
if not isinstance(json_dict, list):
json_list = []
for key in json_dict:
json_list.append([key, 'ot' if type(json_dict[key]) is not list else ('sp' if json_dict[key][0:1] == [-1] else 're'), json_dict[key]])
json_dict = json_list
config = ConfigEntity.from_json_dict({"index": index, "time": "", "code_hash": code_hash, "entity": json_dict})
# config = ConfigEntity.from_json_dict({"i": index, "t": "", "c": code_hash, "e": json_dict})
return config
def config_to_json(config):
if config is None:
return {}
if isinstance(config, str):
return json.loads(config)
jobj = config.to_json_dict()['entity']
# jobj = config.to_json_dict()['e']
json_dict = dict()
for i in range(len(jobj)):
assert(jobj[i][1] in ['sp', 'ot', 're'])
json_dict[jobj[i][0]] = jobj[i][2]
return json_dict
num_trials = int(os.environ['STEP']) if 'STEP' in os.environ else 0
config = os.environ.get('CONFIG', '').strip()
if config != '':
best_config = config
elif 'NNI_TRIAL_JOB_ID' in os.environ:
if os.environ['NNI_TRIAL_JOB_ID'] == '@':
search_space = get_search_space(task.config_space)
json_space = json.dumps(search_space)
dump_to_file='./search_space.json'
print("\n>> Writing Search Space to '%s', Search Space = %s;" % (dump_to_file, json_space))
with open("search_space.json", "w") as fp:
fp.write(json_space)
sys.exit(0)
try:
import nni
params_given = nni.get_next_parameter()
if params_given is None:
raise
local_dir_id = os.environ['NNI_TRIAL_JOB_ID']
except:
params_given = default_tune_op.get_choice_example()
local_dir_id = '_'
t = run_config_entity(params_given, local_dir_id)
gflops = compute_gflops(task.flop, t)
print('[Antares-engine] Final entity result is: %g' % gflops)
try:
nni.report_final_result(gflops)
except:
print('[Antares-engine] (not reporting final result to NNI.)')
exit(0)
elif num_trials > 0:
dev_num = platform_config.get_execution_parallism()
if dev_num <= 0:
raise Exception("No valid device found for backend: %s." % backend)
batch_size = int(os.environ.get('BATCH', '16'))
from concurrent.futures import ThreadPoolExecutor
try:
if platform_config.allow_concurrent_compile_execution():
raise Exception()
worker_size = 1
except:
worker_size = batch_size
thread_pool = ThreadPoolExecutor(max_workers=worker_size)
task.antares_helper = Mock()
task.antares_helper.json_to_config = json_to_config
task.antares_helper.config_to_json = config_to_json
task.antares_helper.to_json_search_space = get_search_space
tuner_type = os.environ.get('TUNER', '')
if not tuner_type:
comp = os.environ['COMPUTE_V1']
if '=!' in comp and 'plan/' not in comp[comp.find(' ##') + 1:] and ';' not in comp and backend in ['c-rocm', 'c-cuda', 'c-hlsl', 'c-ocl']:
tuner_type = 'AutoTVM2'
else:
tuner_type = 'XGBoost'
print(' >> MAKE_PARA = %d/%d, EXEC_PARA = %d, TUNER = %s' % (worker_size, batch_size, dev_num, tuner_type))
auto_commit = os.environ.get('COMMIT', '')
if auto_commit:
saved_code = codehub_db(os.environ['COMPUTE_V1'])
if saved_code is not None and auto_commit != 'force':
raise Exception("Saved code has existed in codehub. Please try COMMIT=force to override it.")
os.environ.pop('COMMIT')
try:
tuner = importlib.import_module('tuner.%s.main' % tuner_type)
tuner = tuner.MainTuner(task)
except:
raise Exception('>> Cannot import Antares Tuner: %s' % tuner_type)
if tuner is not None:
AntaresGlobal.current_step = 0
def measure_batch(inputs):
results, futures = [], []
best_slot = -1
expected_timecost = tuner.task.best.timecost
for i in range(len(inputs)):
futures.append(thread_pool.submit(run_config_entity, config_to_json(inputs[i].config), AntaresGlobal.current_step + i + 1, expected_timecost, i % dev_num))
for i in range(len(inputs)):
t = futures[i].result()
if t < tuner.task.best.timecost:
best_slot = AntaresGlobal.current_step + i + 1
tuner.task.best.timecost = t
tuner.task.best.config = inputs[i].config
tuner.task.best.occur = best_slot
results.append(autotvm.measure.MeasureResult(costs=(t,), error_no=0, all_cost=i, timestamp=time.time()))
AntaresGlobal.current_step += len(results)
print('\nSTEP[%d / %d] Current Best Config = %s, Perf = %g Gflops, MemRatio = %g %%, Occur Step = %d;' % (
AntaresGlobal.current_step,
num_trials,
json.dumps(config_to_json(tuner.task.best.config)),
compute_gflops(tuner.task.flop, tuner.task.best.timecost),
compute_mem_ratio(tuner.task.best.timecost),
tuner.task.best.occur))
if auto_commit and best_slot >= 0:
with open(local_get_dir_file('my_kernel.cc', best_slot), 'r') as fp:
device_source = fp.read()
with open(local_get_dir_file('result.txt', best_slot), 'r') as fp:
t = float(fp.read().split()[0])
kernel_path = codehub_db(os.environ['COMPUTE_V1'], source_code=device_source + '\n// Saved Perf = %g sec / run; Step Produced = %d;' % (t, best_slot))
print(' >> Update current code to codehub: %s' % kernel_path)
return results
tuner.task.best = Mock()
tuner.task.best.timecost = float('inf')
tuner.task.best.config = None
tuner.task.best.occur = -1
tuner.measure_batch = measure_batch
tuner.measure_batch.n_parallel = batch_size
callbacks = []
history_log_for_transfer_learning = os.environ.get('RECORD', '')
if history_log_for_transfer_learning:
callbacks.append(autotvm.callback.log_to_file(history_log_for_transfer_learning))
# Enable Transfer Learning for Incremental Task
if os.path.exists(history_log_for_transfer_learning):
print(' >> Loading incremental history from log file: %s ..' % history_log_for_transfer_learning)
tuner.load_history(autotvm.record.load_from_file(history_log_for_transfer_learning))
tuner.tune(n_trial=num_trials, measure_option=autotvm.measure_option(
builder=autotvm.LocalBuilder(n_parallel=batch_size),
runner=autotvm.LocalRunner(repeat=3, min_repeat_ms=100, timeout=4)
), callbacks=callbacks)
assert not math.isinf(tuner.task.best.timecost), "Not valid config found in the whole tuning."
best_config = json.dumps(config_to_json(tuner.task.best.config))
if auto_commit:
device_source = codehub_db(os.environ['COMPUTE_V1'])
codehub_db(os.environ['COMPUTE_V1'], source_code=device_source + '\n// Antares Tuning Completed in %d steps.' % AntaresGlobal.current_step)
print("\n[Best Config] CONFIG='%s' ==> Performance is up to %f Gflops, occurred at step %d / %d; time per run = %g sec." % (
best_config,
compute_gflops(tuner.task.flop, tuner.task.best.timecost),
tuner.task.best.occur,
num_trials,
tuner.task.best.timecost))
if hasattr(tuner, 'cleanup'):
tuner.cleanup()
else:
raise Exception('Unrecognized tuner type: `%s`' % tuner_type)
exit(0)
else:
if os.environ['OP'] == 'auto.generic':
saved_code = codehub_db(os.environ['COMPUTE_V1'])
if saved_code is not None:
print(" >> Using Saved Code from Codehub:")
print("===========================")
print(saved_code)
print("===========================")
exit(0)
best_config = ''
assert isinstance(best_config, str)
if verbose:
print("====>> [Current Config Option]", best_config)
if best_config.startswith('['):
from tvm import auto_scheduler
origin_cfg = json.loads(best_config)
origin_cfg = {
"i": [['["main_compute.<locals>.auto_template"]', 'cuda -keys=cuda,gpu -max_num_threads=%d -thread_warp_size=%d' % (
device_properties().max_threads_per_block, device_properties().warp_size
)], origin_cfg],
"r": [[0], 0, 0, 0],
"v": "v0.2",
}
origin_cfg_file = local_get_dir_file('my_kernel.cfg')
with open(origin_cfg_file, 'w') as fp:
fp.write(json.dumps(origin_cfg))
origin_cfg = tvm.auto_scheduler.measure_record.load_records(origin_cfg_file)
@auto_scheduler.register_workload
def auto_template():
_, arg_bufs = default_tune_op.get_template_op()
return arg_bufs
target = tvm.target.Target("cuda")
auto_task = auto_scheduler.create_task(auto_template, (), target)
for inp, res in origin_cfg:
s, arg_bufs = auto_task.compute_dag.apply_steps_from_state(inp.state)
break
else:
config = json_to_config(json.loads(best_config)) if best_config else task.config_space
with ApplyConfig(config):
with tvm.target.Target(tvm_target):
s, arg_bufs = default_tune_op.get_template_op()
device_source, kernel_path = get_target_source(s, arg_bufs)
if code_only:
return device_source
if verbose:
print("====================================")
print(device_source)
print("====================================\n")
dev_id = int(os.environ.get('DEV_KEY', '0'))
result = evaluate_perf(kernel_path, task.flop, dev_id)
exit(0 if result is not None else 1)
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
try:
RECEIVED_PARAMS = nni.get_next_parameter()
LOG.debug(RECEIVED_PARAMS)
PARAMS = generate_default_params()
PARAMS.update(RECEIVED_PARAMS)
args = get_args()
print(PARAMS['learning_rate'])
config = process_config_UtsClassification_bayes_optimization(
args.config, PARAMS['learning_rate'], PARAMS['model_name'],
PARAMS['batch_size'])
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
nni.report_final_result(evaluater.f1)
print('done')
except Exception as e:
LOG.exception(e)
raise
import nni
import time
if __name__ == '__main__':
nni.get_next_parameter()
time.sleep(3)
nni.report_final_result(0.5)
parser.add_argument('-f',
'--load',
dest='load',
type=str,
default=False,
help='Load model from a .pth file')
return parser.parse_args()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(message)s')
args = get_args()
#使用nni的get_next_parameter()获取一组超参组合
nni_args = nni.get_next_parameter()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
logging.info(f'Using device {device}')
#初始化网络，使用nni_args中获取的参数：其中的dropout_rate与hidden_size用于网络参数调整
net = Net(dropout_rate=nni_args["dropout_rate"],hidden_size=nni_args["hidden_size"])
if args.load:
net.load_state_dict(torch.load(args.load, map_location=device))
logging.info(f'Model loaded from {args.load}')
net.to(device=device)
#重写训练网络部分，在nni框架支持下，使用nni_args中获取的batch_size与lr
try:
train_net(net=net,
epochs=args.epochs,
import time
import nni
if __name__ == '__main__':
print('trial start')
params = nni.get_next_parameter()
print('params:', params)
epochs = 2
for i in range(epochs):
nni.report_intermediate_result(0.1 * (i + 1))
time.sleep(1)
nni.report_final_result(0.8)
print('trial done')
"""#Training Arguments"""
datadir = './data'
data_name = 'cifar10'
fraction = float(0.1)
num_epochs = int(300)
select_every = int(20)
feature = 'dss'
num_runs = 1 # number of random runs
model_name = 'ResNet164'
device = "cuda" if torch.cuda.is_available() else "cpu"
strategy = "GLISTER"
print("Using Device:", device)
config = nni.get_next_parameter()
"""#Training Runs"""
for run in range(num_runs):
train_model(num_epochs, data_name, datadir, feature, config['model'],
fraction, select_every, config['lr'], run, device,
'GradMatch-Explore')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device,
# 'GradMatchPB-Explore')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Random')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GLISTER')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GradMatch')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'GradMatchPB')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Random-Online')
# train_model(num_epochs, data_name, datadir, feature, model_name, fraction, select_every, learning_rate, run, device, 'Full')
parser.add_argument('--reg_qi', type=float, dest='reg_qi', default=None)
args = parser.parse_args()
return args
def main(params):
logger.debug("Args: %s", str(params))
logger.debug('Number of epochs %d', params["n_epochs"])
svd = svd_training(params)
# Save SVD model to the output directory for later use
output_dir = os.environ.get('NNI_OUTPUT_DIR')
surprise.dump.dump(os.path.join(output_dir, 'model.dump'), algo=svd)
if __name__ == "__main__":
try:
tuner_params = nni.get_next_parameter()
logger.debug("Hyperparameters: %s", tuner_params)
params = vars(get_params())
# in the case of Hyperband, use STEPS to allocate the number of epochs SVD will run for
if 'STEPS' in tuner_params:
steps_param = tuner_params['STEPS']
params['n_epochs'] = int(np.rint(steps_param))
params.update(tuner_params)
main(params)
except Exception as exception:
logger.exception(exception)
raise
import yaml
parser = argparse.ArgumentParser(description='Launch PENN training')
parser.add_argument('--config',
default='PedSM/PENN/training/default_config.yaml',
help='Configuration File Location')
# TODO: these parsers below should do something but at the moment they don't
parser.add_argument('--load-checkpoint',
default=False,
help='Start from a saved checkpoint')
parser.add_argument('--verbose', '-v', action='count', default=0)
args = parser.parse_args()
config = yaml.safe_load(open(args.config))
try:
torch.manual_seed(42) # Always set this as we want to reproduce values
updated_params = nni.get_next_parameter()
params = generate_default_params()
params.update(updated_params)
main(config=config,
restore_old_checkpoint=args.load_checkpoint,
verbosity=args.verbose,
params=params)
except Exception as exc:
logging.debug('Failed to Complete Search')
logging.debug(exc)
raise
