def main():
log.debug("Loading data from '%s'." % args.data)
data = torch.load(args.data + "/data.pt")
vocabs = data["vocabs"]
dev_data = get_dataset(data, args, "dev")
test_data = get_dataset(data, args, "test")
state_dict = torch.load(args.export_path)
args.word_emb_size = state_dict["module.word_lut"].size(1)
args.mention_len = dev_data.get_mention_len()
args.context_len = dev_data.get_context_len()
log.debug("Building model...")
model = Model(args, vocabs)
model = DataParallel(model)
model.load_state_dict(state_dict)
model.to(DEVICE)
runner = Runner(model, None, None, vocabs, None, None, dev_data, test_data,
args)
log.info("INFERENCE ON DEV DATA:")
runner.instance_printer.show(dev_data)
runner.print_full_validation(dev_data, "DEV")
log.info("\n\nINFERENCE ON TEST DATA:")
runner.instance_printer.show(test_data)
runner.print_full_validation(test_data, "TEST")
def main():
log.debug("Loading data from '%s'." % args.data)
data = torch.load(args.data + "/data.pt")
vocabs = data["vocabs"]
dev_data = get_dataset(data, args, "dev")
test_data = get_dataset(data, args, "test")
state_dict = torch.load("models/" + args.export_path + ".pt")
args.type_dims = state_dict["type_lut.weight"].size(1)
log.debug("Building model...")
model = figet.Models.Model(args, vocabs)
model.load_state_dict(state_dict)
if len(args.gpus) >= 1:
model.cuda()
type2vec = model.type_lut.weight.data
coach = figet.Coach(model, None, vocabs, None, dev_data, test_data,
type2vec, None, args)
log.info("INFERENCE ON DEV DATA:")
coach.instance_printer.show(dev_data)
coach.print_results(dev_data, "DEV")
log.info("\n\nINFERENCE ON TEST DATA:")
coach.instance_printer.show(test_data)
coach.print_results(test_data, "TEST")
def get_generators(is_test=False):
"""Get the dataset class according to the neptune parameters."""
datasets = get_dataset(is_test=is_test)
# Fetch the processor
processor = bootstrap.get_processor()
# If len is 1, then return the test set
if len(datasets) == 1:
gen = data.DataGenerator(is_training=False,
is_test=True,
dataset=datasets[0],
batch_size=20, # For some reason, 18000/batch_size must be an integer
shuffle=not is_test,
processor=processor)
return gen
else:
data_generators = []
is_training = True
for dataset in datasets:
data_generators.append(data.DataGenerator(is_training=is_training,
is_test=False,
dataset=dataset,
batch_size=params.batch_size,
shuffle=is_training,
processor=processor))
is_training = False
return tuple(data_generators)
def main(params, date, epoch, gpu):
net = PonderEnhancer(params['model'])
ckpt = get_model_ckpt(params, date, epoch)
net.load_state_dict(torch.load(ckpt))
net.cuda()
net.eval()
# run test
test_dset = get_dataset(params['test_data_config'])
test_dataloader = get_dataloader(params, test_dset, train=False)
loss_fn = get_loss_fn(params['loss'])
fs = params['test_data_config']['fs']
per_db_results = {}
for (clean, noise, mix, file_db) in tqdm.tqdm(test_dataloader):
clean, mix = clean.cuda(), mix.cuda()
db = file_db[0][:-4]
# Train
pred, ponder = net(mix, verbose=False) # change debug -> verbose
_, loss_ponder = loss_fn(clean, pred, ponder)
if db not in per_db_results:
per_db_results[db] = {'enhance': [], 'ponder': []}
# get the perceptual metrics
np_clean = clean.detach().cpu().numpy().reshape(-1)
np_pred = pred.detach().cpu().numpy().reshape(-1)
per_db_results[db]['enhance'].append(pystoi.stoi(clean, enhanced, sr))
per_db_results[db]['ponder'].append(loss_ponder.item())
# save it all
save_dict(per_db_results, 'stoi', epoch, date, params)
def get_samelist():
print('Loading features dataframe to generate "same_list" ... ')
# make same user list, diff user list
combine = get_dataset(d_type='30days')
df_test = combine.loc[TRAIN_SHAPE:]
same_list = list(set(combine['card']) & set(df_test['card']))
del combine, df_test
gc.collect()
return same_list
def main(argv):
torch.manual_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
hlog.flags()
dataset = get_dataset()
model = pick_model(dataset)
model.prepare(dataset)
if isinstance(model, nn.Module):
path = os.path.join(FLAGS.model_dir, FLAGS.model)
checkpoint = torch.load(path)
model.load_state_dict(checkpoint)
realized = set()
examples = pick_examples(dataset)
while len(realized) < FLAGS.n_sample:
try:
templ, names = next(examples)
except StopIteration:
break
datum = make_batch([(templ, templ) for _ in range(10)],
dataset.vocab,
staged=True)
(inps, outs), scores = model.sample(datum.inp_data, datum.out_data)
keep = []
for inp, out, score in zip(inps, outs, scores):
inp_realized, inp_used = dataset.realize(inp, names)
out_realized, out_used = dataset.realize(out, names)
if ((not FLAGS.output_only)
and len(inp_used) == 0) or len(out_used) == 0:
continue
if len(inp_used | out_used) != len(names):
continue
if not ((FLAGS.output_only or dataset.novel(inp=inp_realized))
and dataset.novel(out=out_realized)):
continue
if (inp_realized, out_realized) in realized:
continue
keep.append(((inp_realized, out_realized), score))
for (inp_realized, out_realized), score in keep:
with hlog.task(str(len(realized))):
hlog.value("inp", " ".join(dataset.vocab.decode(templ[0])))
hlog.value("out", " ".join(dataset.vocab.decode(templ[1])))
hlog.value("var", names)
hlog.value("score", score)
with hlog.task("realized"):
hlog.value("inp", " ".join(inp_realized))
hlog.value("out", " ".join(out_realized))
realized.add((inp_realized, out_realized))
data = [{"inp": inp, "out": out} for inp, out in realized]
with open(FLAGS.write, "w") as fh:
json.dump(data, fh, indent=2)
def main(argv):
torch.manual_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
hlog.flags()
if not os.path.exists(FLAGS.model_dir):
os.mkdir(FLAGS.model_dir)
dataset = get_dataset()
model = ContextModel(dataset.vocab).to(_flags.device())
def callback(i_epoch):
pass
train(dataset, model, dataset.sample_ctx_train, callback, staged=False)
def main(argv):
dataset = get_dataset()
vocab = dataset.vocab
for inp, out in dataset.get_train():
print(" ".join(vocab.decode(inp)))
print(" ".join(vocab.decode(out)))
print()
print("\n\n\n\n")
for inp, out in dataset.get_val():
print(" ".join(vocab.decode(inp)))
print(" ".join(vocab.decode(out)))
print()
def retrain_models(args, old_networks, train_loader, test_loader, config, tensorboard_obj=None, initial_acc=None, nicks=None):
accuracies = []
retrained_networks = []
# nicks = []
# assert len(old_networks) >= 4
for i in range(len(old_networks)):
nick = nicks[i]
# if i == len(old_networks) - 1:
# nick = 'naive_averaging'
# elif i == len(old_networks) - 2:
# nick = 'geometric'
# else:
# nick = 'model_' + str(i)
# nicks.append(nick)
print("Retraining model : ", nick)
if initial_acc is not None:
start_acc = initial_acc[i]
else:
start_acc = -1
if args.dataset.lower()[0:7] == 'cifar10':
if args.reinit_trainloaders:
print('reiniting trainloader')
retrain_loader, _ = cifar_train.get_dataset(config, no_randomness=args.no_random_trainloaders)
else:
retrain_loader = train_loader
output_root_dir = "{}/{}_models_ensembled/".format(args.baseroot, (args.dataset).lower())
output_root_dir = os.path.join(output_root_dir, args.exp_name, nick)
os.makedirs(output_root_dir, exist_ok=True)
retrained_network, acc = cifar_train.get_retrained_model(args, retrain_loader, test_loader, old_networks[i], config, output_root_dir, tensorboard_obj=tensorboard_obj, nick=nick, start_acc=initial_acc[i])
elif args.dataset.lower() == 'mnist':
if args.reinit_trainloaders:
print('reiniting trainloader')
retrain_loader, _ = get_dataloader(args, no_randomness=args.no_random_trainloaders)
else:
retrain_loader = train_loader
start_acc = initial_acc[i]
retrained_network, acc = get_retrained_model(args, retrain_loader, test_loader, old_network=old_networks[i], tensorboard_obj=tensorboard_obj, nick=nick, start_acc=start_acc, retrain_seed=args.retrain_seed)
retrained_networks.append(retrained_network)
accuracies.append(acc)
return retrained_networks, accuracies
def get_model_activations(args,
models,
config=None,
layer_name=None,
selective=False,
personal_dataset=None):
import compute_activations
from data import get_dataloader
if args.activation_histograms and args.act_num_samples > 0:
if args.dataset == 'mnist':
unit_batch_train_loader, _ = get_dataloader(args, unit_batch=True)
elif args.dataset.lower()[0:7] == 'cifar10':
if config is None:
config = args.config # just use the config in arg
unit_batch_train_loader, _ = cifar_train.get_dataset(
config, unit_batch_train=True)
if args.activation_mode is None:
activations = compute_activations.compute_activations_across_models(
args, models, unit_batch_train_loader, args.act_num_samples)
else:
if selective and args.update_acts:
activations = compute_activations.compute_selective_activation(
args, models, layer_name, unit_batch_train_loader,
args.act_num_samples)
else:
if personal_dataset is not None:
# personal training set is passed which consists of (inp, tgts)
print('using the one from partition')
loader = partition.to_dataloader_from_tens(
personal_dataset[0], personal_dataset[1], 1)
else:
loader = unit_batch_train_loader
activations = compute_activations.compute_activations_across_models_v1(
args,
models,
loader,
args.act_num_samples,
mode=args.activation_mode)
else:
activations = None
return activations
def main(argv):
torch.manual_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
hlog.flags()
if FLAGS.augment is not None:
with open(FLAGS.augment) as fh:
aug_data = json.load(fh)
else:
aug_data = []
dataset = get_dataset(aug_data=aug_data, invert=FLAGS.invert)
model = GeneratorModel(dataset.vocab, copy=True,
self_attention=False).to(_flags.device())
fine_tune = [True]
def sample():
if fine_tune[0]:
return dataset.sample_train(aug_ratio=FLAGS.aug_ratio)
else:
return dataset.sample_train(aug_ratio=FLAGS.aug_ratio)
def callback(i_epoch):
if not fine_tune[0] and i_epoch >= 20:
hlog.log("FINE_TUNE")
fine_tune[0] = True
model.eval()
final = i_epoch == FLAGS.n_epochs - 1
with hlog.task("eval_train", timer=False):
train_data = [dataset.sample_train() for _ in range(1000)]
evaluate(model, train_data, dataset)
with hlog.task("eval_val", timer=False):
val_data = dataset.get_val()
val_acc = evaluate(model, val_data, dataset, vis=final, beam=final)
if FLAGS.TEST and (final or FLAGS.test_curve):
with hlog.task("eval_test", timer=False):
test_data = dataset.get_test()
evaluate(model, test_data, dataset, beam=final)
if (i_epoch + 1) % FLAGS.n_checkpoint == 0:
torch.save(
model.state_dict(),
os.path.join(FLAGS.model_dir, "model.%05d.chk" % i_epoch))
return val_acc
train(dataset, model, sample, callback, staged=False)
def main(argv):
torch.manual_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
hlog.flags()
if FLAGS.augment is not None:
with open(FLAGS.augment) as fh:
aug_data = json.load(fh)
else:
aug_data = []
dataset = get_dataset(aug_data=aug_data)
if FLAGS.use_mkn:
mkn_main(dataset)
else:
rnn_main(dataset)
def main(argv):
hlog.flags()
assert FLAGS.augment is not None
with open(FLAGS.augment) as fh:
aug_data = json.load(fh)
dataset = get_dataset()
train_data = dataset.get_train()
utts = []
if FLAGS.concat:
for _, utt in train_data:
s = " ".join(dataset.vocab.decode(utt))
utts.append(s)
for pair in aug_data:
s = " ".join(pair["out"])
utts.append(s)
with open(FLAGS.write, "w") as fh:
for utt in utts:
print(utt, file=fh)
def main():
LAYERS = 3
pkl_fname = "data/preprocess/stage1_train_set_rgb.pkl"
images, masks = get_dataset(pkl_fname)
logging.info("read train set: %s, %s", images.shape, masks.shape)
logging.info("image:[%s, %s], mask:[%s, %s]", np.max(images), np.min(images), np.max(masks), np.min(masks))
# pred_size, offset = unet_size(256, LAYERS)
# logging.info("pred_size: %d, offset: %d", pred_size, offset)
# images = padding_array(images, offset, default_val=0.0)
# masks = padding_array(masks, offset, default_val=False)
# args.data_dir = args.data_dir.strip()
# if len(args.data_dir) >= 0:
# fnames = [os.path.join(args.data_dir, x) for x in fnames]
train_ratio = 0.9
n_train = int(len(images)*train_ratio)
logging.info("train_ratio: %s, n_train: %s, n_val: %s", train_ratio, n_train, len(images)-n_train)
convert_dataset(images[:n_train], masks[:n_train], "data/tfrecords/256x256/train", 4)
convert_dataset(images[n_train:], masks[n_train:], "data/tfrecords/256x256/val", 2)
def main():
ap = argparse.ArgumentParser()
ap.add_argument('-w',
'--working_dir',
default='train',
help='Where to save checkpoints and logs')
ap.add_argument(
'-i',
'--predict_input_dir',
help='Directory of files produced by the preprocessing script')
ap.add_argument(
'-o',
'--predict_output_dir',
help='Directory of files produced by the preprocessing script')
ap.add_argument(
'-r',
'--predict_raw_dir',
help='Directory of files produced by the preprocessing script')
ap.add_argument('-p',
'--checkpoint_number',
default=9,
help='checkpoint path of model')
ap.add_argument('-b',
'--balance',
type=float,
default=0,
help='The dropout percentage')
args = ap.parse_args()
# args = ap.parse_args(args=["../googletrends_japanese_data_5000","--working_dir","../googletrends_japanese_train_5000", "--test_dir", "../local_data_japan"])
params_path = os.path.join(args.working_dir, 'params.csv')
params = {}
with open(params_path, 'r') as f:
for i in csv.reader(f):
params[i[0]] = i[1]
DATA_DIR = params["DATA_DIR"]
#文章をノードごとに分割
file_path = args.predict_raw_dir
doc_data, _, _ = parse_doc(util.get_filenames(file_path))
info_file = os.path.join(DATA_DIR, 'info.pkl')
with open(info_file, 'rb') as fp:
info = pickle.load(fp)
info['num_train_examples'] = len(doc_data)
train_steps = math.ceil(info['num_train_examples'] /
int(params["batch_size"]))
predict_set_file = os.path.join(args.predict_input_dir, 'train.tfrecords')
predict_dataset = get_dataset(predict_set_file,
1,
repeat=False,
shuffle=False)
kwargs = {
'input_size': info['num_words'] + info['num_tags'],
'hidden_size': int(params['hidden_units']),
'num_layers': int(params['num_layers']),
'dropout': float(params['dropout']),
'dense_size': int(params['dense_size'])
}
clf = LeafClassifier(**kwargs)
checkpoint_path = args.working_dir + "/ckpt/model.{:03d}.h5".format(
int(args.checkpoint_number))
clf.model = load_model(checkpoint_path)
_, y_pred = clf.eval_recall(predict_dataset,
train_steps,
desc="",
balance=args.balance)
filenames = util.get_filenames(file_path)
raw_filenames = [i.split("/")[-1] for i in filenames]
#予測結果を保存
pred_index = [bool(i) for i in y_pred]
delete_index = [not bool(i) for i in y_pred]
counter = 0
for i in range(len(raw_filenames)):
numpy_data = np.array(doc_data[raw_filenames[i]])
contents = numpy_data[pred_index[counter:counter + len(numpy_data)]]
delete_contents = numpy_data[delete_index[counter:counter +
len(numpy_data)]]
counter += len(numpy_data)
content_text = ""
delete_text = ""
for content in contents:
content_text += str(content) + "\n"
for delete_content in delete_contents:
delete_text += str(delete_content) + "\n"
dir_path = args.predict_output_dir
os.makedirs(dir_path, exist_ok=True)
with open(
os.path.join(
dir_path,
raw_filenames[i].replace("/", "_").replace(".", "_") +
".txt"), 'w') as file:
file.write(content_text)
with open(
os.path.join(
dir_path,
raw_filenames[i].replace("/", "_").replace(".", "_") +
"delete.txt"), 'w') as file:
file.write(delete_text)
def get_dataloader(args, unit_batch = False, no_randomness=False):
if unit_batch:
bsz = (1, 1)
else:
bsz = (args.batch_size_train, args.batch_size_test)
if no_randomness:
enable_shuffle = False
else:
enable_shuffle = True
if args.dataset.lower() == 'mnist':
train_loader = torch.utils.data.DataLoader(
torchvision.datasets.MNIST('./files/', train=True, download=args.to_download,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize(
# only 1 channel
(0.1307,), (0.3081,))
])),
batch_size=bsz[0], shuffle=enable_shuffle
)
test_loader = torch.utils.data.DataLoader(
torchvision.datasets.MNIST('./files/', train=False, download=args.to_download,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize(
(0.1307,), (0.3081,))
])),
batch_size=bsz[1], shuffle=enable_shuffle
)
return train_loader, test_loader
elif args.dataset.lower() == 'cifar10':
if args.cifar_style_data:
train_loader, test_loader = cifar_train.get_dataset(args.config)
else:
train_loader = torch.utils.data.DataLoader(
torchvision.datasets.CIFAR10('./data/', train=True, download=args.to_download,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize(
# Note this normalization is not same as in MNIST
# (mean_ch1, mean_ch2, mean_ch3), (std1, std2, std3)
(0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])),
batch_size=bsz[0], shuffle=enable_shuffle
)
test_loader = torch.utils.data.DataLoader(
torchvision.datasets.CIFAR10('./data/', train=False, download=args.to_download,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize(
# (mean_ch1, mean_ch2, mean_ch3), (std1, std2, std3)
(0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])),
batch_size=bsz[1], shuffle=enable_shuffle
)
return train_loader, test_loader
# checkpoint_type = 'final' # which checkpoint to use for ensembling (either of 'best' or 'final)
if args.dataset == 'mnist':
train_loader, test_loader = get_dataloader(args)
retrain_loader, _ = get_dataloader(
args, no_randomness=args.no_random_trainloaders)
elif args.dataset.lower()[0:7] == 'cifar10':
args.cifar_init_lr = config['optimizer_learning_rate']
if args.second_model_name is not None:
assert second_config is not None
assert args.cifar_init_lr == second_config[
'optimizer_learning_rate']
# also the below things should be fine as it is just dataloader loading!
print('loading {} dataloaders'.format(args.dataset.lower()))
train_loader, test_loader = cifar_train.get_dataset(config)
retrain_loader, _ = cifar_train.get_dataset(
config, no_randomness=args.no_random_trainloaders)
models = []
accuracies = []
for idx in range(args.num_models):
print("loading model with idx {} and checkpoint_type is {}".format(
idx, args.ckpt_type))
if args.dataset.lower()[0:7] == 'cifar10' and (
args.model_name.lower()[0:5] == 'vgg11'
or args.model_name.lower()[0:6] == 'resnet'):
if idx == 0:
config_used = config
def finetune(epochs, train_steps):
warmup_steps = int(train_steps * epochs * 0.1)
train_dataset = get_dataset()
model = load_model(train_steps, warmup_steps)
train(model, train_dataset, epochs, train_steps)
def main():
parser = ArgumentParser()
parser.add_argument("--num_tweets", type=int, default=5, help="Number of tweets to generate")
parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device (cuda or cpu)")
parser.add_argument("--no_sample", action='store_true', help="Set to use greedy decoding instead of sampling")
parser.add_argument("--max_length", type=int, default=40, help="Maximum length of the output utterances")
parser.add_argument("--min_length", type=int, default=10, help="Minimum length of the output utterances")
parser.add_argument("--seed", type=int, default=0, help="Seed")
parser.add_argument("--temperature", type=int, default=0.7, help="Sampling softmax temperature")
parser.add_argument("--top_k", type=int, default=4, help="Filter top-k tokens before sampling (<=0: no filtering)")
args = parser.parse_args()
model = 'gpt2'
p = Path('./runs')
posibilities = [x for x in p.iterdir() if x.is_dir()]
print('Please Select a Checkpoint: ')
for i, check in enumerate(posibilities):
print(f'{i}) {str(check)}')
selection = int(input('> '))
model_checkpoint = posibilities[selection]
distractor_path = "Elizabeth Warren.csv"
distractor = []
with open(distractor_path) as csvfile:
reader = csv.reader(csvfile)
for row in reader:
distractor.append(row)
def clean(data):
return [tweet[1:] for tweet in data if tweet[2].count(' ') > 3][1:]
distractor = clean(distractor)
num_distractor = len(distractor)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__file__)
logger.info(pformat(args))
if args.seed != 0:
random.seed(args.seed)
torch.random.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logger.info("Get pretrained model and tokenizer")
tokenizer_class, model_class = GPT2Tokenizer, GPT2LMHeadModel
tokenizer = tokenizer_class.from_pretrained(model_checkpoint)
model = model_class.from_pretrained(model_checkpoint)
model.to(args.device)
add_special_tokens_(model, tokenizer)
logger.info("Sample a twitter user")
dataset = get_dataset(tokenizer)
personalities = [dialog["name"] for dataset in dataset.values() for dialog in dataset]
personality = random.choice(personalities)
name = tokenizer.decode(personality)
logger.info("Selected personality: %s", name)
for i in range(args.num_tweets):
base_tweet = distractor[random.randint(0, num_distractor-1)]
context = tokenizer.encode(base_tweet[2])
with torch.no_grad():
out_ids = sample_sequence(personality, context, tokenizer, model, args)
out_text = tokenizer.decode(out_ids, skip_special_tokens=True)
print(f'\n\nTweet {i})')
print(f"Original {base_tweet[0]} Tweet:")
print(base_tweet[1])
print('\nContext:')
print(base_tweet[2])
print(f"\nGenerated {name} Tweet: ")
print(out_text)
def main(args):
utils.init_distributed_mode(args)
print(args)
device = torch.device(args.device)
# Data loading code
print("Loading data")
dataset, num_classes = get_dataset(args.dataset, "train", get_transform(train=True), args.data_path)
dataset_test, _ = get_dataset(args.dataset, "val", get_transform(train=False), args.data_path)
print("Creating data loaders")
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test)
else:
train_sampler = torch.utils.data.RandomSampler(dataset)
test_sampler = torch.utils.data.SequentialSampler(dataset_test)
if args.aspect_ratio_group_factor >= 0:
group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor)
train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size)
else:
train_batch_sampler = torch.utils.data.BatchSampler(
train_sampler, args.batch_size, drop_last=True)
data_loader = torch.utils.data.DataLoader(
dataset, batch_sampler=train_batch_sampler, num_workers=args.workers,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(
dataset_test, batch_size=5,
sampler=test_sampler, num_workers=args.workers,
collate_fn=utils.collate_fn)
print("Creating model")
model = torchvision.models.detection.__dict__[args.model](num_classes=num_classes,
pretrained=args.pretrained)
model.to(device)
model_without_ddp = model
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu])
model_without_ddp = model.module
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(
params, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.lr_step_size, gamma=args.lr_gamma)
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_gamma)
if args.resume:
checkpoint = torch.load(args.resume, map_location='cpu')
model_without_ddp.load_state_dict(checkpoint['model'])
optimizer.load_state_dict(checkpoint['optimizer'])
lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
args.start_epoch = checkpoint['epoch'] + 1
if args.test_only:
evaluate(model, data_loader_test, device=device)
return
print("Start training")
start_time = time.time()
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
train_one_epoch(model, optimizer, data_loader, device, epoch, args.print_freq)
lr_scheduler.step()
if args.output_dir:
utils.save_on_master({
'model': model_without_ddp.state_dict(),
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict(),
'args': args,
'epoch': epoch},
os.path.join(args.output_dir, 'model_{}.pth'.format(epoch)))
# evaluate after every epoch
evaluate(model, data_loader_test, device=device)
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
print('Training time {}'.format(total_time_str))
def main():
np.random.seed(12345)
LAYERS = 3
padding_pkl_fname = "data/preprocess/stage1_train_set_padding.pkl"
dump = False
if dump:
pkl_fname = "data/preprocess/stage1_train_set.pkl"
images, masks = get_dataset(pkl_fname)
logging.info("read train set: %s, %s", images.shape, masks.shape)
logging.info("image:[%s, %s], mask:[%s, %s]", np.max(images),
np.min(images), np.max(masks), np.min(masks))
pred_size, offset = unet_size(256, LAYERS)
logging.info("pred_size: %d, offset: %d", pred_size, offset)
images = padding_array(images, offset, default_val=0.0)
masks = padding_array(masks, offset, default_val=False)
logging.info("shape after padded: %s, %s", images.shape, masks.shape)
with open(padding_pkl_fname, "wb") as f:
pickle.dump((images, masks), f, protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(padding_pkl_fname, "rb") as f:
images, masks = pickle.load(f)
# test_data(images, masks, 1679)
data_provider = image_util.SimpleDataProvider(images, masks)
logging.info("data_provider.channels: %s, data_provider.n_class: %s",
data_provider.channels, data_provider.n_class)
# test_data_provider(data_provider)
net = unet.Unet(
channels=data_provider.channels,
n_class=data_provider.n_class,
cost='cross_entropy',
layers=LAYERS,
features_root=64,
cost_kwargs=dict(regularizer=0.001),
)
net.load_weight("log/20180414/model.cpkt")
for i in range(2):
x_test, y_test = data_provider(1)
prediction = net.infer(x_test)
logging.info("%s, %s", x_test.shape, y_test.shape)
mask = prediction[0, ..., 1] > 0.3
img_cropped = crop_to_shape(x_test[0, ..., 0], mask.shape)
gt = y_test[0, ..., 1]
gt_cropped = crop_to_shape(gt, mask.shape)
fig, ax = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(12, 5))
ax[0].imshow(img_cropped, aspect="auto")
ax[1].imshow(gt_cropped, aspect="auto")
ax[2].imshow(mask, aspect="auto")
ax[0].set_title("Input")
ax[1].set_title("Ground truth")
ax[2].set_title("Prediction")
fig.tight_layout()
logging.info("%s, %s", mask.shape, gt_cropped.shape)
logging.info("dice: %f", calc_dice(mask, gt_cropped))
plt.show()
import torch import time import matplotlib.pyplot as plt from sklearn.decomposition import PCA from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from sklearn.manifold import TSNE from train import get_dataset NUM = 2000 pca_cls = PCA(n_components=3) # pca_cls = LDA(n_components=3) # pca_cls = TSNE(n_components=3) train_loader, test_loader = get_dataset(1, 1, 6000) train_x = torch.flatten(train_loader.dataset.data, 1).numpy() train_y = train_loader.dataset.targets.numpy() test_x = torch.flatten(test_loader.dataset.data, 1).numpy()[:NUM] test_y = test_loader.dataset.targets.numpy()[:NUM] start = time.time() pca_cls.fit(train_x, train_y) print(time.time() - start) start = time.time() new_X = pca_cls.transform(test_x) # new_X = pca_cls.fit_transform(test_x) new_y = test_y print(time.time() - start)
def train_ray(opt, checkpoint_dir=None, data_dir="../data"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = get_dataset(opt)
g = data[0]
if opt['gpu'] < 0:
cuda = False
else:
cuda = True
g = g.int().to(opt['gpu'])
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_classes
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([opt['num_heads']] * opt['num_layers']) + [opt['num_out_heads']]
models = []
optimizers = []
datas = [g for i in range(opt['num_init'])]
for split in range(opt['num_init']):
if opt['model'] == 'GAT':
model = GAT(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, heads, F.elu, opt['in_drop'],
opt['attn_drop'], opt['negative_slope'],
opt['residual'], opt)
elif opt['model'] == 'AGNN':
model = AGNN(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, opt['in_drop'], opt)
train_this = train
model = model.to(device)
models.append(model)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
# model = model.to(device)
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = get_optimizer(opt['optimizer'],
parameters,
lr=opt['lr'],
weight_decay=opt['weight_decay'])
optimizers.append(optimizer)
# The `checkpoint_dir` parameter gets passed by Ray Tune when a checkpoint
# should be restored.
if checkpoint_dir:
checkpoint = os.path.join(checkpoint_dir, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint)
model.load_state_dict(model_state)
optimizer.load_state_dict(optimizer_state)
for epoch in range(1, opt['epochs']):
loss = np.mean([
train_this(model, optimizer, features, train_mask,
labels)[0].item()
for model, optimizer in zip(models, optimizers)
])
train_accs, val_accs, tmp_test_accs = average_test(models, datas)
with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
best = np.argmax(val_accs)
path = os.path.join(checkpoint_dir, "checkpoint")
torch.save(
(models[best].state_dict(), optimizers[best].state_dict()),
path)
tune.report(loss=loss,
accuracy=np.mean(val_accs),
test_acc=np.mean(tmp_test_accs),
train_acc=np.mean(train_accs))
def train_ray_int(opt, checkpoint_dir=None, data_dir="../data"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = get_dataset(opt)
g = data[0]
if opt['gpu'] < 0:
cuda = False
else:
cuda = True
g = g.int().to(opt['gpu'])
# if opt["num_splits"] > 0:
# dataset.data = set_train_val_test_split(
# 23 * np.random.randint(0, opt["num_splits"]), # random prime 23 to make the splits 'more' random. Could remove
# dataset.data,
# num_development=5000 if opt["dataset"] == "CoauthorCS" else 1500)
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_classes
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([opt['num_heads']] * opt['num_layers']) + [opt['num_out_heads']]
if opt['model'] == 'GAT':
model = GAT(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, heads, F.elu, opt['in_drop'], opt['attn_drop'],
opt['negative_slope'], opt['residual'], opt)
elif opt['model'] == 'AGNN':
model = AGNN(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, opt['in_drop'], opt)
model = model.to(device)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = get_optimizer(opt["optimizer"],
parameters,
lr=opt["lr"],
weight_decay=opt["weight_decay"])
if checkpoint_dir:
checkpoint = os.path.join(checkpoint_dir, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint)
model.load_state_dict(model_state)
optimizer.load_state_dict(optimizer_state)
train_this = train
this_test = test_OGB if opt['dataset'] == 'ogbn-arxiv' else test
best_time = best_epoch = train_acc = val_acc = test_acc = 0
for epoch in range(1, opt["epoch"]):
# loss = train(model, optimizer, data)
loss = train_this(model, optimizer, features, train_mask,
labels)[0].item()
if opt["no_early"]:
tmp_train_acc, tmp_val_acc, tmp_test_acc = this_test(model, g)
best_time = opt['time']
else:
tmp_train_acc, tmp_val_acc, tmp_test_acc = this_test(model, g)
if tmp_val_acc > val_acc:
best_epoch = epoch
train_acc = tmp_train_acc
val_acc = tmp_val_acc
test_acc = tmp_test_acc
with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
path = os.path.join(checkpoint_dir, "checkpoint")
torch.save((model.state_dict(), optimizer.state_dict()), path)
tune.report(loss=loss,
accuracy=val_acc,
test_acc=test_acc,
train_acc=train_acc,
best_time=best_time,
best_epoch=best_epoch)
ensemble_root_dir = "{}/{}_models/".format(args.baseroot, (args.dataset).lower())
ensemble_dir = ensemble_root_dir + args.load_models
utils.mkdir(ensemble_dir)
# checkpoint_type = 'final' # which checkpoint to use for ensembling (either of 'best' or 'final)
if args.dataset=='mnist':
train_loader, test_loader = get_dataloader(args)
elif args.dataset.lower() == 'cifar10':
args.cifar_init_lr = config['optimizer_learning_rate']
if args.second_model_name is not None:
assert second_config is not None
assert args.cifar_init_lr == second_config['optimizer_learning_rate']
# also the below things should be fine as it is just dataloader loading!
print('loading {} dataloaders'.format(args.dataset.lower()))
train_loader, test_loader = cifar_train.get_dataset(config)
models = []
accuracies = []
local_accuracies = []
for idx in range(args.num_models):
print("loading model with idx {} and checkpoint_type is {}".format(idx, args.ckpt_type))
if args.dataset.lower()[0:7] == 'cifar10' and (args.model_name.lower()[0:5] == 'vgg11' or args.model_name.lower()[0:6] == 'resnet'):
if idx == 0:
config_used = config
elif idx == 1:
config_used = second_config
model, accuracy = cifar_train.get_pretrained_model(
config_used, os.path.join(ensemble_dir, 'model_{}/{}.checkpoint'.format(idx, args.ckpt_type)),
def run():
parser = ArgumentParser()
parser.add_argument("--dataset_cache",
type=str,
default='./dataset.bin',
help="Path or url of the dataset cache")
parser.add_argument(
"--model",
type=str,
default="openai-gpt",
help="Model type (openai-gpt or gpt2)",
choices=['openai-gpt',
'gpt2']) # anything besides gpt2 will load openai-gpt
parser.add_argument("--model_checkpoint",
type=str,
default="",
help="Path, url or short name of the model")
parser.add_argument(
"--max_history",
type=int,
default=2,
help="Number of previous utterances to keep in history")
parser.add_argument("--device",
type=str,
default="cuda" if torch.cuda.is_available() else "cpu",
help="Device (cuda or cpu)")
parser.add_argument("--no_sample",
action='store_true',
help="Set to use greedy decoding instead of sampling")
parser.add_argument("--max_length",
type=int,
default=20,
help="Maximum length of the output utterances")
parser.add_argument("--min_length",
type=int,
default=1,
help="Minimum length of the output utterances")
parser.add_argument("--seed", type=int, default=0, help="Seed")
parser.add_argument("--temperature",
type=int,
default=0.7,
help="Sampling softmax temperature")
parser.add_argument(
"--top_k",
type=int,
default=0,
help="Filter top-k tokens before sampling (<=0: no filtering)")
parser.add_argument(
"--top_p",
type=float,
default=0.9,
help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__file__)
logger.info(pformat(args))
if args.model_checkpoint == "":
raise ValueError(
"Interacting with Model requires passing a finetuned model_checkpoint"
)
if args.seed != 0:
random.seed(args.seed)
torch.random.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logger.info("Get pretrained model and tokenizer")
tokenizer_class, model_class = (GPT2Tokenizer, GPT2LMHeadModel)
tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
model = model_class.from_pretrained(args.model_checkpoint)
model.to(args.device)
add_special_tokens_(model, tokenizer)
logger.info("Sample a twitter user")
dataset = get_dataset(tokenizer)
personalities = [
dialog["name"] for dataset in dataset.values() for dialog in dataset
]
personality = random.choice(personalities)
logger.info("Selected personality: %s", tokenizer.decode(personality))
previous_tweet = []
while True:
raw_text = input("Please enter a previous tweet: ")
while not raw_text:
print('Prompt should not be empty!')
raw_text = input("Please enter a previous tweet: ")
previous_tweet = tokenizer.encode(raw_text)
with torch.no_grad():
out_ids = sample_sequence(personality, previous_tweet, tokenizer,
model, args)
out_text = tokenizer.decode(out_ids, skip_special_tokens=True)
print('New Tweet:')
print(out_text)
def main(argv):
for i in range(10):
with open(AUG % i) as fh:
aug_data = json.load(fh)
dataset = get_dataset(aug_data=aug_data)
