def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset(
batch_size=batch_size)
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(int(dataset_size / 2))
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(int(dataset_size / 2))):
loss_identity, kl_loss = train_step(self, source, optimizer)
if batch_on % record_batch == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size / 2), 'of size', batch_size)
self.loss_identity += [loss_identity]
self.kl_loss += [kl_loss]
batch_on += 1
duration = time.time() - start
print(int(duration / 60), "minutes &", int(duration % 60),
"seconds, for epoch", i)
if i % record_epochs == 0:
self.loss['Identity'] = self.loss_identity
self.loss['KL'] = self.kl_loss
utils.test_model(self,
source,
num=i,
test=True,
name=model_name,
details='identity')
for style in zip(image_dataset.take(1)):
utils.test_model(self,
source,
style,
num=i,
test=True,
name=model_name,
details='transfer')
break
#act = keract.get_activations(self, source)
#keract.display_activations(act)
print('\n')
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
self.save(fname)
time.sleep(.5)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
model_val = AE_A(training=False)
image_dataset, dataset_size = utils.create_dataset(batch_size=batch_size)
dataset_size = int((dataset_size - 1)/2)
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(dataset_size)
start_time = time.time()
for i in range(num_epochs):
print(f'Starting epoch {i}/{num_epochs}')
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(dataset_size)):
try:
loss_identity, kl_loss = self.train_step(source, optimizer)
if batch_on % record_batch == 0:
print(f'Beginning batch #{batch_on} out of {dataset_size} of size {batch_size}')
self.loss_identity += [loss_identity]
self.kl_loss += [kl_loss]
except Exception:
print(f'Batch #{batch_on} failed. Continuing with next batch.')
batch_on += 1
self.save(fname)
time.sleep(0.5)
duration = time.time() - start
utils.print_time_remaining(i, num_epochs, duration)
if i % display_mod == 0:
model_val.load(model_name, compile=False)
model_val.loss['Identity'] = self.loss_identity
model_val.loss['KL'] = self.kl_loss
utils.test_model(model_val, source, num=i, test=True, name=model_name, details='identity')
for style in zip(image_dataset.take(1)):
utils.test_model(model_val, source, style, num=i, test=True, name=model_name, details='merge')
break
#act = keract.get_activations(self, source)
#keract.display_activations(act)
print('\n')
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset(
batch_size=batch_size)
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(int(dataset_size))
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(int(dataset_size))):
source = utils.get_random_crop(np.array(source), 32, 32)
loss_identity = train_step(self, source, optimizer)
if batch_on % record_batch == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size), 'of size', batch_size)
self.loss_identity += [loss_identity]
batch_on += 1
if i % record_epochs == 0:
self.loss['Identity'] = self.loss_identity
utils.test_model(self,
source,
num=i,
test=True,
name=model_name,
details='identity')
'''for style in zip(image_dataset.take(1)):
style = utils.get_random_crop(style, 32, 32)
utils.test_model(self, source, style, num=i, test=True, name=model_name, details='transfer')
break'''
print('\n')
duration = time.time() - start
utils.print_time_remaining(i, num_epochs, duration)
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
self.save(fname)
time.sleep(.5)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def standard_train(args, input_var, network, tang_output):
"""Train a network normally, output the network, a function to make predictions and training losses"""
# Create data
X = utils.create_dataset(args.npts)
# Create a list of batches (a list of batch idxs splitting X in batches of size batch_size)
list_batches = utils.get_list_batches(args.npts, args.batch_size)
# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.squared_error(prediction, tang_output)
loss = loss.mean()
# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=args.learning_rate, momentum=0.9)
# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var], loss, updates=updates)
# train network
list_loss = utils.train_network(train_fn, X, list_batches, args.nb_epoch)
# Create a prediction function to evaluate after training
predict_fn = utils.get_prediction_fn(input_var, network)
return network, predict_fn, list_loss
def evaluate_test_set(model, test, x_to_ix, y_to_ix):
y_true = list()
y_pred = list()
for batch, targets, lengths, raw_data in utils.create_dataset(test, x_to_ix, y_to_ix, batch_size=1):
batch, targets, lengths = utils.sort_batch(batch, targets, lengths)
pred = model(torch.autograd.Variable(batch), lengths.cpu().numpy())
pred_idx = torch.max(pred, 1)[1]
y_true += list(targets.int())
y_pred += list(pred_idx.data.int())
print(len(y_true), len(y_pred))
print(classification_report(y_true, y_pred))
print(confusion_matrix(y_true, y_pred))
y_test = y_true
y_test_pred = y_pred
name = 'test'
print(f'{name} set, Counter(y): {counter(y_test)}')
print(f'cm of {name} set: ', metrics.confusion_matrix(y_true=y_test, y_pred=y_test_pred))
report = metrics.classification_report(y_test, y_test_pred)
# print(report)
model.accuracy = metrics.accuracy_score(y_test, y_test_pred)
model.F1 = metrics.f1_score(y_test, y_test_pred, average='weighted')
model.recall = metrics.recall_score(y_test, y_test_pred, average='weighted')
model.precision = metrics.precision_score(y_test, y_test_pred, average='weighted')
print(f'{name} set, accuracy: {model.accuracy}, F1: {model.F1}, recall: {model.recall}, '
f'precision: {model.precision}')
def train(batch_size, ctx, num_epochs, path, lr=1e-4, wd=1e-5, params_file="autoencoder_ucsd_convae.params"):
# Dataloader for training dataset
dataloader = utils.create_dataset(path, batch_size, shuffle=True)
# Get model
model = ConvolutionalAutoencoder()
model.hybridize()
# Initialiize
model.collect_params().initialize(mx.init.Xavier('gaussian'), ctx=ctx)
# Loss
l2loss = gluon.loss.L2Loss()
optimizer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': lr, 'wd': wd})
# Start training loop
for epoch in range(num_epochs):
for image in dataloader:
image = image.as_in_context(ctx)
with mx.autograd.record():
reconstructed = model(image)
loss = l2loss(reconstructed, image)
loss.backward()
optimizer.step(batch_size)
print('epoch [{}/{}], loss:{:.4f}'.format(epoch + 1, num_epochs, mx.nd.mean(loss).asscalar()))
# Save parameters
model.save_parameters(params_file)
return model, params_file
def main(unused_argv):
# Export model
export_tf_model(FLAGS.export_path)
# Find latest frozen pb
subdirs = [x for x in Path(FLAGS.export_path + '/frozen_pb').iterdir()
if x.is_dir() and 'temp' not in str(x)]
latest = str(sorted(subdirs)[-1])
# Create predictor
predict_fn = predictor.from_saved_model(latest)
dataset = create_dataset(path=FLAGS.test_path,
buffer_size=25,
batch_size=25,
num_epochs=1)
iterator = dataset.make_one_shot_iterator()
# Eager execution for obtaining batch data from dataset
x_val = iterator.get_next().numpy()
z_val = np.zeros([x_val.shape[0], z_dim])
dict_in = {'x': x_val, 'z':z_val}
# Make predictions and fetch results from output dict
predictions = predict_fn(dict_in)
x = predictions['x']
y = predictions['y']
# Show all source v.s. generated results
compare_all(x, y, x.shape[0])
def train_model(model, optimizer, train, dev, x_to_ix, y_to_ix, batch_size,
max_epochs):
criterion = nn.NLLLoss(size_average=False)
for epoch in range(max_epochs):
print('Epoch:', epoch)
y_true = list()
y_pred = list()
total_loss = 0
for batch, targets, lengths, raw_data in utils.create_dataset(
train, x_to_ix, y_to_ix, batch_size=batch_size):
batch, targets, lengths = utils.sort_batch(batch, targets, lengths)
model.zero_grad()
pred, loss = apply(model, criterion, batch, targets, lengths)
loss.backward()
optimizer.step()
pred_idx = torch.max(pred, 1)[1]
y_true += list(targets.int())
y_pred += list(pred_idx.data.int())
total_loss += loss
acc = accuracy_score(y_true, y_pred)
val_loss, val_acc = evaluate_validation_set(model, dev, x_to_ix,
y_to_ix, criterion)
print(
"Train loss: {} - acc: {} \nValidation loss: {} - acc: {}".format(
total_loss.data.float() / len(train), acc, val_loss, val_acc))
return model
def main(learning_rate, epochs, hidden_units):
"""
feature_columns is a list and contains two dict：
- dense_features: {feat: dense_feature_name}
- sparse_features: {feat: sparse_feature_name, feat_num: the number of this feature}
train_X: [dense_train_X, sparse_train_X]
test_X: [dense_test_X, sparse_test_X]
"""
feature_columns, train_X, test_X, train_y, test_y = create_dataset()
# ============================Build Model==========================
model = Deep_Crossing(feature_columns, hidden_units)
model.summary()
# ============================model checkpoint======================
check_path = 'save/deep_crossing_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path,
save_weights_only=True,
verbose=1,
period=5)
# =========================Compile============================
model.compile(loss=binary_crossentropy,
optimizer=Adam(learning_rate=learning_rate),
metrics=[AUC()])
# ===========================Fit==============================
model.fit(train_X,
train_y,
epochs=epochs,
callbacks=[checkpoint],
batch_size=128,
validation_split=0.2)
# ===========================Test==============================
print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
def test_09_test_dosmodes():
modes = ['readonly', 'hidden', 'system', 'archive', 'offline', 'sparse']
ds_name = 'dosmode_test'
target = f'{pool_name}/{ds_name}'
path = f'/mnt/{target}'
testpaths = [
f'{path}/testfile',
f'{path}/testdir',
]
with create_dataset(target):
cmd = [f'touch {testpaths[0]}', f'mkdir {testpaths[1]}']
results = SSH_TEST(' && '.join(cmd), user, password, ip)
assert results['result'] is True, str(results)
for p in testpaths:
results = POST('/filesystem/get_dosmode', p)
assert results.status_code == 200, results
expected_flags = results.json()
for m in modes:
to_set = {m: not expected_flags[m]}
results = POST('/filesystem/set_dosmode', {
'path': p,
'dosmode': to_set
})
assert results.status_code == 200, results.text
expected_flags.update(to_set)
results = POST('/filesystem/get_dosmode', p)
assert results.status_code == 200, results
assert results.json() == expected_flags
def standard_train(args, input_var, network, tang_output):
"""Train a network normally, output the network, a function to make predictions and training losses"""
# Create data
X = utils.create_dataset(args.npts)
# Create a list of batches (a list of batch idxs splitting X in batches of size batch_size)
list_batches = utils.get_list_batches(args.npts, args.batch_size)
# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.squared_error(prediction, tang_output)
loss = loss.mean()
# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(
loss, params, learning_rate=args.learning_rate, momentum=0.9)
# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var], loss, updates=updates)
# train network
list_loss = utils.train_network(train_fn, X, list_batches, args.nb_epoch)
# Create a prediction function to evaluate after training
predict_fn = utils.get_prediction_fn(input_var, network)
return network, predict_fn, list_loss
def train_model(self, fname=None):
time.sleep(5)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset()
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source, style in zip(image_dataset.take(int(dataset_size / 4)),
image_dataset.take(int(dataset_size /
4))):
try:
loss_content, loss_style, loss_identity = self.train_step(
source, style, optimizer)
if batch_on % 10 == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size / 4), 'of size', batch_size)
self.loss_content += [loss_content]
self.loss_style += [loss_style]
self.loss_identity += [loss_identity]
except Exception:
print("Batch #", batch_on,
"failed. Continuing with next batch.")
batch_on += 1
duration = time.time() - start
print(int(duration / 60), "minutes &", int(duration % 60),
"seconds, for epoch", i)
if i % 20 == 0:
utils.test_model(self, source, style, num=i, name=model_name)
print('\n')
image_dataset, _ = utils.create_dataset()
self.save(fname)
time.sleep(1)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def test_07_test_filesystem_stat_filetype(request):
"""
This test checks that file types are properly
identified through the filesystem plugin in middleware.
There is an additional check to make sure that paths
in the ZFS CTL directory (.zfs) are properly flagged.
"""
depends(request, ["pool_04"], scope="session")
ds_name = 'stat_test'
snap_name = f'{ds_name}_snap1'
path = f'/mnt/{pool_name}/{ds_name}'
targets = ['file', 'directory', 'symlink', 'other']
cmds = [
f'mkdir {path}/directory', f'touch {path}/file',
f'ln -s {path}/file {path}/symlink', f'mkfifo {path}/other'
]
with create_dataset(f'{pool_name}/{ds_name}'):
results = SSH_TEST(' && '.join(cmds), user, password, ip)
assert results['result'] is True, str(results)
for x in targets:
target = f'{path}/{x}'
results = POST('/filesystem/stat/', target)
assert results.status_code == 200, f'{target}: {results.text}'
statout = results.json()
assert statout['type'] == x.upper(), str(statout)
assert not statout['is_ctldir']
result = POST(
"/zfs/snapshot/", {
'dataset': f'{pool_name}/{ds_name}',
'name': snap_name,
'recursive': False,
})
assert result.status_code == 200, result.text
for x in targets:
target = f'{path}/.zfs/snapshot/{snap_name}/{x}'
results = POST('/filesystem/stat/', target)
assert results.status_code == 200, f'{target}: {results.text}'
statout = results.json()
assert statout['type'] == x.upper(), str(statout)
assert statout['is_ctldir']
results = POST('/filesystem/stat/',
f'{path}/.zfs/snapshot/{snap_name}')
assert results.status_code == 200, results.text
assert results.json()['is_ctldir']
results = POST('/filesystem/stat/', f'{path}/.zfs/snapshot')
assert results.status_code == 200, results.text
assert results.json()['is_ctldir']
results = POST('/filesystem/stat/', f'{path}/.zfs')
assert results.status_code == 200, results.text
assert results.json()['is_ctldir']
def __init__(self, d_path, c_path, arch, lr, hidden, epochs, gpu, training,
predict, top, cat_names):
if training:
self.data_path = d_path
self.checkpoint_path = c_path
self.model = model_types[model_arr.index(arch)]
self.model_input = model_inputs[model_arr.index(arch)]
self.model_hidden = hidden
self.learn_rate = lr
self.epochs = epochs
self.device = torch.device('cuda:0' if gpu else 'cpu')
self.model.to(self.device)
for p in self.model.parameters():
p.requires_grad = False
classifier = nn.Sequential(
OrderedDict([
('fc1', nn.Linear(self.model_input, self.model_hidden)),
('rl1', nn.ReLU(inplace=True)),
('dr1', nn.Dropout(p=self.drop_p)),
('fc2', nn.Linear(self.model_hidden, self.model_output)),
('out', nn.LogSoftmax(dim=1))
]))
self.model.classifier = classifier
self.optimizer = optim.Adam(self.model.classifier.parameters(),
lr=self.learn_rate)
self.create_loaders()
train_dataset = utils.create_dataset(self.data_path + '/train',
self.re_size, self.norm_means,
self.norm_stdv,
self.train_batch)
self.model.class_to_idx = train_dataset.class_to_idx
else:
self.model = model_types[model_arr.index(arch)]
classifier = nn.Sequential(
OrderedDict([
('fc1', nn.Linear(self.model_input, self.model_hidden)),
('rl1', nn.ReLU(inplace=True)),
('dr1', nn.Dropout(p=self.drop_p)),
('fc2', nn.Linear(self.model_hidden, self.model_output)),
('out', nn.LogSoftmax(dim=1))
]))
self.model.classifier = classifier
self.optimizer = optim.Adam(self.model.classifier.parameters(),
lr=self.learn_rate)
self.checkpoint_path = c_path
self.predict_path = predict
self.top_k = top
self.category_names = cat_names
self.load_checkpoint()
def main():
# add argumentation
parser = argparse.ArgumentParser(
description='MobileNet_v2_DeepLab_v3 Pytorch Implementation')
parser.add_argument(
'--dataset',
default='cityscapes',
choices=['cityscapes', 'other'],
help='Dataset used in training MobileNet v2+DeepLab v3')
parser.add_argument('--root',
default='./data/cityscapes',
help='Path to your dataset')
parser.add_argument('--epoch',
default=None,
help='Total number of training epoch')
parser.add_argument('--lr', default=None, help='Base learning rate')
parser.add_argument('--pretrain',
default=None,
help='Path to a pre-trained backbone model')
parser.add_argument('--resume_from',
default=None,
help='Path to a checkpoint to resume model')
args = parser.parse_args()
params = Params()
# parse args
if not os.path.exists(args.root):
if params.dataset_root is None:
raise ValueError('ERROR: Root %s not exists!' % args.root)
else:
params.dataset_root = args.root
if args.epoch is not None:
params.num_epoch = args.epoch
if args.lr is not None:
params.base_lr = args.lr
if args.pretrain is not None:
params.pre_trained_from = args.pretrain
if args.resume_from is not None:
params.resume_from = args.resume_from
LOG('Network parameters:')
print_config(params)
# create dataset and transformation
LOG('Creating Dataset and Transformation......')
datasets = create_dataset(params)
LOG('Creation Succeed.\n')
# create model
LOG('Initializing MobileNet and DeepLab......')
net = MobileNetv2_DeepLabv3(params, datasets)
LOG('Model Built.\n')
# let's start to train!
net.Train()
net.Test()
def single_run(config, training=True):
start = time.time()
if config['model_fn'].endswith('hdf5'):
# using preset config
model = models.use_saved_model(config['model_fn'], **config)
else:
model_fn = getattr(models, config['model_fn'])
model = model_fn(**config)
model.summary()
print('Model compiled after {}'.format(runtime(start)))
tmp = dt.datetime.now().strftime("%Y-%m-%d-%H-%M-%S_%f")
config['tmp'] = tmp
config_name = '../config/config_{}_{}.json'.format(tmp, config['name'])
output_name = '../output/out_{}_{}.json'.format(tmp, config['name'])
config['model_w_name'] = "../weights/weights_{}_{}.hdf5".format(
config['name'], tmp)
config['output_name'] = output_name
print('Saving configuration file to: {}'.format(config_name), flush=True)
print('and output file to: {}'.format(output_name), flush=True)
with open(config_name, 'w') as f:
json.dump(config, f, indent=4)
print('Using following configuration:')
print(json.dumps(config, indent=2))
if training:
labels, data, meta = create_dataset('train.json', True, **config)
if config.get('pseudo_train', False):
idxs, test, test_meta = create_dataset('test.json', False,
**config)
dataset = ((labels, data, meta), (idxs, test, test_meta))
else:
dataset = (labels, data, meta)
print('Data loaded after {}'.format(runtime(start)))
model = train(dataset, model, **config)
print('Model trained after {}'.format(runtime(start)))
idxs, test, test_meta = create_dataset('test.json', False, **config)
dataset = (idxs, test, test_meta)
evaluate(model, dataset, **config)
print('Scriped successfully completed after {}'.format(runtime(start)))
def main(epochs,
enable_function,
buffer_size,
batch_size,
mode,
growth_rate,
output_classes,
depth_of_model=None,
num_of_blocks=None,
num_layers_in_each_block=None,
data_format='channels_last',
bottleneck=True,
compression=0.5,
weight_decay=1e-4,
dropout_rate=0.,
pool_initial=False,
include_top=True,
train_mode='custom_loop',
data_dir=None,
num_gpu=1):
devices = ['/device:GPU:{}'.format(i) for i in range(num_gpu)]
strategy = tf.distribute.MirroredStrategy(devices)
train_dataset, test_dataset, _ = utils.create_dataset(
buffer_size, batch_size, data_format, data_dir)
with strategy.scope():
model = densenet.DenseNet(mode, growth_rate, output_classes,
depth_of_model, num_of_blocks,
num_layers_in_each_block, data_format,
bottleneck, compression, weight_decay,
dropout_rate, pool_initial, include_top)
trainer = Train(epochs, enable_function, model, batch_size, strategy)
train_dist_dataset = strategy.experimental_distribute_dataset(
train_dataset)
test_dist_dataset = strategy.experimental_distribute_dataset(
test_dataset)
print('Training...')
if train_mode == 'custom_loop':
return trainer.custom_loop(train_dist_dataset, test_dist_dataset,
strategy)
elif train_mode == 'keras_fit':
raise ValueError(
'`tf.distribute.Strategy` does not support subclassed models yet.'
)
else:
raise ValueError(
'Please enter either "keras_fit" or "custom_loop" as the argument.'
)
def main():
# We currently support pynative mode with device GPU
context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU')
epoch_size = 1
batch_size = 32
mnist_path = "/data/chengzi/zhusuan-mindspore/data/MNIST"
repeat_size = 1
# Define model parameters
z_dim = 40
x_dim = 32 * 32
# create the network
generator = Generator(x_dim, z_dim, batch_size)
variational = Variational(x_dim, z_dim, batch_size)
network = zs.variational.ELBO(generator, variational)
# define loss
# learning rate setting
lr = 0.001
net_loss = ReduceMeanLoss()
# define the optimizer
print(network.trainable_params()[0])
net_opt = nn.Adam(network.trainable_params(), lr)
model = Model(network, net_loss, net_opt)
ds_train = create_dataset(os.path.join(mnist_path, "train"), batch_size,
repeat_size)
model.train(epoch_size,
ds_train,
callbacks=[LossMonitor()],
dataset_sink_mode=False)
print(network.trainable_params()[0])
iterator = ds_train.create_tuple_iterator()
for item in iterator:
batch_x = item[0].reshape(32, 32 * 32)
break
z, _ = network.variational(Tensor(batch_x), None, None)
sample, _, _, _ = network.generator(None, z, None)
sample = sample.asnumpy()
save_img(batch_x, 'result/origin_x.png')
save_img(sample, 'result/reconstruct_x.png')
for i in range(4):
sample, _, _, _ = network.generator(None, None, None)
sample = sample.asnumpy()
samples = sample if i == 0 else np.concatenate([samples, sample],
axis=0)
save_img(samples, 'result/sample_x.png', num=4 * batch_size)
def main():
# add argumentation
parser = argparse.ArgumentParser(description='MobileNet_v2_DeepLab_v3 Pytorch Implementation')
#todo maybe make it work with multiple datasets?
#parser.add_argument('--dataset', default='cityscapes', choices=['cityscapes', 'other'],
# help='Dataset used in training MobileNet v2+DeepLab v3')
parser.add_argument('--root', default='./data/cityscapes', help='Path to your dataset')
parser.add_argument('--epoch', default=None, help='Total number of training epoch')
parser.add_argument('--lr', default=None, help='Base learning rate')
parser.add_argument('--pretrain', default=None, help='Path to a pre-trained backbone model')
parser.add_argument('--resume_from', default=None, help='Path to a checkpoint to resume model')
parser.add_argument('--logdir', default=None, help='Directory to save logs for Tensorboard')
parser.add_argument('--batch_size', default=128, help='Batch size for training')
args = parser.parse_args()
params = Params()
# parse args
if not os.path.exists(args.root):
if params.dataset_root is None:
raise ValueError('ERROR: Root %s doesn\'t exist!' % args.root)
else:
params.dataset_root = args.root
if args.epoch is not None:
params.num_epoch = int(args.epoch)
if args.lr is not None:
params.base_lr = args.lr
if args.pretrain is not None:
params.pre_trained_from = args.pretrain
if args.resume_from is not None:
params.resume_from = args.resume_from
if args.logdir is not None:
params.logdir = args.logdir
params.summary_dir, params.ckpt_dir = create_train_dir(params.logdir)
params.train_batch = int(args.batch_size)
LOG('Network parameters:')
print_config(params)
# create dataset and transformation
LOG('Creating Dataset and Transformation......')
datasets = create_dataset(params)
LOG('Creation Succeed.\n')
# create model
LOG('Initializing MobileNet and DeepLab......')
net = MobileNetv2_DeepLabv3(params, datasets)
LOG('Model Built.\n')
# let's start to train!
net.Train()
net.Test()
def evaluate_validation_set(model, devset, x_to_ix, y_to_ix, criterion):
y_true = list()
y_pred = list()
total_loss = 0
for batch, targets, lengths, raw_data in utils.create_dataset(devset, x_to_ix, y_to_ix, batch_size=1):
batch, targets, lengths = utils.sort_batch(batch, targets, lengths)
pred, loss = apply(model, criterion, batch, targets, lengths)
pred_idx = torch.max(pred, 1)[1]
y_true += list(targets.int())
y_pred += list(pred_idx.data.int())
total_loss += loss
acc = accuracy_score(y_true, y_pred)
return total_loss.data.float() / len(devset), acc
def evaluate_test_set(model, test, x_to_ix, y_to_ix):
y_true = list()
y_pred = list()
for batch, targets, lengths, raw_data in utils.create_dataset(test, x_to_ix, y_to_ix, batch_size=1):
batch, targets, lengths = utils.sort_batch(batch, targets, lengths)
pred = model(torch.autograd.Variable(batch), lengths.cpu().numpy())
pred_idx = torch.max(pred, 1)[1]
y_true += list(targets.int())
y_pred += list(pred_idx.data.int())
print(len(y_true), len(y_pred))
print(classification_report(y_true, y_pred))
print(confusion_matrix(y_true, y_pred))
def ex3():
x, y = create_dataset(N=POINTS, K=CLASSES, D=DIMENSION)
y = tf.keras.utils.to_categorical(y, num_classes=CLASSES)
x_train, x_test, y_train, y_test = train_test_split(x, y,
test_size=TEST_SPLIT,
random_state=42,
shuffle=True)
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train,
test_size=VAL_SPLIT / (1 - TEST_SPLIT),
random_state=42,
shuffle=True)
# create, train and evaluate model using tf.keras
# TODO Fill me
plot_contour_tf(model, x, y)
def main(sample_num,
embed_dim,
learning_rate,
epochs,
batch_size,
mode='max',
attention_hidden_unit=None):
"""
:param sample_num: the num of training sample
:param embed_dim: the dimension of all embedding layer
:param learning_rate:
:param epochs:
:param batch_size:
:param mode
:param attention_hidden_unit:
:return:
"""
feature_columns, train_X, test_X, train_y, test_y = create_dataset(
sample_num, embed_dim)
# ============================Build Model==========================
model = AFM(feature_columns,
mode,
attention_hidden_unit=attention_hidden_unit)
model.summary()
# ============================model checkpoint======================
# check_path = 'save/afm_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
# checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
# verbose=1, period=5)
# =========================Compile============================
model.compile(loss=binary_crossentropy,
optimizer=Adam(learning_rate=learning_rate),
metrics=[AUC()])
# ===========================Fit==============================
model.fit(
train_X,
train_y,
epochs=epochs,
# callbacks=[checkpoint],
batch_size=batch_size,
validation_split=0.1)
# ===========================Test==============================
print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
def ex3():
x, y = create_dataset(N=POINTS, K=CLASSES, D=DIMENSION)
y = tf.keras.utils.to_categorical(y, num_classes=CLASSES)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=TEST_SPLIT,
random_state=42,
shuffle=True)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=VAL_SPLIT /
(1 - TEST_SPLIT),
random_state=42,
shuffle=True)
log_dir = Path("logs", "fit",
datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
model = tf.keras.Sequential()
model.add(tf.keras.Input(shape=(DIMENSION, )))
model.add(tf.keras.layers.Dense(10, activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dense(10, activation=tf.keras.activations.relu))
model.add(tf.keras.layers.Dense(10, activation=tf.keras.activations.relu))
model.add(
tf.keras.layers.Dense(CLASSES,
activation=tf.keras.activations.sigmoid))
opt = tf.keras.optimizers.SGD(learning_rate=LEARNING_RATE)
# opt = tf.keras.optimizers.Adam(learning_rate=1e-3)
model.compile(optimizer=opt,
loss=tf.losses.mean_squared_error,
metrics=[tf.keras.metrics.categorical_accuracy])
model.fit(x_train,
y_train,
batch_size=1,
epochs=EPOCH,
validation_data=(x_val, y_val),
callbacks=[tensorboard_callback])
print(f"Test accuracy: {model.evaluate(x_test, y_test)[1] * 100:.3f}%")
plot_contour_tf(model, x, y)
def main(learning_rate, epochs, hidden_units):
"""
feature_columns is a list and contains two dict：
- dense_features: {feat: dense_feature_name}
- sparse_features: {feat: sparse_feature_name, feat_num: the number of this feature,
embed_dim: the embedding dimension of this feature }
train_X: [dense_train_X, sparse_train_X]
test_X: [dense_test_X, sparse_test_X]
"""
feature_columns, train_X, test_X, train_y, test_y = create_dataset()
# ============================Build Model==========================
model = DCN(feature_columns, hidden_units)
model.summary()
# =============================Tensorboard=========================
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
log_dir = 'logs/' + current_time
tensorboard = tf.keras.callbacks.TensorBoard(
log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_grads=False,
write_images=True,
embeddings_freq=0, embeddings_layer_names=None,
embeddings_metadata=None, embeddings_data=None, update_freq=500
)
# ============================model checkpoint======================
check_path = 'save/dcn_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
verbose=1, period=4)
# =========================Compile============================
model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
metrics=[AUC()])
# ===========================Fit==============================
model.fit(
train_X,
train_y,
epochs=epochs,
callbacks=[tensorboard, checkpoint],
batch_size=128,
validation_split=0.2
)
# ===========================Test==============================
print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
def data(data_file_path):
"""
Input : Path to dataset
Output : Data in the required format
"""
scaler = MinMaxScaler()
data_sheet = pd.read_csv(data_file_path)
_, window_size = get_window_size(data_sheet)
features, values = create_dataset(data_sheet, window_size)
features = scaler.fit_transform(features)
values = scaler.fit_transform(values)
features = features.reshape(features.shape[0], features.shape[1], 1)
train_size = (int)(0.6 * features.shape[0])
all_truths = np.asarray(data_sheet['label'])
X_train = features[:train_size]
y_train = values[:train_size]
X_test = features[train_size:]
y_test = values[train_size:]
return X_train, y_train, X_test, y_test, window_size, train_size, all_truths
def create_speaker_models():
import constants as c
import numpy as np
import os
from utils import create_dataset
model_path = os.path.join(c.ROOT, 'Models/model_14_percent_best_so_far.pt')
save_speaker_models_path = os.path.join(c.ROOT, 'speaker_models')
enrollment_set = os.path.join(c.ROOT, '50_first_ids.txt')
indexed_labels = np.load(c.ROOT + '/50_first_ids.npy',
allow_pickle=True).item()
dataset = create_dataset(indexed_labels=indexed_labels,
origin_file_path=enrollment_set)
if not os.path.exists(save_speaker_models_path):
os.mkdir(save_speaker_models_path)
if not torch.cuda.is_available():
model = C3D2(100, 1).load_checkpoint(
torch.load(model_path, map_location=lambda storage, loc: storage))
# model = C3D2(100, 1).load_checkpoint(torch.load(model_path))
else:
model = C3D2(100, 1).load_checkpoint(torch.load(model_path))
model.eval()
for i in range(len(dataset)):
# get the inputs
train_input, _ = dataset.__getitem__(i)
[a, b, cc, d] = train_input.shape
train_input = torch.from_numpy(train_input.reshape((1, a, b, cc, d)))
if torch.cuda.is_available():
train_input = Variable(train_input.cuda())
else:
train_input = Variable(train_input)
current_id = dataset.sound_files[i][0:7]
speaker_model = model(train_input, development=False)
torch.save(speaker_model, '{}/{}.pt'.format(save_speaker_models_path,
current_id))
def test_08_test_fiilesystem_statfs_flags(request):
"""
This test verifies that changing ZFS properties via
middleware causes mountinfo changes visible via statfs.
"""
depends(request, ["pool_04"], scope="session")
ds_name = 'statfs_test'
target = f'{pool_name}/{ds_name}'
target_url = target.replace('/', '%2F')
path = f'/mnt/{target}'
# tuple: ZFS property name, property value, mountinfo value
properties = [
("readonly", "ON", "RO"),
("readonly", "OFF", "RW"),
("atime", "OFF", "NOATIME"),
("exec", "OFF", "NOEXEC"),
("acltype", "NFSV4", "NFS4ACL"),
("acltype", "POSIX", "POSIXACL"),
]
with create_dataset(target):
for p in properties:
# set option we're checking and make sure it's really set
payload = {p[0]: p[1]}
if p[0] == 'acltype':
payload.update({
'aclmode':
'RESTRICTED' if p[1] == 'NFSV4' else 'DISCARD'
})
results = PUT(f'/pool/dataset/id/{target_url}', payload)
assert results.status_code == 200, results.text
prop_out = results.json()[p[0]]
assert prop_out['value'] == p[1]
# check statfs results
results = POST('/filesystem/statfs/', path)
assert results.status_code == 200, results.text
mount_flags = results.json()['flags']
assert p[
2] in mount_flags, f'{path}: ({p[2]}) not in {mount_flags}'
def sobolev_train(args, input_var, network, tang_output):
"""Train a network with Sobolev, output the network, a function to make predictions and training losses"""
# Create data
X = utils.create_dataset(args.npts)
# Create a list of batches (a list of batch idxs splitting X in batches of size batch_size)
list_batches = utils.get_list_batches(args.npts, args.batch_size)
# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.squared_error(prediction, tang_output)
loss = loss.mean()
# Add jacobian (J) output for Sobolev training
J_teacher = theano.gradient.jacobian(tang_output.flatten(), input_var)
J_student = theano.gradient.jacobian(prediction.flatten(), input_var)
loss_sobolev = lasagne.objectives.squared_error(J_teacher.flatten(),
J_student.flatten())
loss_sobolev = args.sobolev_weight * loss_sobolev.mean()
loss_total = loss + loss_sobolev
# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(
loss_total, params, learning_rate=args.learning_rate, momentum=0.9)
# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var], [loss, loss_sobolev],
updates=updates)
# train network
list_loss, list_loss_J = utils.train_network_sobolev(
train_fn, X, list_batches, args.nb_epoch)
# Create a prediction function to evaluate after training
predict_fn = utils.get_prediction_fn(input_var, network)
return network, predict_fn, list_loss, list_loss_J
def main():
parser = argparse.ArgumentParser(
description='MobileNet_V2 Pytorch Implementation')
parser.add_argument('--dataset',
default='cifar10',
choices=['imagenet', 'cifar10', 'cifar100', 'other'],
help='Dataset used in training MobileNet V2')
parser.add_argument('--root',
default='./data/cifar10',
help='Path to your dataset')
args = parser.parse_args()
# parse args
if args.dataset == 'cifar10':
params = CIFAR10_params()
elif args.dataset == 'cifar100':
params = CIFAR100_params()
else:
params = Params()
params.dataset_root = args.root
if not os.path.exists(args.root):
print('ERROR: Root %s not exists!' % args.root)
exit(1)
""" TEST CODE """
# params = CIFAR100_params
# params.dataset_root = '/home/ubuntu/cifar100'
# create model
print('\nInitializing MobileNet......')
net = MobileNetv2(params)
print('Initialization Done.\n')
# create dataset and transformation
print('Loading Data......')
dataset = create_dataset(params)
print('Data Loaded.\n')
# let's start to train!
net.train_n_epoch(dataset)
def generate_dateset():
x, y = create_dataset(N=POINTS, K=CLASSES, D=DIMENSION)
x = np.expand_dims(x, -1)
# transform data to categorical
y = tf.keras.utils.to_categorical(y, num_classes=CLASSES)
y = np.expand_dims(y, -1)
# automatically split dataset in train/test with 20% as test
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=TEST_SPLIT,
random_state=42,
shuffle=True)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=VAL_SPLIT /
(1 - TEST_SPLIT),
random_state=42,
shuffle=True)
return x, y, x_train, x_val, x_test, y_train, y_val, y_test
def sobolev_train(args, input_var, network, tang_output):
"""Train a network with Sobolev, output the network, a function to make predictions and training losses"""
# Create data
X = utils.create_dataset(args.npts)
# Create a list of batches (a list of batch idxs splitting X in batches of size batch_size)
list_batches = utils.get_list_batches(args.npts, args.batch_size)
# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.squared_error(prediction, tang_output)
loss = loss.mean()
# Add jacobian (J) output for Sobolev training
J_teacher = theano.gradient.jacobian(tang_output.flatten(), input_var)
J_student = theano.gradient.jacobian(prediction.flatten(), input_var)
loss_sobolev = lasagne.objectives.squared_error(J_teacher.flatten(), J_student.flatten())
loss_sobolev = args.sobolev_weight * loss_sobolev.mean()
loss_total = loss + loss_sobolev
# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss_total, params, learning_rate=args.learning_rate, momentum=0.9)
# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var], [loss, loss_sobolev], updates=updates)
# train network
list_loss, list_loss_J = utils.train_network_sobolev(train_fn, X, list_batches, args.nb_epoch)
# Create a prediction function to evaluate after training
predict_fn = utils.get_prediction_fn(input_var, network)
return network, predict_fn, list_loss, list_loss_J
IMG_DIR = os.path.join(DATA_DIR, 'raw', 'CVPPP', 'CVPPP2017_LSC_training',
'training', 'A1')
OUT_DIR = os.path.join(DATA_DIR, 'processed', 'CVPPP', 'lmdb')
try:
os.makedirs(OUT_DIR)
except BaseException:
pass
for subset in ['training', 'validation']:
lst_filepath = os.path.join(DATA_DIR, 'metadata', 'CVPPP', subset + '.lst')
lst = np.loadtxt(lst_filepath, dtype='str', delimiter=' ')
img_paths = []
ins_ann_paths = []
semantic_ann_paths = []
for image_name in lst:
img_path = os.path.join(IMG_DIR, image_name + '_rgb.png')
ins_ann_path = os.path.join(INSTANCE_ANN_DIR, image_name + '.npy')
sem_ann_path = os.path.join(SEMANTIC_ANN_DIR, image_name + '.npy')
if os.path.isfile(img_path) and os.path.isfile(
ins_ann_path) and os.path.isfile(sem_ann_path):
img_paths.append(img_path)
ins_ann_paths.append(ins_ann_path)
semantic_ann_paths.append(sem_ann_path)
out_path = os.path.join(OUT_DIR, '{}-lmdb'.format(subset))
create_dataset(out_path, img_paths, semantic_ann_paths, ins_ann_paths)
os.makedirs(OUT_DIR)
except BaseException:
pass
for _i, subset in enumerate(SUBSETS):
semantic_ann_paths_all = glob.glob(
os.path.join(SEMANTIC_ANN_DIR, subset, '*.npy'))
semantic_ann_paths, instance_ann_paths, image_paths = [], [], []
for f in semantic_ann_paths_all:
name = os.path.splitext(os.path.basename(f))[0]
_dir = name.split('_')[0]
semantic_ann_path = f
instance_ann_path = os.path.join(
INSTANCE_ANN_DIR, subset, name + '.npy')
img_path = os.path.join(
IMG_DIR, subset, _dir, name + '_leftImg8bit.png')
# if np.load(instance_ann_path).shape[-1] > 20:
# continue
semantic_ann_paths.append(semantic_ann_path)
instance_ann_paths.append(instance_ann_path)
image_paths.append(img_path)
out_path = os.path.join(OUT_DIR, '{}-lmdb'.format(SUBSET_NAMES[_i]))
create_dataset(out_path, image_paths, semantic_ann_paths,
instance_ann_paths)
