def main():
# logging configuration
logging.basicConfig(level=logging.INFO,
format="[%(asctime)s]: %(levelname)s: %(message)s")
# command line paser
opt = parse.parse_arg()
# GPU
opt.cuda = opt.gpuid >= 0
if opt.gpuid >= 0:
torch.cuda.set_device(opt.gpuid)
else:
logging.info("WARNING: RUN WITHOUT GPU")
# prepare dataset
opt.dataset = 'mnist'
db = dataset.prepare_db(opt)
#add wavelets:
opt.wavelets = True
opt.cutoff = 20
# initalize neural decision forest
NDF = model.prepare_model(opt)
# prepare optimizer
optim, sche = optimizer.prepare_optim(NDF, opt)
# train the neural decision forest5
best_acc = trainer.train(NDF, optim, sche, db, opt)
logging.info('The best evaluation accuracy is %f' % best_acc)
def main():
train_data_fft, train_data_raw, train_data_mfcc, train_label, valid_data_fft, valid_data_raw, valid_data_mfcc, valid_label = load_data(
)
model = prepare_model(window_size, mfcc_size)
model.load_weights("model5.hdf5")
valmodel(model, valid_data_fft, valid_data_raw, valid_data_mfcc,
valid_label)
def main():
train_data_fft, train_data_raw, train_data_mfcc, train_label, valid_data_fft, valid_data_raw, valid_data_mfcc, valid_label = load_data(
)
model = prepare_model(window_size, mfcc_size)
train_model(model, train_data_fft, train_data_raw, train_data_mfcc,
train_label, valid_data_fft, valid_data_raw, valid_data_mfcc,
valid_label)
test_model(model, valid_data_fft, valid_data_raw, valid_data_mfcc,
valid_label)
def __init__(self, classes, model_path, rf_path, knn_path):
self.model = prepare_model(config)
self.model.load_weights(model_path)
self.rf_outclass = pickle.load(open(rf_path, 'rb'))
self.knn = pickle.load(open(knn_path, 'rb'))
self.classes_name_to_id = classes
self.classes_id_to_name = {}
for key, val in self.classes_name_to_id.items():
self.classes_id_to_name[val] = key
def train_model(train_filelist, config, classes, rare_classes):
batch_size = config['train_params']['batch_size']
different_classes_per_batch = config['train_params']['different_classes_per_batch']
nb_epoch = config['train_params']['num_epochs']
steps_per_epoch = config['train_params']['steps_per_epoch']
img_size = config['img']['img_size']
model = prepare_model(config)
callbacks = get_callbacks(config)
model.fit_generator(data_generator(train_filelist, classes, rare_classes, img_size, batch_size, different_classes_per_batch),
steps_per_epoch=steps_per_epoch, epochs=nb_epoch, callbacks=callbacks)
return model
def main():
# parse command line input
opt = utils.parse_arg()
# Set GPU
opt.cuda = opt.gpuid >= 0
if opt.cuda:
torch.cuda.set_device(opt.gpuid)
else:
utils.time_str("GPU acceleration is disabled.")
# prepare data
db = data_prepare.prepare_db(opt)
imagenet = data_prepare.ImageNetSmallData(opt, type='centres')
# imagenet = None
#
# # add imagenet dataset
db.update({'imagenet': imagenet})
# initialize the model
pre_trained_model = model.prepare_model(opt)
# prepare M_0(x) model, which is a fixed pre-trained model
opt.num_output = 1000
fixed_model = model.prepare_model(opt)
# prepare centres
if not os.path.exists('../datasets/imagenet/train_centres.txt'):
imagenet = data_prepare.ImageNetSmallData(opt, type='all')
trainer.prepare_centres(fixed_model, imagenet, opt)
# configurate the optimizer
optim, sche = optimizer.prepare_optim(pre_trained_model, opt)
# train the model
trainer.train(pre_trained_model, optim, sche, db, opt, model_0=fixed_model)
# trainer.train(pre_trained_model, optim, sche, db, opt, model_0 = None)
# save the trained model
if opt.save:
utils.save_model(pre_trained_model, opt)
def main():
# parse command line input
opt = utils.parse_arg()
# Set GPU
opt.cuda = opt.gpuid >= 0
if opt.cuda:
torch.cuda.set_device(opt.gpuid)
else:
utils.time_str("GPU acceleration is disabled.")
# prepare data
db = data_prepare.prepare_db(opt)
# initializa the model
pre_trained_model = model.prepare_model(opt)
# configurate the optimizer
optim, sche = optimizer.prepare_optim(pre_trained_model, opt)
# train the model
trainer.train(pre_trained_model, optim, sche, db, opt)
# save the trained model
utils.save_model(pre_trained_model, opt)
# Copyright (c) 2019. All rights reserved. # Author: Ruoqi Yang # @Imperial College London, HKU alumni # mailto: [email protected] # This file is part of the quantitative research of Nuode Fund, contact # [email protected] for commercial use. from chip import CDS from model import prepare_model from mpl_toolkits.mplot3d import Axes3D import matplotlib.pyplot as plt from utils import fp, Parameters clf, X_test, y_test = prepare_model(CDS.from_ticker('IF'), '2016-06-13', **Parameters.standard) for case in ['实际', '预测']: prediction = clf.predict(X_test) if case == '实际': ups = X_test[y_test == 1] downs = X_test[y_test == 0] if case == '预测': ups = X_test[prediction == 1] downs = X_test[prediction == 0] fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.scatter(ups.values[:, 0], ups.values[:, 1],
def main():
# logging configuration
logging.basicConfig(level=logging.INFO,
format="[%(asctime)s]: %(message)s")
# parse command line input
opt = utils.parse_arg()
# Set GPU
opt.cuda = opt.gpuid >= 0
if opt.cuda:
torch.cuda.set_device(opt.gpuid)
else:
# please use GPU for training, CPU version is not supported for now.
raise NotImplementedError
#logging.info("GPU acceleration is disabled.")
# prepare training and validation dataset
db = data_prepare.prepare_db(opt)
# sanity check for FG-NET dataset, not used for now
# assertion: the total images in the eval set lists should be 1002
total_eval_imgs = sum([len(db['eval'][i]) for i in range(len(db['eval']))])
print(total_eval_imgs)
if db['train'][0].name == 'FGNET':
assert total_eval_imgs == 1002, 'The preparation of the evalset is incorrect.'
# training
if opt.train:
best_MAEs = []
last_MAEs = []
# record the current time
opt.save_dir += time.asctime(time.localtime(time.time()))
# for FG-NET, do training multiple times for leave-one-out validation
# for CACD, do training just once
for exp_id in range(len(db['train'])):
# initialize the model
model_train = model.prepare_model(opt)
#print("model shape:")
# print(db['train'].head)
#print( np.array(db['eval']).shape)
# configurate the optimizer and learning rate scheduler
optim, sche = optimizer.prepare_optim(model_train, opt)
# train the model and record mean average error (MAE)
model_train, MAE, last_MAE = trainer.train(model_train, optim,
sche, db, opt, exp_id)
best_MAEs += MAE
last_MAEs.append(last_MAE.data.item())
# remove the trained model for leave-one-out validation
if exp_id != len(db['train']) - 1:
del model_train
#np.save('./MAE.npy', np.array(best_MAEs))
#np.save('./Last_MAE.npy', np.array(last_MAEs))
# save the final trained model
#utils.save_model(model_train, opt)
# testing a pre-trained model
elif opt.evaluate:
# path to the pre-trained model
save_dir = opt.test_model_path
#example: save_dir = '../model/CACD_MAE_4.59.pth'
model_loaded = torch.load(save_dir)
# test the model on the evaluation set
# the last subject is the test set (compatible with FG-NET)
trainer.evaluate(model_loaded, db['eval'][-1], opt)
return
from model import prepare_model
from main import mfcc_size, window_size, MAX_MINDWAVE_VALUE, MAX_RAW
import socket
import struct
import librosa
import numpy as np
model = prepare_model(window_size, mfcc_size)
model.load_weights("model6.hdf5")
message_size = 520
def run_server(port=4242):
host = ''
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
print("waiting")
s.bind((host, port))
s.listen(1)
conn, addr = s.accept()
fft = [0.] * window_size * 8
raw = [0.] * window_size * 512
while (True):
msg = conn.recv(4 * message_size)
received = struct.unpack('<%si' % message_size, msg)
normalized_fft = [x / MAX_MINDWAVE_VALUE for x in received[512:]]
assert len(normalized_fft) == 8
normalized_raw = [x / MAX_RAW for x in received[:512]]
fft = fft[8:] + normalized_fft
raw = raw[512:] + normalized_raw
def run_scenario(scenario_id, args=None):
logd = create_logger(appname='{} - {} - details'.format(args.app_name, scenario_id),
logfile=join(args.scenario_log_dir,'scenario_{}_details.txt'.format(scenario_id)),
msg_format='%(asctime)s - %(message)s')
logp = create_logger(appname='{} - {} - progress'.format(args.app_name, scenario_id),
logfile=join(args.scenario_log_dir, 'scenario_{}_progress.txt'.format(scenario_id)),
msg_format='%(asctime)s - %(message)s')
logd.info('starting scenario {}'.format(scenario_id))
# get connection, along with useful tools attached
conn = connection(args, scenario_id, args.template_id, logd)
# time steps
ti = datetime.strptime(args.initial_timestep, args.timestep_format)
tf = datetime.strptime(args.final_timestep, args.timestep_format)
dates = [date for date in rrule.rrule(rrule.MONTHLY, dtstart=ti, until=tf)]
timestep_dict = OrderedDict()
conn.OAtHPt = {}
for date in dates:
oat = date.strftime(args.timestep_format)
hpt = date.strftime(args.hydra_timestep_format)
timestep_dict[date] = [hpt, oat]
conn.OAtHPt[oat] = hpt
template_attributes = conn.call('get_template_attributes', {'template_id': conn.template.id})
attr_names = {}
for ta in template_attributes:
attr_names[ta.id] = ta.name
# create the model
instance = prepare_model(model_name='OpenAgua',
network=conn.network,
template=conn.template,
attr_names=attr_names,
timestep_format=args.timestep_format,
timestep_dict=timestep_dict)
logd.info('model created')
opt = SolverFactory(args.solver)
results = opt.solve(instance, tee=False)
#logd.info('model solved')
old_stdout = sys.stdout
sys.stdout = summary = StringIO()
results.write()
sys.stdout = old_stdout
logd.info('model solved\n' + summary.getvalue())
if (results.solver.status == SolverStatus.ok) and (results.solver.termination_condition == TerminationCondition.optimal):
# this is feasible and optimal
logd.info('Optimal feasible solution found.')
outputnames = {'S': 'storage', 'I': 'inflow', 'O': 'outflow'}
#outputnames = {'I': 'inflow', 'O': 'outflow'}
result = conn.save_results(instance, outputnames)
logd.info('Results saved.')
elif results.solver.termination_condition == TerminationCondition.infeasible:
logd.info('WARNING! Problem is infeasible. Check detailed results.')
# do something about it? or exit?
else:
# something else is wrong
logd.info('WARNING! Something went wrong. Likely the model was not built correctly.')
# Still we will report that the model is complete...
if args.foresight == 'perfect':
msg = 'completed timesteps {} - {} | 1/1'.format(ti, tf)
logd.info(msg)
logp.info(msg)
# ===========================
# start the per timestep loop
# ===========================
#T = len(dates)
#for t, date in enumerate(dates):
# ===========================
# prepare the time steps to use in the optimization run
# ===========================
# ===========================
# prepare the inflow forecast model
# ===========================
# For now, forecast based on mean monthly inflow at each catchment node
# However, this can be changed in the future
# ===========================
# run the model
# ===========================
#if new_model:
#model = create_model(data)
#instance = model.create_instance()
#else:
#instance = update_instance(instance, S0, inflow)
#instance.preprocess()
# solve the model
#results = solver.solve(instance)
# load the results
#instance.solutions.load_from(results)
# set initial conditions for the next time step
#S0 = instance.S[isIDs[0]].value
#if S0 is None:
#S0 = 0.0
# ===========================
# save results to memory
# ===========================
#logd.info('completed timestep {} | {}/{}'.format(dt.date.strftime(date, args.timestep_format), t+1, T))
# ===========================
# save results to Hydra Server
# ===========================
return
def train(dataroot_path=None, weights_directory=None, num_samples=None):
# print(tf.__version__)
# print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
# tf.debugging.set_log_device_placement(True)
text = _dutils.open_file('item_names_lower.txt')
idx2char, char2idx = _dutils.create_dictionaries(text, True)
dictionary_size = len(idx2char.values())
'''
Load necessary hyperparameters from JSON files.
'''
general_hyperparameters = _dutils.load_dictionary(
'hyperparameters/hyperparameters.json')
#If, for some reason the JSON value is not the same, update the values
if general_hyperparameters.get('vocab_size') != dictionary_size:
general_hyperparameters['vocab_size'] = dictionary_size
model_hyperparameters = _dutils.load_dictionary(
'hyperparameters/model_hyperparameters.json')
'''
Set the necessary logging directories
'''
checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir,
"weights-improvement-{epoch:02d}")
'''
Set callback functions here.
Tensorboard callback doesn't seem to
work well, so I take it out.
'''
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.8,
patience=2,
min_lr=0.0001)
# Name of the checkpoint files
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_prefix,
save_best_only=True,
mode='max',
save_weights_only=True)
'''
Prepare the model for training.
'''
model = _model.prepare_model(model_hyperparameters,
load_weights=False,
weight_directory=checkpoint_dir,
verbose=False)
'''
Prepare the dataset for training.
'''
padded_encoded_text = _dutils.preprocess_dataset(text)
text_train, text_test = _dutils.create_train_test_data(
padded_encoded_text,
general_hyperparameters['seq_len'],
num_samples=num_samples,
rand_state=420,
verbose=True)
train_dataset = _dutils.create_dataset(text_train, dictionary_size,
idx2char, char2idx, True, None)
test_dataset = _dutils.create_dataset(text_test, dictionary_size, idx2char,
char2idx, True, None)
dset = train_dataset.shuffle(general_hyperparameters['buffer_size']).batch(
general_hyperparameters['batch_size'])
t_dset = test_dataset.shuffle(
general_hyperparameters['buffer_size']).batch(
general_hyperparameters['batch_size'])
# Begin training the model.
history = model.fit(dset,
epochs=general_hyperparameters['epochs'],
validation_data=t_dset,
validation_freq=1,
verbose=1,
callbacks=[reduce_lr, checkpoint_callback])
history_json_file = "history_{}.json".format(
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
_dutils.save_dictionary(history, history_json_file)
parser.add_argument('--heads', type=str, default='[["icon", 1, 1], ["test", 1, 1], ["synt", 1, 1], ["synt", 5, 3], ["synt", 10, 5]]',
help='Necessary heads with random forest and kNN in format \'[(index_type, index_pictures_per_class, knn_neighbors)]\'')
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_ids
classes, classes_rare, classes_often = get_classes(args.real_path)
data_folders = []
if args.real_path != '':
data_folders.append(args.real_path)
if args.synt_path != '':
data_folders.append(args.synt_path)
train_filelist = get_list_of_files(args.exp_name, data_folders, classes_often + classes_rare if args.train_type == 'all_classes' else classes_often)
if args.train_only_heads:
model = prepare_model(config)
model.load_weights(args.exp_name + "_" + args.nn_model)
else:
model = train_model(train_filelist, config, classes, classes_rare)
model.save(args.exp_name + "_" + args.nn_model)
img_shape = (config['img']['img_size'], config['img']['img_size'], 3)
for head in json.loads(args.heads):
print("Training head: ", head)
if head[0] == 'icon':
index_x, index_y = read_index_icon(args.icons_path, img_shape, classes, args.replication_factor)
if head[0] == 'test':
index_file, noindex_file = create_test_index(args.exp_name, args.test_path, args.use_existing_index)
index_x, index_y = read_index(index_file, img_shape, classes, args.replication_factor)
if head[0] == 'synt':
def main(args):
'''
init
'''
# read images and preprocess
if args.content_img:
content_img = read_image(args.content_img, args.hard_width)
style_img = read_image(args.style_img, args.hard_width)
# get stacked 0./1. masks
if args.mask_n_colors > 1: # colorful
target_masks_origin, style_masks_origin = read_colorful_mask(
args.target_mask, args.style_mask, args.hard_width,
args.mask_n_colors)
else: # single mask
if args.target_mask is None:
if args.content_img:
target_masks_origin = np.ones(content_img.shape[0:3]).astype(
np.float32)
else:
target_masks_origin = np.ones(style_img.shape[0:3]).astype(
np.float32)
else:
target_masks_origin = read_single_mask(args.target_mask,
args.hard_width)
if args.style_mask is None:
style_masks_origin = np.ones(style_img.shape[0:3]).astype(
np.float32)
else:
style_masks_origin = read_single_mask(args.style_mask,
args.hard_width)
# init img & target shape
if args.content_img:
target_shape = content_img.shape
init_img = get_init_image(content_img, args.init_noise_ratio)
else:
target_shape = [1] + list(target_masks_origin.shape[1:3]) + [3]
init_img = np.random.uniform(-20., 20.,
target_shape).astype(np.float32)
# check shape & number of masks
if args.content_img and content_img.shape[
1:3] != target_masks_origin.shape[1:3]:
print('content and mask have different shape')
sys.exit(0)
if style_img.shape[1:3] != style_masks_origin.shape[1:3]:
print('style and mask have different shape')
sys.exit(0)
if target_masks_origin.shape[0] != style_masks_origin.shape[0]:
print('content and style have different masks')
sys.exit(0)
'''
compute features & build net
'''
# prepare model weights
vgg_weights = model.prepare_model(args.model_path)
# feature maps of specific layers
if args.content_img:
content_features = compute_features(vgg_weights,
args.feature_pooling_type,
content_img, args.content_layers)
style_features = compute_features(vgg_weights, args.feature_pooling_type,
style_img, args.style_layers)
# masks of specific layers
target_masks = compute_layer_masks(target_masks_origin, args.style_layers,
args.mask_downsample_type)
style_masks = compute_layer_masks(style_masks_origin, args.style_layers,
args.mask_downsample_type)
# build net
target_net = build_target_net(vgg_weights, args.feature_pooling_type,
target_shape)
'''
loss
'''
if args.content_img:
content_loss = sum_content_loss(target_net, content_features,
args.content_layers,
args.content_layers_weights,
args.content_loss_normalization)
else:
content_loss = 0.
style_masked_loss = sum_masked_style_loss(target_net, style_features,
target_masks, style_masks,
args.style_layers,
args.style_layers_weights,
args.mask_normalization_type)
if args.tv_weight != 0.:
tv_loss = sum_total_variation_loss(target_net['input'], target_shape)
else:
tv_loss = 0.
total_loss = args.content_weight * content_loss + \
args.style_weight * style_masked_loss + \
args.tv_weight * tv_loss
'''
train
'''
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(args.learning_rate)
train_op = optimizer.minimize(total_loss)
#init
init_op = tf.global_variables_initializer(
) # must! Adam has some varibales to init
sess = tf.Session()
sess.run(init_op)
sess.run(target_net['input'].assign(init_img))
#train
for i in range(args.iteration):
sess.run(train_op)
if i % args.log_iteration == 0:
print('Iteration %d: loss = %f' %
(i + 1, sess.run(total_loss)))
result = sess.run(target_net['input'])
output_path = os.path.join(args.output_dir,
'result_%s.png' % (str(i).zfill(4)))
write_image(output_path, result)
elif args.optimizer == 'lbfgs':
optimizer = tf.contrib.opt.ScipyOptimizerInterface(
total_loss,
method='L-BFGS-B',
options={
'maxiter': args.iteration,
'disp': args.log_iteration
})
# init
init_op = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init_op)
sess.run(target_net['input'].assign(init_img))
# train
optimizer.minimize(sess)
'''
out
'''
print('Iteration %d: loss = %f' % (args.iteration, sess.run(total_loss)))
result = sess.run(target_net['input'])
output_path = os.path.join(args.output_dir, 'result_final.png')
write_image(output_path, result)
