def extract_regression_parameters(reg_file):
test_artifact = read_zipped_json(reg_file)
all_options = []
for i, test in enumerate(test_artifact):
layers = extract_layers(test['model']['layers'])
options = test['model']['training_options']
drate = options.get('dropout_rate', 0.0)
bn = options.get('batch_normalization', False)
if drate and not bn:
layers = add_dropouts(layers, drate)
for key in [
'randomize', 'activation_functions', 'layer_sizes',
'activation_function', 'learn_residuals',
'batch_normalizaiton', 'dropout_rate'
]:
options.pop(key, None)
make_model(layer_sequence({'layers': layers}), (6, ))
options['layers'] = layers
options['seed'] = i
all_options.append(options)
return all_options
def make_and_run(input_model, _layers=None, lookback=12, epochs=4, **kwargs):
input_features = [
'tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm',
'wind_speed_mps', 'rel_hum'
]
# column in dataframe to bse used as output/target
outputs = ['blaTEM_coppml']
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=lookback,
lr=0.001,
inputs=input_features,
outputs=outputs,
epochs=epochs,
**kwargs)
nn_config['layers'] = _layers
df = pd.read_csv('../data/all_data_30min.csv')
_model = input_model(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals)
_model.build_nn()
_ = _model.train_nn(indices='random')
_ = _model.predict(use_datetime_index=False)
return _model
def make_and_run(input_model, _layers=None, lookback=12, epochs=4, **kwargs):
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=lookback,
lr=0.001,
epochs=epochs,
**kwargs)
nn_config['layers'] = _layers
df = pd.read_csv("../data/nasdaq100_padding.csv")
_model = input_model(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals
)
_model.build_nn()
_ = _model.train_nn(indices='random')
_ = _model.predict(use_datetime_index=False)
return _model
def create_model(args):
# define model
# 加载模型结构
resnet = utils.make_model(args)
model = resnet_model.resnext101_32x8d()
# 加载模型参数
# 固定参数进行训练 一定要注意添加的位置
model.fc = nn.Sequential(nn.Dropout(0.5),
nn.Linear(2048, args.num_classes))
summary(model.cuda(), input_size=(3, 288, 288))
exit()
# 读取当前模型参数
model_dict = model.state_dict()
# 将预训练的网络中不属于模型的层的参数剃掉
pretrained_dict = resnet.state_dict()
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if k in model_dict
}
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
# model = model.module
print("load model's parameters success")
exit()
return model
def make_multi_model(input_model,
from_config=False,
config_path=None,
weights=None,
batch_size=8,
lookback=19,
lr=1.52e-5,
ignore_nans=True,
**kwargs):
data_config, nn_config, total_intervals = make_model(
batch_size=batch_size,
lookback=lookback,
lr=lr,
ignore_nans=ignore_nans,
**kwargs)
data_config['inputs'] = [
'tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)',
'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3', 'ORGP_INppm'
]
data_config['outputs'] = [
'obs_chla_1',
'obs_chla_3',
'obs_chla_8' # , 'obs_chla_12'
]
fpath = "D:\\playground\\paper_with_codes\\dl_ts_prediction\\data"
df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv'))
df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv'))
df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv'))
# df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv'))
df_1.index = pd.to_datetime(df_1['date'])
df_3.index = pd.to_datetime(df_3['date'])
df_8.index = pd.to_datetime(df_8['date'])
# df_12.index = pd.to_datetime(df_12['date'])
if from_config:
_model = input_model.from_config(
config_path=config_path,
data=[
df_1,
df_3,
df_8 #, df_12
])
_model.build_nn()
_model.load_weights(weights)
else:
_model = input_model(
data_config=data_config,
nn_config=nn_config,
data=[
df_1,
df_3,
df_8 # , df_12
])
return _model
def stylize(args):
content_image = utils.load_image_to_tensor(args.content_image,args.cuda)
content_image.unsqueeze_(0)
content_image = Variable(content_image)
model = utils.make_model(args)
model.load_state_dict(torch.load(args.model))
output_image = model(content_image)
output_image = output_image.data
output_image.squeeze_(0)
utils.save_tensor_to_image(output_image,args.output_image,args.cuda)
def __init__(self, options):
"""
Args:
options: configs.Config类的实例
"""
self.options = options
input_shape = (options.target_size[0], options.target_size[1],
len(options.use_chans))
print('Making %s...'%options.model_name)
if utils.is_py3(): #本地运行
best_model = os.path.join(options.output_path, options.weight_path)
self.model = utils.make_model(self.options.model_name, input_shape)
if tf.gfile.Exists(best_model):
print('Load weights from %s.'%best_model)
self.model.load_weights(best_model)
else: #PAI上运行
path = os.path.join(options.output_path, options.weight_path)
self.model = utils.make_model(self.options.model_name, input_shape)
if tf.gfile.Exists(path):
utils.copy_from_oss([path], using_cache=False)
best_model = options.weight_path
print('Load weights from %s.'%best_model)
self.model.load_weights(best_model)
self.log_path = '%s-log.txt'%options.run_name
if utils.is_py3():
self.log_path = os.path.join(options.output_path, self.log_path)
else:
utils.copy_from_oss(
[os.path.join(options.output_path, self.log_path)],
using_cache=False)
if self.options.init_epoch < 0:
self.options.init_epoch = get_initial_epoch(self.log_path)
self.metrics = [utils.f1_score]
self.loss_function = utils.dice_coef_loss
def LeNet5(batch_size, num_particles):
input_shape = _input_shape(batch_size)
return make_model(
stax.serial(
GeneralConv(('NCHW', 'OIHW', 'NHWC'),
out_chan=6,
filter_shape=(5, 5),
strides=(1, 1),
padding="VALID"), Relu,
MaxPool(window_shape=(2, 2), strides=(2, 2), padding="VALID"),
Conv(out_chan=16,
filter_shape=(5, 5),
strides=(1, 1),
padding="SAME"), Relu,
MaxPool(window_shape=(2, 2), strides=(2, 2), padding="SAME"),
Conv(out_chan=120,
filter_shape=(5, 5),
strides=(1, 1),
padding="VALID"), Relu,
MaxPool(window_shape=(2, 2),
strides=(2, 2), padding="SAME"), Flatten, Dense(84), Relu,
Dense(10), LogSoftmax), input_shape, num_particles)
import pandas as pd
from utils import make_model
from models import Model
if __name__ == "__main__":
input_features = [
'tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm',
'wind_speed_mps', 'rel_hum'
]
# column in dataframe to bse used as output/target
outputs = ['blaTEM_coppml']
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=1,
inputs=input_features,
outputs=outputs,
lr=0.0001)
df = pd.read_csv('../data/all_data_30min.csv')
model = Model(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals)
model.build_nn()
history = model.train_nn(indices='random')
y, obs = model.predict(st=0,
def main(args):
"""Main driver for evaluating different models.
Can be used in both training and testing mode.
"""
if args.seed is None:
args.seed = np.random.randint(1234567890)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# Use the factory method to make both model and environment (e.g. every
# call to make_env() will spawn a new environment using same parameters).
# Note that models aren't very large and can be run quickly on CPU.
env_creator = make_env(args.max_steps, args.is_test, args.render)
model_creator = make_model(args, torch.device('cpu'))
envs = []
for _ in range(args.remotes):
envs.append(
EnvWrapper.remote(env_creator, model_creator, args.seed_env))
# Trainable model does a significant amount of more work, so put on GPU
device = torch.device(
'cpu' if args.no_cuda or not torch.cuda.is_available() else 'cuda')
model = make_model(args, device)()
if args.checkpoint is not None:
model.load_checkpoint(args.checkpoint)
# Train
if not args.is_test:
checkpoint_dir = os.path.join('checkpoints', args.model)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
# Some methods have specialized memory implementations
memory = make_memory(args.model, args.buffer_size)
memory.load(**vars(args))
# Perform a validation step every full pass through the data
iters_per_epoch = args.buffer_size // args.batch_size
# Keep a running average of n-epochs worth of rollouts
step_queue = deque(maxlen=1 * args.rollouts * args.remotes)
reward_queue = deque(maxlen=step_queue.maxlen)
loss_queue = deque(maxlen=iters_per_epoch)
best = -np.inf
results = []
start = time.time()
for episode in range(args.max_epochs * iters_per_epoch):
loss = model.train(memory, **vars(args))
loss_queue.append(loss)
if episode % args.update_iter == 0:
model.update()
# Validation step;
# Here we take the weights from the current network, and distribute
# them to all remote instances. While the network trains for another
# epoch, these instances will run in parallel & evaluate the policy.
# If an epoch finishes before remote instances, training will be
# halted until outcomes are returned
if episode % iters_per_epoch == 0:
cur_episode = '%d' % (episode // iters_per_epoch)
model.save_checkpoint(os.path.join(checkpoint_dir,
cur_episode))
# Collect results from the previous epoch.
# Note that all steps of a rollout are returned, but only the
# last one will have the reward for that episode.
for device in ray.get(results):
for ep in device:
# (s0, act, r, s1, terminal, timestep)
step_queue.append(ep[-1][-1])
reward_queue.append(ep[-1][2])
# Update weights of remote network & perform rollouts
results = test(envs, model.get_weights(), args.rollouts,
args.explore)
print('Epoch: %s, Step: %2.4f, Reward: %1.2f, Loss: %2.4f, '\
'Took:%2.4fs' %
(cur_episode, np.mean(step_queue), np.mean(reward_queue),
np.mean(loss_queue), time.time() - start))
if np.mean(reward_queue) > best:
best = np.mean(reward_queue)
model.save_checkpoint(os.path.join(checkpoint_dir, 'best'))
start = time.time()
print('---------- Testing ----------')
# Note if this happens naturally after the model has been trained
# (and not from the --test flag), this will be wrong as it will use
# the training objects instead of testing.
results = test(envs, model.get_weights(), args.rollouts, args.explore)
steps, rewards = [], []
for device in ray.get(results):
for ep in device:
# (s0, act, r, s1, terminal, timestep)
steps.append(ep[-1][-1])
rewards.append(ep[-1][2])
print('Average across (%d) episodes: Step: %2.4f, Reward: %1.2f' %
(len(rewards), np.mean(steps), np.mean(rewards)))
# this file tests that given activations are working both as layers as well as activation functions withing a layer
from models import Model
from utils import make_model
import pandas as pd
data_config, nn_config, _ = make_model()
layers = {}
for lyr in [
'PRELU', "RELU", "TANH", "ELU", "LEAKYRELU", "THRESHOLDRELU", "SELU",
'sigmoid', 'hardsigmoid', 'crelu', 'relu6', 'softmax', 'softplus',
'softsign'
]:
layers[lyr] = {'config': {}}
layers["Dense"] = {'config': {'units': 1}}
nn_config['layers'] = layers
nn_config['epochs'] = 2
data_config['lookback'] = 1
df = pd.read_csv("../data/nasdaq100_padding.csv")
model = Model(data_config, nn_config, df)
model.build_nn()
model.train_nn()
def train(args):
torch.manual_seed(args.seed)
train_loader_LR, train_loader_HR, dataset_len = utils.make_dataset(args)
transformer = utils.make_model(args)
if args.updata:
transformer.load_state_dict(torch.load(args.modeldir))
vgg = utils.make_vggmodel(args)
optimizer = Adam(transformer.parameters(), lr=args.lr)
mse_loss = torch.nn.MSELoss()
if args.cuda:
mse_loss.cuda()
for e in range(args.epochs):
log_msg = "pix_weight = "+str(args.pix_weight)+" content_weight = "+str(args.content_weight)
print(log_msg)
transformer.train()
agg_content_loss = 0
agg_pix_loss = 0
count = 0
for batch_id, ((x, x_),(style,y_)) in enumerate(zip(train_loader_LR,train_loader_HR)):
n_batch = len(x)
count += n_batch
optimizer.zero_grad()
pix_x_v = Variable(x)
pix_s_v = Variable(style,requires_grad=False)
pix_loss = 0
content_loss = 0
if args.cuda:
x=x.cuda()
style=style.cuda()
pix_x_v=pix_x_v.cuda()
pix_s_v=pix_s_v.cuda()
output = transformer(pix_x_v)
pix_loss = args.pix_weight * mse_loss(output, pix_s_v)
vgg_s = Variable(style.clone(),requires_grad=False)
vgg_x = utils.init_vgg_input(output)
vgg_s = utils.init_vgg_input(vgg_s)
feature_x = vgg(vgg_x)
feature_s = vgg(vgg_s)
f_s_v = Variable(feature_s[1].data, requires_grad=False)
content_loss = args.content_weight * mse_loss(feature_x[1], f_s_v)
total_loss = 0
if args.pix_weight>0:
total_loss+=pix_loss
if args.content_weight>0:
total_loss+=content_loss
total_loss.backward()
optimizer.step()
agg_content_loss += content_loss.data[0]
agg_pix_loss += pix_loss.data[0]
up = 10000
if(batch_id + 1) % args.log_interval == 0:
mesg = "{}\tEpoch {}:\t[{}/{}]\tcontent: {:.6f}\tpix: {:.6f}\ttotal: {:.6f}".format(\
time.ctime(), e + 1, count, dataset_len,\
agg_content_loss*up / (batch_id + 1),\
agg_pix_loss *up/ (batch_id + 1),\
(agg_content_loss + agg_pix_loss)*up / (batch_id + 1))
print(mesg)
if e%2 == 0:
transformer.eval()
transformer.cpu()
save_model_filename = "epoch_" + str(args.epochs) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str(
args.content_weight) + "_SRCNN_" + str(args.srcnn) + ".model"
save_model_path = os.path.join(args.save_model_dir, save_model_filename)
torch.save(transformer.state_dict(), save_model_path)
transformer.eval()
transformer.cpu()
save_model_filename = "epoch_" + str(args.epochs) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str(
args.content_weight) + "_SRCNN_" + str(args.srcnn) + ".model"
save_model_path = os.path.join(args.save_model_dir, save_model_filename)
torch.save(transformer.state_dict(), save_model_path)
print("\nDone, trained model saved at", save_model_path)
def main():
"""
The main executable function
"""
parser = make_argument_parser()
args = parser.parse_args()
input_dirs = args.inputdirs
tf = args.factor
valid_chroms = args.validchroms
valid_input_dirs = args.validinputdirs
test_chroms = args.testchroms
epochs = args.epochs
patience = args.patience
learningrate = args.learningrate
seed = args.seed
utils.set_seed(seed)
dropout_rate = args.dropout
L = args.seqlen
w = args.motifwidth
utils.L = L
utils.w = w
utils.w2 = w / 2
negatives = args.negatives
assert negatives > 0
meta = args.meta
gencode = args.gencode
motif = args.motif
num_motifs = args.kernels
num_recurrent = args.recurrent
num_dense = args.dense
features = ['bigwig']
if tf:
print 'Single-task training:', tf
singleTask = True
if meta:
print 'Including metadata features'
features.append('meta')
if gencode:
print 'Including genome annotations'
features.append('gencode')
else:
print 'Multi-task training'
singleTask = False
#Cannot use any metadata features
assert not meta
assert not gencode
if args.outputdir is None:
clobber = True
output_dir = args.outputdirc
else:
clobber = False
output_dir = args.outputdir
try: # adapted from dreme.py by T. Bailey
os.makedirs(output_dir)
except OSError as exc:
if exc.errno == errno.EEXIST:
if not clobber:
print >> sys.stderr, (
'output directory (%s) already exists '
'but you specified not to clobber it') % output_dir
sys.exit(1)
else:
print >> sys.stderr, ('output directory (%s) already exists '
'so it will be clobbered') % output_dir
print 'Loading genome'
genome = utils.load_genome()
if valid_input_dirs:
print 'You specified at least one validation input directory'
assert singleTask # This option only works for single-task training
print 'Loading ChIP labels'
if singleTask:
chip_bed_list, nonnegative_regions_bed_list = \
utils.load_chip_singleTask(input_dirs, tf)
if valid_input_dirs:
valid_chip_bed_list, valid_nonnegative_regions_bed_list = \
utils.load_chip_singleTask(valid_input_dirs, tf)
num_tfs = 1
else:
assert len(
input_dirs) == 1 # multi-task training only supports one cell line
input_dir = input_dirs[0]
tfs, positive_windows, y_positive, nonnegative_regions_bed = \
utils.load_chip_multiTask(input_dir)
num_tfs = len(tfs)
print 'Loading bigWig data'
bigwig_names, bigwig_files_list = utils.load_bigwigs(input_dirs)
num_bigwigs = len(bigwig_names)
if valid_input_dirs:
valid_bigwig_names, valid_bigwig_files_list = utils.load_bigwigs(
valid_input_dirs)
assert valid_bigwig_names == bigwig_names
if not singleTask:
bigwig_files = bigwig_files_list[0]
if meta:
print 'Loading metadata features'
meta_names, meta_list = utils.load_meta(input_dirs)
if valid_input_dirs:
valid_meta_names, valid_meta_list = utils.load_load(
valid_input_dirs)
assert valid_meta_names == meta_names
else: # meta option was not selected, pass empty metadata features to the functions
meta_list = [[] for bigwig_files in bigwig_files_list]
if valid_input_dirs:
valid_meta_list = [[] for bigwig_files in valid_bigwig_files_list]
print 'Making features'
if singleTask:
if not valid_input_dirs: #validation directories not used, must pass placeholder values
valid_chip_bed_list = None
valid_nonnegative_regions_bed_list = None
valid_bigwig_files_list = None
valid_meta_list = None
datagen_train, datagen_valid = \
utils.make_features_singleTask(chip_bed_list,
nonnegative_regions_bed_list, bigwig_files_list, bigwig_names,
meta_list, gencode, genome, epochs, negatives, valid_chroms, test_chroms,
valid_chip_bed_list, valid_nonnegative_regions_bed_list,
valid_bigwig_files_list, valid_meta_list)
else:
datagen_train, datagen_valid = \
utils.make_features_multiTask(positive_windows, y_positive,
nonnegative_regions_bed, bigwig_files, bigwig_names,
genome, epochs, valid_chroms, test_chroms)
print 'Building model'
if num_recurrent == 0:
print 'You specified 0 LSTM units. Omitting BLSTM layer'
if num_recurrent < 0:
print 'You specified less than 0 LSTM units. Replacing BLSTM layer with global max-pooling layer'
if meta or gencode:
num_meta = 0
if meta:
num_meta = len(meta_names)
if gencode:
num_meta += 6
model = utils.make_meta_model(num_tfs, num_bigwigs, num_meta,
num_motifs, num_recurrent, num_dense,
dropout_rate)
else:
model = utils.make_model(num_tfs, num_bigwigs, num_motifs,
num_recurrent, num_dense, dropout_rate)
if motif:
assert singleTask # This option only works with single-task training
motifs_db = utils.load_motif_db('resources/HOCOMOCOv9.meme')
if tf in motifs_db:
print 'Injecting canonical motif'
pwm = motifs_db[tf]
pwm += 0.001
pwm = pwm / pwm.sum(axis=1)[:, np.newaxis]
pwm = np.log2(pwm / 0.25)
utils.inject_pwm(model, pwm)
output_tf_file = open(output_dir + '/chip.txt', 'w')
if singleTask:
output_tf_file.write("%s\n" % tf)
else:
for tf in tfs:
output_tf_file.write("%s\n" % tf)
output_tf_file.close()
output_feature_file = open(output_dir + '/feature.txt', 'w')
for feature in features:
output_feature_file.write("%s\n" % feature)
output_feature_file.close()
output_bw_file = open(output_dir + '/bigwig.txt', 'w')
for bw in bigwig_names:
output_bw_file.write("%s\n" % bw)
output_bw_file.close()
if meta:
output_meta_file = open(output_dir + '/meta.txt', 'w')
for meta_name in meta_names:
output_meta_file.write("%s\n" % meta_name)
output_meta_file.close()
model_json = model.to_json()
output_json_file = open(output_dir + '/model.json', 'w')
output_json_file.write(model_json)
output_json_file.close()
train(datagen_train, datagen_valid, model, epochs, patience, learningrate,
output_dir)
self.k_model = self.compile(model_inputs=inputs, outputs=predictions)
return
if __name__ == "__main__":
input_features = ['tide_cm', 'wat_temp_c', 'sal_psu', 'air_temp_c', 'pcp_mm', 'pcp3_mm', 'wind_speed_mps',
'rel_hum']
# column in dataframe to bse used as output/target
outputs = ['blaTEM_coppml', 'sul1_coppml']
data_config, nn_config, total_intervals = make_model(batch_size=4,
lookback=15,
inputs=input_features,
outputs=outputs,
lr=0.0001,
epochs=20,
val_fraction=0.3, # TODO why less than 0.3 give error here?
test_fraction=0.3
)
cwd = os.getcwd()
df = pd.read_csv(os.path.join(os.path.dirname(cwd), "data\\all_data_30min.csv"))
df.index = pd.to_datetime(df['Date_Time2'])
model = MultiSite(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals
)
def make_multi_model(input_model, from_config=False, config_path=None, weights=None,
batch_size=8, lookback=19, lr=1.52e-5, **kwargs):
data_config, nn_config, total_intervals = make_model(batch_size=batch_size,
lookback=lookback,
lr=lr,
ignore_nans=True,
**kwargs)
lookback = data_config['lookback']
data_config['inputs'] = ['tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)',
'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3', 'ORGP_INppm']
data_config['outputs'] = ['obs_chla_1', 'obs_chla_3', 'obs_chla_8', 'obs_chla_12']
data_config['val_fraction'] = 0.2
fpath = "D:\\playground\\paper_with_codes\\dl_ts_prediction\\data"
df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv'))
df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv'))
df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv'))
df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv'))
chl_1_nonan_idx = df_1[:-lookback][~df_1['obs_chla_1'][:-lookback].isna().values].index # 175
chl_3_nonan_idx = df_3[:-lookback][~df_3['obs_chla_3'][:-lookback].isna().values].index # 181
chl_8_nonan_idx = df_8[:-lookback][~df_8['obs_chla_8'][:-lookback].isna().values].index # 380
chl_12_nonan_idx = df_12[:-lookback][~df_12['obs_chla_12'][:-lookback].isna().values].index # 734
# len(list(set().union(chl_1_nonan_idx.to_list(), chl_3_nonan_idx.to_list(), chl_8_nonan_idx.to_list(), chl_12_nonan_idx.to_list()
# ))) = 1162
train_idx_chl_1, test_idx_chl_1 = train_test_split(chl_1_nonan_idx, test_size=data_config['val_fraction'], random_state=313)
train_idx_chl_3, test_idx_chl_3 = train_test_split(chl_3_nonan_idx, test_size=data_config['val_fraction'], random_state=313)
train_idx_chl_8, test_idx_chl_8 = train_test_split(chl_8_nonan_idx, test_size=data_config['val_fraction'], random_state=313)
train_idx_chl_12, test_idx_chl_12 = train_test_split(chl_12_nonan_idx, test_size=data_config['val_fraction'], random_state=313)
_train_idx = list(set().union(train_idx_chl_1.to_list(),
train_idx_chl_3.to_list(),
train_idx_chl_8.to_list(),
train_idx_chl_12.to_list()
)) # 863
_test_idx = list(set().union(test_idx_chl_1.to_list(),
test_idx_chl_3.to_list(),
test_idx_chl_8.to_list(),
test_idx_chl_12.to_list()
)) # 406
df_1.index = pd.to_datetime(df_1['date'])
df_3.index = pd.to_datetime(df_3['date'])
df_8.index = pd.to_datetime(df_8['date'])
df_12.index = pd.to_datetime(df_12['date'])
if from_config:
_model = input_model.from_config(config_path=config_path,
data=[df_1, df_3, df_8, df_12])
_model.build_nn()
_model.load_weights(weights)
else:
_model = input_model(data_config=data_config,
nn_config=nn_config,
data = [df_1, df_3, df_8, df_12]
)
_model.build_nn()
return _model, _train_idx, _test_idx
dirpath,
cached=True)
X_train = X_train.loc[:, [
'sim_time', 'left_pwm', 'right_pwm', 'theta(t-1)_cos', 'theta(t-1)_sin'
]].values.reshape(-1, timestep, p)
X_test = X_test.loc[:, [
'sim_time', 'left_pwm', 'right_pwm', 'theta(t-1)_cos', 'theta(t-1)_sin'
]].values.reshape(-1, timestep, p)
theta_y_train = y_train.loc[:, ['theta_cos', 'theta_sin']].values.reshape(
-1, timestep, J)
theta_y_test = y_test.loc[:, ['theta_cos', 'theta_sin']].values.reshape(
-1, timestep, J)
# pdb.set_trace()
num_batches = 16
theta_model = make_model(num_batches, timestep, layers_dims, lr=1e-4)
train_trial_names = load_obj(dirpath, 'train_trial_names')
test_trial_names = load_obj(dirpath, 'test_trial_names')
iterations = 3
epochs = 30
# learning curver
train_loss_history = []
test_loss_history = []
# decoded_y_train = decode_angles(y_train)
# y_test = decode_angles(y_test)
for it in range(iterations):
print("iteration %d" % it)
history = theta_model.fit(X_train,
theta_y_train,
def main(args):
"""Main driver for evaluating different models.
Can be used in both training and testing mode.
"""
if args.seed is None:
args.seed = np.random.randint(1234567890)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# Make the remote environments; the models aren't very large and can
# be run fairly quickly on the cpus. Save the GPUs for training
env_creator = make_env(args.max_steps, args.is_test, args.render)
model_creator = make_model(args, torch.device('cpu'))
envs = []
for _ in range(args.remotes):
envs.append(EnvWrapper(env_creator, model_creator, args.seed_env))
# We'll put the trainable model on the GPU if one's available
device = torch.device(
'cpu' if args.no_cuda or not torch.cuda.is_available() else 'cuda')
model = make_model(args, device)()
if args.checkpoint is not None:
model.load_checkpoint(args.checkpoint)
# Train
if not args.is_test:
checkpoint_dir = os.path.join('checkpoints', args.model)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
# Some methods have specialized memory implementations
memory = make_memory(args.model, args.buffer_size)
memory.load(**vars(args))
# Perform a validation step every full pass through the data
iters_per_epoch = args.buffer_size // args.batch_size
# Keep a running average of n-epochs worth of rollouts
step_queue = deque(maxlen=1 * args.rollouts * args.remotes)
reward_queue = deque(maxlen=step_queue.maxlen)
loss_queue = deque(maxlen=iters_per_epoch)
start = time.time()
for episode in range(args.max_epochs * iters_per_epoch):
loss = model.train(memory, **vars(args))
loss_queue.append(loss)
if episode % args.update_iter == 0:
model.update()
# Validation step;
# Here we take the weights from the current network, and distribute
# them to all remote instances. While the network trains for another
# epoch, these instances will run in parallel & evaluate the policy.
# If an epoch finishes before remote instances, training will be
# halted until outcomes are returned
if (episode + 1) % iters_per_epoch == 0:
cur_episode = '%d' % (episode // iters_per_epoch)
model.save_checkpoint(os.path.join(checkpoint_dir,
cur_episode))
# Collect results from the previous epoch
for device in test(envs, model.get_weights(), args.rollouts,
args.explore):
for ep in device:
# (s0, act, r, s1, terminal, timestep)
step_queue.append(ep[-1][-1])
reward_queue.append(ep[-1][2])
print('Epoch: %s, Step: %2.4f, Reward: %1.2f, Loss: %2.4f, '\
'Took:%2.4fs' %
(cur_episode, np.mean(step_queue), np.mean(reward_queue),
np.mean(loss_queue), time.time() - start))
start = time.time()
print('---------- Testing ----------')
steps, rewards = [], []
for device in test(envs, model.get_weights(), args.rollouts, args.explore):
for ep in device:
# (s0, act, r, s1, terminal, timestep)
steps.append(ep[-1][-1])
rewards.append(ep[-1][2])
print('Average across (%d) episodes: Step: %2.4f, Reward: %1.2f' %
(args.rollouts * args.remotes, np.mean(steps), np.mean(rewards)))
print(out.isna().sum())
# put four chunks of missing intervals
intervals = [(100, 200), (1000, 8000), (10000, 31000)]
for interval in intervals:
st, en = interval[0], interval[1]
out[st:en] = np.nan
df["NDX"] = out
print("{} nan values created in NDX column".format(out.isna().sum()))
# verify the missing values
print(df[98:108])
data_config, nn_config, _ = make_model(batch_size=32, lookback=5, lr=0.0001)
model = DualAttentionModel(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=[(0, 99), (200, 999), (8000, 9999),
(31000, 40561)])
model.build_nn()
history = model.train_nn(indices='random')
y, obs = model.predict(indices=model.test_indices, use_datetime_index=False)
# tr_y, tr_obs = model.predict(indices=model.train_indices, pref='train', use_datetime_index=False)
model.view_model(st=0, save=True) # takes a lot of time to save all plots
"Dropout": {
'config': {
'rate': 0.3
}
},
"Dense": {
'config': {
'units': 1
}
}
}
data_config, nn_config, total_intervals = make_model(layers=layers,
batch_size=12,
lookback=15,
lr=8.95e-5,
ignore_nans=False,
inputs=input_features,
outputs=outputs,
epochs=10)
fname = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"data\\data_30min.csv")
df = pd.read_csv(fname) # must be 2d dataframe
model = CustomModel(data_config=data_config, nn_config=nn_config, data=df)
model.build_nn()
history = model.train_nn(indices='random', tensorboard=True)
test_pred, test_obs = model.predict(indices=model.test_indices)
from models import IMVLSTMModel
from utils import make_model
import pandas as pd
lookback = 10
epochs = 50
df = pd.read_csv("../data/nasdaq100_padding.csv")
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=lookback,
lr=0.001,
use_predicted_output=True,
epochs=epochs)
model = IMVLSTMModel(data_config=data_config, nn_config=nn_config, data=df)
model.build_nn()
h = model.train_nn(st=0, en=1000)
x, y = model.predict(st=0, en=1000)
model.plot_activations()
split=True)
# X columns: ['sim_time', 'left_pwm', 'right_pwm', 'model_pos_x(t-1)', 'model_pos_y(t-1)', 'theta(t-1)_cos', 'theta(t-1)_sin']
# pdb.set_trace()
# X_train = X_train.values.reshape(
# -1, timestep, p)
# X_test = X_test.values.reshape(
# -1, timestep, p)
# y_train = y_train.values.reshape(
# -1, timestep, J)
# y_test = y_test.values.reshape(
# -1, timestep, J)
if model_cached:
model = load_model(dirpath, model_fname)
else:
model = make_model(None, layers_dims)
iterations = 300
epochs = 30
# learning curver
train_loss_history = []
test_loss_history = []
# decoded_y_train = decode_angles(y_train)
# y_test = decode_angles(y_test)
# train_trial_names = load_obj(dirpath, 'train_trial_names')
# test_trial_names = load_obj(dirpath, 'test_trial_names')
test_fname = 'v_shape_path_2.csv'
testfile = os.path.join(dirpath, test_fname)
embedding_std = model_config["embedding_weight_std"]
max_positions = model_config["max_positions"]
freeze_embedding = model_config["freeze_embedding"]
trainable_positional_encodings = model_config[
"trainable_positional_encodings"]
use_memory_mask = model_config["use_memory_mask"]
query_position_rate = model_config["query_position_rate"]
key_position_rate = model_config["key_position_rate"]
window_backward = model_config["window_backward"]
window_ahead = model_config["window_ahead"]
key_projection = model_config["key_projection"]
value_projection = model_config["value_projection"]
dv3 = make_model(
n_speakers, speaker_dim, speaker_embed_std, embed_dim, padding_idx,
embedding_std, max_positions, n_vocab, freeze_embedding,
filter_size, encoder_channels, n_mels, decoder_channels, r,
trainable_positional_encodings, use_memory_mask,
query_position_rate, key_position_rate, window_backward,
window_ahead, key_projection, value_projection, downsample_factor,
linear_dim, use_decoder_states, converter_channels, dropout)
summary(dv3)
# =========================loss=========================
loss_config = config["loss"]
masked_weight = loss_config["masked_loss_weight"]
priority_freq = loss_config["priority_freq"] # Hz
priority_bin = int(priority_freq / (0.5 * sample_rate) * linear_dim)
priority_freq_weight = loss_config["priority_freq_weight"]
binary_divergence_weight = loss_config["binary_divergence_weight"]
guided_attention_sigma = loss_config["guided_attention_sigma"]
criterion = TTSLoss(
masked_weight=masked_weight,
},
'Dropout_1': {
'rate': 0.3
},
'Dense_2': {
'units': 16,
'activation': 'relu'
},
'Dense_3': {
'units': 9
}
}
data_config, nn_config, _ = make_model(
inputs=['input_' + str(i) for i in range(cols - 1)],
outputs=['input_' + str(cols - 1)],
lookback=1,
layers=layers,
epochs=100)
# Define Quantiles
quantiles = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
# Initiate Model
model = QuantileModel(data_config, nn_config, data)
# Assign loss for the model
model.loss = qloss
# the quantiles must also be assigned to the model for post-processing purpose
model.quantiles = quantiles
# Build the NN
# this example shows how to build the Models from `from_checkout` class method
# first we will train and save a simple model and load it from config file
import pandas as pd
import os
from utils import make_model
from dl4seq.main import Model
data_config, nn_config, total_intervals = make_model(lookback=1)
fname = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data\\nasdaq100_padding.csv")
df = pd.read_csv(fname)
model = Model(data_config=data_config,
nn_config=nn_config,
data=df,
)
model.build_nn()
history = model.train_nn(indices='random')
# for clarity, delete the model, although it is overwritten
del model
# Load the `Model` from checkpoint, provide the checkpoint
cpath = "provide complete path of config file"
model = Model.from_config(cpath, data=df)
print('Test f1 of %s: %0.3f.' % (options.run_name, f1))
print('Scoring time cost: %0.2f(min).' % ((time.time() - t0) / 60))
if __name__ == '__main__':
options = None
if utils.is_py3():
# Python3
options = configs.get_config('local_config_end2end.json')
else:
# Python2
options = configs.get_config(argparse.ArgumentParser())
test_files = json.load(open(options.train_val_test_config))['test']
input_shape = (options.target_size[0], options.target_size[1],
len(options.use_chans))
model = utils.make_model(options.model_name, input_shape)
if utils.is_py3():
p = os.path.join(options.output_path, options.weight_path)
print("Load weights from '%s'." % p)
model.load_weights(p)
for k in range(len(test_files)):
test_files[k] = os.path.join(options.input_path, options.data_path,
test_files[k])
else:
path = os.path.join(options.output_path, options.weight_path)
if tf.gfile.Exists(path):
print('Copy %s to %s.' % (path, options.weight_path))
tf.gfile.Copy(path, options.weight_path, overwrite=True)
else:
print('There is not pretrained model: %s' % path)
model.load_weights(options.weight_path)
def main():
"""We're going to create a beautiful app with Streamlit"""
menu = [
"Accueil", "Pandas Profile", "D-Tale", "Visualisation",
"Nuage de mots", "Machine Learning", "A propos"
]
selection = st.sidebar.selectbox("Fonctions", menu)
if selection == "Pandas Profile":
components.html(ha.alert_panda_prof(), height=190)
my_data = st.file_uploader("Charger le fichier CSV", type=['csv'])
if my_data is not None:
df = pd.read_csv(my_data)
st.dataframe(df.head(10))
eda_profil = ProfileReport(df,
title='Pandas Profiling Report...',
explorative=True)
st_profile_report(eda_profil)
elif selection == "D-Tale":
components.html(ha.alert_dtale(), height=190)
data_file = st.file_uploader("Charger le fichier CSV", type=['csv'])
if data_file is not None:
df = pd.read_csv(data_file)
st.dataframe(df.head())
d = dtale.show(df)
d.open_browser()
if st.button("Générer le rapport"):
report = sv.analyze(df)
report.show_html()
utils.st_display_sweetviz("SWEETVIZ_REPORT.html")
components.html(ha.alert_warning(), 1000)
elif selection == "Visualisation":
# Image d'entête
image = Image.open('dataviz.png')
col2, col1 = st.beta_columns([1, 3])
col2.image(
"https://idoc-projets.ias.u-psud.fr/redmine/attachments/download/121/sunburst.gif",
caption='',
width=None,
use_column_width=True)
col1.image(image, caption='', width=None, use_column_width=True)
# Visualisation catégorie / rating / reviews
cat1, cat2 = st.beta_columns(2)
datas = utils.lire_dataset(my_db_clean)
fig = px.histogram(datas,
x='Rating',
y='Category',
title='Somme des notes par catégorie',
color='Category')
cat1.plotly_chart(fig)
fig = px.histogram(datas,
x='Reviews',
y='Category',
title='Somme des commentaires par catégorie',
color='Category')
cat2.plotly_chart(fig)
# Visualisation sunburst / pie
perc1, perc2 = st.beta_columns(2)
fig = px.sunburst(datas,
path=['Type', 'Category', 'Genres'],
title='Types, Catégories et genres')
perc1.plotly_chart(fig)
fig = px.pie(datas,
names='Type',
title='Pourcentage apllication gratuites/Payantes',
color_discrete_sequence=px.colors.sequential.RdBu)
perc2.plotly_chart(fig)
# Visualisation histo
hist1, hist2 = st.beta_columns(2)
hist_data = [list(datas['Rating'])]
group_labels = ['Rating']
fig = ff.create_distplot(hist_data, group_labels)
hist1.plotly_chart(fig)
counts, bins = np.histogram(datas.Rating, bins=range(0, 6, 1))
bins = 0.5 * (bins[:-1] + bins[1:])
fig = px.bar(x=bins,
y=counts,
labels={
'x': 'Rating',
'y': 'Count'
},
title='Distribution des notes')
hist2.plotly_chart(fig)
elif selection == "Nuage de mots":
# General
img1, img2 = st.beta_columns(2)
img1.image('datas/wordcloud/general.png')
img2.image('datas/wordcloud/free_app.png')
# Free App
img3, img4 = st.beta_columns(2)
img3.image('datas/wordcloud/free_app_pos.png')
img4.image('datas/wordcloud/free_app_neg.png')
# Paid app
img5, img6 = st.beta_columns(2)
img5.image('datas/wordcloud/paid_app.png')
img6.image('datas/wordcloud/paid_app_pos.png')
img7, img8 = st.beta_columns(2)
img7.image('datas/wordcloud/paid_app_neg.png')
elif selection == "Machine Learning":
image = Image.open('machine learning.jpg')
col1, col2 = st.beta_columns([3, 1])
col2.image(
"https://static.wixstatic.com/media/bb7b70_d5fde322f7914060b7d997ba9d506a50~mv2.gif",
caption='',
width=None,
use_column_width=True)
col1.image(image, caption='', width=None, use_column_width=True)
if st.checkbox("Afficher le dataset"):
datas = utils.lire_dataset(my_db)
st.write(datas.head())
if st.checkbox("Afficher graph valeurs manquantes"):
col1, col2 = st.beta_columns([2, 1])
df = datas.isnull()
fig = px.imshow(df)
col1.plotly_chart(fig)
col2.write(datas.isnull().sum())
col2.write(
"On peut voir que la colonne **Rating** contient la plupart des valeurs manquantes. A sa suite on a **Current Ver**, **Adroid ver** et **Type**."
)
if st.checkbox("Afficher DB ok"):
datas = utils.lire_dataset(my_db_clean)
st.write(datas.head())
mat1, mat2 = st.beta_columns(2)
fig = px.scatter_matrix(
datas,
dimensions=["Rating", "Reviews", "Size", "Installs", "Price"],
color="Type",
symbol="Type",
title="Matrix de dispersion des variables continues")
fig.update_traces(diagonal_visible=False)
mat1.plotly_chart(fig)
fig = px.imshow(
datas[["Rating", "Reviews", "Size", "Installs",
"Price"]].corr(),
labels=dict(x="", y="", color="Corrélation"),
)
mat2.plotly_chart(fig)
if st.checkbox("Make model"):
#mon_score = utils.transform_var_model(my_db_clean)
mon_score = utils.make_model(my_db_clean)
st.success(mon_score)
elif selection == "A propos":
#st.subheader("Team presentation")
components.html(hp.pied_de_page(), height=800)
#components.iframe('http://www.ingemedia.net/',height=1000, scrolling=True)
else:
components.html(hp.entete_de_page(), height=1600)
# Compute gradients
trainable_vars = self.trainable_variables
gradients = tape.gradient(loss, trainable_vars)
# Update weights
self.optimizer.apply_gradients(zip(gradients, trainable_vars))
# Update metrics (includes the metric that tracks the loss)
self.compiled_metrics.update_state(y, y_pred)
# Return a dict mapping metric names to current value
return {m.name: m.result() for m in self.metrics}
data_config, nn_config, total_intervals = make_model(lstm_units=64,
dropout=0.4,
rec_dropout=0.5,
lstm_act='relu',
batch_size=32,
lookback=1,
lr=8.95e-5)
df = pd.read_csv('../data/nasdaq100_padding.csv')
model = Model(data_config=data_config,
nn_config=nn_config,
data=df
)
model.KModel = CustomModel
model.build_nn()
