def __init__(self, db):
Model.__init__(self, db)
self._table = 'Versao'
self._selectItens = [
'codigoFipe', 'codigoAnoModelo', 'codigoModelo', 'codigoMarca',
'codigoTipoVeiculo', 'codigoTabelaReferencia', 'titulo', 'valor'
]
def __init__(self, db):
Model.__init__(self, db)
self._table = 'TabelaReferencia'
self._selectItens = [
'codigoTabelaReferencia', 'codigoTipoVeiculo', 'titulo',
'hasCrawled'
]
def generate_signal_prices(dfs,
feature_names_filename,
model_params_filename,
period=1e7,
sample_th=10,
returns_th=5):
file = open(model_params_filename, 'r')
model_params = json.load(file)
file.close()
features_config = load_feature_config(feature_names_filename)
timestamps = [subsample(df, sample_th) for df in dfs]
model = Model(**model_params)
returns = [calc_return(df, period) for df in dfs]
features = [generate_features(df, features_config) for df in dfs]
returns = [calc_return(df, period) for df in dfs]
returns = [get_non_duplicated(x) for x in returns]
features = [get_non_duplicated(feature) for feature in features]
sub_features = [
features[i].loc[timestamps[i]] for i in range(len(timestamps))
]
sub_returns = [
returns[i].loc[timestamps[i]] for i in range(len(timestamps))
]
signal_prices_list = []
for i in range(len(features)):
test_features = features[i]
X_test = get_non_duplicated(test_features)
cond = (X_test.isnull().sum(axis=1) == 0)
X_test = X_test.loc[cond]
df = get_non_duplicated(dfs[i])
df = df.loc[cond]
mid_prices = 0.5 * (df['ap0'] + df['bp0'])
X_train = []
y_train = []
for j in range(len(features)):
if i == j:
continue
train_features = sub_features[j]
train_returns = sub_returns[j]
cond = (~train_returns.isnull()) & (train_features.isnull().sum(
axis=1) == 0)
train_features = train_features.loc[cond]
train_returns = train_returns.loc[cond]
X_train.append(train_features)
y_train.append(train_returns)
X_train = pd.concat(X_train)
y_train = pd.concat(y_train)
X_train, y_train = subsample_returns(X_train, y_train, returns_th)
model.fit(X_train.values, y_train.values)
pred = model.predict(X_test)
pred = pd.Series(pred, index=X_test.index)
signal_prices = mid_prices + pred
signal_prices_list.append(signal_prices)
return signal_prices_list
def load_model(save_dir, sess, training=False, decoding=False, **kwargs):
with open(os.path.join(save_dir, 'config.pkl'), 'rb') as f:
saved_args = cPickle.load(f)
saved_args.__dict__.update(kwargs)
model = Model(saved_args, training=training, decoding=decoding)
with sess.as_default():
tf.global_variables_initializer().run()
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(save_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
return model
def __init__(self, db): Model.__init__(self, db) self._table = 'Versao' self._selectItens = ['codigoFipe', 'codigoAnoModelo','codigoModelo','codigoMarca','codigoTipoVeiculo','codigoTabelaReferencia','titulo','valor']
def __init__(self, db): Model.__init__(self, db) self._table = 'Modelo' self._selectItens = ['codigoModelo','codigoMarca','codigoTipoVeiculo','codigoTabelaReferencia','titulo','hasCrawled']
t = []
t.extend([transforms.ToTensor()])
trans = transforms.Compose(t)
pair = [seg, seg]
transformed = list(trans(*pair))
seg = transformed[0]
return to_one_hot(seg)
if (__name__ == '__main__'):
args = parse_args()
with open(args.val_txt) as f:
lines = f.readlines()
f.close()
model = Model(pretrainRes=False, temp=args.temp, uselayer=4)
if (args.multiGPU):
model = nn.DataParallel(model)
print("=> loading checkpoint '{}'".format(args.checkpoint_dir))
checkpoint = torch.load(args.checkpoint_dir)
best_loss = checkpoint['best_loss']
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{} ({})' (epoch {})".format(
args.checkpoint_dir, best_loss, checkpoint['epoch']))
model.cuda()
model.eval()
for cnt in range(0, len(lines)):
line = lines[cnt]
tmp = line.strip().split('/')[-1]
video_nm = tmp
def __init__(self):
Model.__init__(self, "prayers")
def post(self):
model = Model()
text = request.json['text']
predict = model.predict(text)
return jsonify(predict)
def __init__(self):
Model.__init__(self, "notes")
def train(args):
_shm = args.model == 'shm'
data_loader = TextLoader(args.data_dir,
args.batch_size,
args.seq_length,
shm=_shm)
args.vocab_size = data_loader.transformer.vocab_size
args.boundary_symbols = [
data_loader.transformer.vocab[sym] for sym in args.boundary_symbols
]
# check compatibility if training is continued from previously saved model
if args.init_from is not None:
# check if all necessary files exist
assert os.path.isdir(
args.init_from), " %s must be a a path" % args.init_from
assert os.path.isfile(os.path.join(args.init_from, "config.pkl")),\
"config.pkl file does not exist in path %s" % args.init_from
assert os.path.isfile(os.path.join(args.init_from, "transformer.pkl")),\
"transformer.pkl file does not exist in path %s" % args.init_from
ckpt = tf.train.get_checkpoint_state(args.init_from)
assert ckpt, "No checkpoint found"
assert ckpt.model_checkpoint_path, "No model path found in checkpoint"
# open old config and check if models are compatible
with open(os.path.join(args.init_from, 'config.pkl'), 'rb') as f:
saved_model_args = cPickle.load(f)
need_be_same = ["model", "rnn_size", "num_layers", "seq_length"]
for checkme in need_be_same:
assert vars(saved_model_args)[checkme] == vars(args)[checkme],\
"Command line argument and saved model disagree on '%s' " % checkme
# open saved vocab/dict and check if vocabs/dicts are compatible
with open(os.path.join(args.init_from, 'transformer.pkl'), 'rb') as f:
transformer = cPickle.load(f)
assert transformer.tokens == data_loader.transformer.tokens, \
"Data and loaded model disagree on character set!"
assert transformer.vocab == data_loader.transformer.vocab, \
"Data and loaded model disagree on dictionary mappings!"
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
cPickle.dump(args, f)
with open(os.path.join(args.save_dir, 'transformer.pkl'), 'wb') as f:
cPickle.dump(data_loader.transformer, f)
model = Model(args)
with tf.Session() as sess:
# instrument for tensorboard
summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter(
os.path.join(args.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
# restore model
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
lr = 0
for e in range(args.num_epochs):
data_loader.reset_batch_pointer()
state = sess.run(model.initial_state)
for b in range(data_loader.num_batches):
start = time.time()
if _shm:
x, y, zs = data_loader.next_batch()
else:
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.targets: y}
if args.decay_every:
new_lr = args.learning_rate * (args.decay_rate
**(b // args.decay_every))
else:
new_lr = args.learning_rate * (args.decay_rate**e)
if new_lr != lr:
lr = new_lr
sess.run(tf.assign(model.lr, lr))
if _shm:
for z, placeholder in zip(zs, model.boundary_data):
feed[placeholder] = z
if _shm or args.model == 'gru':
for i, h in enumerate(model.initial_state):
feed[h] = state[i]
else:
for i, (c, h) in enumerate(model.initial_state):
feed[c] = state[i].c
feed[h] = state[i].h
# instrument for tensorboard
start_run = time.time()
# _ = sess.run([model.train_op], feed)
summ, train_loss, state, _ = sess.run(
[summaries, model.cost, model.final_state, model.train_op],
feed)
end_run = time.time()
writer.add_summary(summ, e * data_loader.num_batches + b)
end = time.time()
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}, time/run = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches, e,
train_loss, end - start, end_run - start_run))
if (e * data_loader.num_batches + b) % args.save_every == 0\
or (e == args.num_epochs - 1 and
b == data_loader.num_batches - 1):
# save for the last result
checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
saver.save(sess,
checkpoint_path,
global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
