def __init__(self, db): Model.__init__(self, db) self._table = 'Versao' self._selectItens = [ 'codigoFipe', 'codigoAnoModelo', 'codigoModelo', 'codigoMarca', 'codigoTipoVeiculo', 'codigoTabelaReferencia', 'titulo', 'valor' ]
def __init__(self, db): Model.__init__(self, db) self._table = 'TabelaReferencia' self._selectItens = [ 'codigoTabelaReferencia', 'codigoTipoVeiculo', 'titulo', 'hasCrawled' ]
def generate_signal_prices(dfs, feature_names_filename, model_params_filename, period=1e7, sample_th=10, returns_th=5): file = open(model_params_filename, 'r') model_params = json.load(file) file.close() features_config = load_feature_config(feature_names_filename) timestamps = [subsample(df, sample_th) for df in dfs] model = Model(**model_params) returns = [calc_return(df, period) for df in dfs] features = [generate_features(df, features_config) for df in dfs] returns = [calc_return(df, period) for df in dfs] returns = [get_non_duplicated(x) for x in returns] features = [get_non_duplicated(feature) for feature in features] sub_features = [ features[i].loc[timestamps[i]] for i in range(len(timestamps)) ] sub_returns = [ returns[i].loc[timestamps[i]] for i in range(len(timestamps)) ] signal_prices_list = [] for i in range(len(features)): test_features = features[i] X_test = get_non_duplicated(test_features) cond = (X_test.isnull().sum(axis=1) == 0) X_test = X_test.loc[cond] df = get_non_duplicated(dfs[i]) df = df.loc[cond] mid_prices = 0.5 * (df['ap0'] + df['bp0']) X_train = [] y_train = [] for j in range(len(features)): if i == j: continue train_features = sub_features[j] train_returns = sub_returns[j] cond = (~train_returns.isnull()) & (train_features.isnull().sum( axis=1) == 0) train_features = train_features.loc[cond] train_returns = train_returns.loc[cond] X_train.append(train_features) y_train.append(train_returns) X_train = pd.concat(X_train) y_train = pd.concat(y_train) X_train, y_train = subsample_returns(X_train, y_train, returns_th) model.fit(X_train.values, y_train.values) pred = model.predict(X_test) pred = pd.Series(pred, index=X_test.index) signal_prices = mid_prices + pred signal_prices_list.append(signal_prices) return signal_prices_list
def load_model(save_dir, sess, training=False, decoding=False, **kwargs): with open(os.path.join(save_dir, 'config.pkl'), 'rb') as f: saved_args = cPickle.load(f) saved_args.__dict__.update(kwargs) model = Model(saved_args, training=training, decoding=decoding) with sess.as_default(): tf.global_variables_initializer().run() saver = tf.train.Saver(tf.global_variables()) ckpt = tf.train.get_checkpoint_state(save_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) return model
def __init__(self, db): Model.__init__(self, db) self._table = 'Versao' self._selectItens = ['codigoFipe', 'codigoAnoModelo','codigoModelo','codigoMarca','codigoTipoVeiculo','codigoTabelaReferencia','titulo','valor']
def __init__(self, db): Model.__init__(self, db) self._table = 'Modelo' self._selectItens = ['codigoModelo','codigoMarca','codigoTipoVeiculo','codigoTabelaReferencia','titulo','hasCrawled']
t = [] t.extend([transforms.ToTensor()]) trans = transforms.Compose(t) pair = [seg, seg] transformed = list(trans(*pair)) seg = transformed[0] return to_one_hot(seg) if (__name__ == '__main__'): args = parse_args() with open(args.val_txt) as f: lines = f.readlines() f.close() model = Model(pretrainRes=False, temp=args.temp, uselayer=4) if (args.multiGPU): model = nn.DataParallel(model) print("=> loading checkpoint '{}'".format(args.checkpoint_dir)) checkpoint = torch.load(args.checkpoint_dir) best_loss = checkpoint['best_loss'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{} ({})' (epoch {})".format( args.checkpoint_dir, best_loss, checkpoint['epoch'])) model.cuda() model.eval() for cnt in range(0, len(lines)): line = lines[cnt] tmp = line.strip().split('/')[-1] video_nm = tmp
def __init__(self): Model.__init__(self, "prayers")
def post(self): model = Model() text = request.json['text'] predict = model.predict(text) return jsonify(predict)
def __init__(self): Model.__init__(self, "notes")
def train(args): _shm = args.model == 'shm' data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length, shm=_shm) args.vocab_size = data_loader.transformer.vocab_size args.boundary_symbols = [ data_loader.transformer.vocab[sym] for sym in args.boundary_symbols ] # check compatibility if training is continued from previously saved model if args.init_from is not None: # check if all necessary files exist assert os.path.isdir( args.init_from), " %s must be a a path" % args.init_from assert os.path.isfile(os.path.join(args.init_from, "config.pkl")),\ "config.pkl file does not exist in path %s" % args.init_from assert os.path.isfile(os.path.join(args.init_from, "transformer.pkl")),\ "transformer.pkl file does not exist in path %s" % args.init_from ckpt = tf.train.get_checkpoint_state(args.init_from) assert ckpt, "No checkpoint found" assert ckpt.model_checkpoint_path, "No model path found in checkpoint" # open old config and check if models are compatible with open(os.path.join(args.init_from, 'config.pkl'), 'rb') as f: saved_model_args = cPickle.load(f) need_be_same = ["model", "rnn_size", "num_layers", "seq_length"] for checkme in need_be_same: assert vars(saved_model_args)[checkme] == vars(args)[checkme],\ "Command line argument and saved model disagree on '%s' " % checkme # open saved vocab/dict and check if vocabs/dicts are compatible with open(os.path.join(args.init_from, 'transformer.pkl'), 'rb') as f: transformer = cPickle.load(f) assert transformer.tokens == data_loader.transformer.tokens, \ "Data and loaded model disagree on character set!" assert transformer.vocab == data_loader.transformer.vocab, \ "Data and loaded model disagree on dictionary mappings!" if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir) with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f: cPickle.dump(args, f) with open(os.path.join(args.save_dir, 'transformer.pkl'), 'wb') as f: cPickle.dump(data_loader.transformer, f) model = Model(args) with tf.Session() as sess: # instrument for tensorboard summaries = tf.summary.merge_all() writer = tf.summary.FileWriter( os.path.join(args.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S"))) writer.add_graph(sess.graph) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables()) # restore model if args.init_from is not None: saver.restore(sess, ckpt.model_checkpoint_path) lr = 0 for e in range(args.num_epochs): data_loader.reset_batch_pointer() state = sess.run(model.initial_state) for b in range(data_loader.num_batches): start = time.time() if _shm: x, y, zs = data_loader.next_batch() else: x, y = data_loader.next_batch() feed = {model.input_data: x, model.targets: y} if args.decay_every: new_lr = args.learning_rate * (args.decay_rate **(b // args.decay_every)) else: new_lr = args.learning_rate * (args.decay_rate**e) if new_lr != lr: lr = new_lr sess.run(tf.assign(model.lr, lr)) if _shm: for z, placeholder in zip(zs, model.boundary_data): feed[placeholder] = z if _shm or args.model == 'gru': for i, h in enumerate(model.initial_state): feed[h] = state[i] else: for i, (c, h) in enumerate(model.initial_state): feed[c] = state[i].c feed[h] = state[i].h # instrument for tensorboard start_run = time.time() # _ = sess.run([model.train_op], feed) summ, train_loss, state, _ = sess.run( [summaries, model.cost, model.final_state, model.train_op], feed) end_run = time.time() writer.add_summary(summ, e * data_loader.num_batches + b) end = time.time() print( "{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}, time/run = {:.3f}" .format(e * data_loader.num_batches + b, args.num_epochs * data_loader.num_batches, e, train_loss, end - start, end_run - start_run)) if (e * data_loader.num_batches + b) % args.save_every == 0\ or (e == args.num_epochs - 1 and b == data_loader.num_batches - 1): # save for the last result checkpoint_path = os.path.join(args.save_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=e * data_loader.num_batches + b) print("model saved to {}".format(checkpoint_path))