def main():
"""Run only when this file is called directly."""
# Setup hyperparameters
CONFIG = get_config()
# Log to File and Console
logger = get_logger(log_to_console=True, log_to_file=CONFIG.LOG_TO_FILE)
# Choose CPU and GPU
torch.set_num_threads(CONFIG.CPU_THREADS)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if not torch.cuda.is_available():
logger.warning("GPU not available: this run could be slow.")
# Setup environment
env = gym.make(CONFIG.ENV_NAME)
eval_env = gym.make(CONFIG.ENV_NAME)
# Fix random seeds
if CONFIG.RANDOM_SEED is not None:
set_global_random_seeds(CONFIG.RANDOM_SEED,
use_numpy=True,
use_torch=True)
# NOTE(seungjaeryanlee): Seed for env and eval_env are different for fair evaluation
set_env_random_seeds(env, CONFIG.RANDOM_SEED)
set_env_random_seeds(eval_env, CONFIG.RANDOM_SEED + 1)
else:
logger.warning(
"Running without a random seed: this run is NOT reproducible.")
# Setup agent and replay buffer
q_net = QNetwork(env.observation_space.shape[0],
env.action_space.n).to(device)
optimizer = optim.RMSprop(
q_net.parameters(),
lr=CONFIG.RMSPROP_LR,
alpha=CONFIG.RMSPROP_DECAY,
eps=CONFIG.RMSPROP_EPSILON,
momentum=CONFIG.RMSPROP_MOMENTUM,
weight_decay=CONFIG.RMSPROP_WEIGHT_DECAY,
centered=CONFIG.RMSPROP_IS_CENTERED,
)
if CONFIG.LOAD_PATH:
# Load parameters if possible
load_models(CONFIG.LOAD_PATH, q_net=q_net, optimizer=optimizer)
dqn_agent = DQNAgent(env, q_net, optimizer, device)
replay_buffer = CircularReplayBuffer(env,
maxlen=CONFIG.REPLAY_BUFFER_SIZE,
device=device)
# Train and evaluate agent
train_eval(dqn_agent, replay_buffer, env, eval_env, device, logger, CONFIG)
def webcam():
models = load_models()
iteration = 0
cam = cv2.VideoCapture(0)
cv2.namedWindow('HackDrone', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('HackDrone', cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
while True:
ret_val, image = cam.read()
if iteration > 5:
# RECOGNITION
print('Recognition')
result = recognize(image, models)
if result is not None:
letter, prob = result
text = 'Letter %s, confidence: %.2f' % (letter, prob)
cv2.putText(image, text, bottomLeftCornerOfText, font,
fontScale, fontColor, lineType)
cv2.imshow('HackDrone', image)
sleep(1)
iteration += 1
if cv2.waitKey(1) == 27:
break # esc to quit
cv2.destroyAllWindows()
def main(args):
vocab = load_vocab()
encoder = CNNEncoder()
decoder = DecoderRNN(512,512,len(vocab))
encoder_state_dict, decoder_state_dict, optimizer, *meta = utils.load_models(args.checkpoint_file,False)
encoder.load_state_dict(encoder_state_dict)
decoder.load_state_dict(decoder_state_dict)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
inp = cv2.imread(args.image_path)
inp = transform(Image.fromarray(inp)).unsqueeze(0)
inp = utils.to_var(inp, volatile=True)
features = encoder(inp)
sampled_ids = decoder.sample(features)
sampled_ids = sampled_ids.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(sampled_ids, vocab)
print('Caption:', sentence)
def __init__(self):
super(FaceEnvironementContinuous, self).__init__()
self.id = "FaceEnv"
self.random_seed = 666
self.action_space = spaces.Box(low=0, high=224, shape=(68, 2),
dtype=np.float32)
self.observation_space = spaces.Box(low=0, high=224, shape=(68, 2),
dtype=np.float32)
(self.embedder,
self.generator,
self.discriminator) = load_models(len(glob.glob(f"{ROOT_DATASET}/*")))
self.embedder = self.embedder.eval()
self.generator = self.generator.eval()
self.discriminator = self.discriminator.eval()
self.landmarks = None
self.landmarks_done = deque(maxlen=MAX_DEQUE_LANDMARKS)
self.contexts = None
self.user_ids = None
self.embeddings = None
self.paramWeights = None
self.paramBias = None
self.layersUp = None
self.iterations = 0
self.episodes = 0
self.max_iter = MAX_ITER_PERSON
self.fig, self.axes = plt.subplots(2, 2)
self.state = None
self._max_episode_steps = 1000
self.p = 0
def main():
with tf.Session(config=sess_config) as sess:
# build environment
_, env = build_env(config.env)
# load actor & critic
actor, critic = build_actor_critic(sess, env)
load_models(sess, config.model_path)
# evaluate final policy
test_actor(env,
actor,
eval_len=config.test_episode_len // 100,
verbose=True)
def __init__(self, path_weights):
super(FaceEnvironementDiscreete, self).__init__()
# 0 Nothing / 1 Left / 2 Right / 3 Up / 4 Down
self.id = "FaceEnv"
self.action_space = spaces.Discrete(68 * 4)
self.observation_space = spaces.Discrete(68 * 2)
self._max_episode_steps = MAX_ITER_PERSON
self.seed()
(self.embedder,
self.generator,
self.discriminator) = load_models(3000)
self.embedder = self.embedder.eval()
self.generator = self.generator.eval()
self.discriminator = self.discriminator.eval()
self.landmarks = None
self.landmarks_done = deque(maxlen=MAX_DEQUE_LANDMARKS)
self.contexts = None
self.user_ids = None
self.embeddings = None
self.paramWeights = None
self.paramBias = None
self.layersUp = None
self.iterations = 0
self.episodes = -1
self.state = None
self.fig, self.axes = plt.subplots(2, 2)
def load_pool_models(pool_name, max_pool_size, agent_config,
exclude_current_from_opponents,
max_pool_size_exclude_current_from_opponent,
best_iteration_opponents, record_videos_new_iteration):
# List all files in the agent pool, create a random agent if there are none
this_folder = os.path.dirname(__file__)
agents_folder = os.path.join(this_folder,
'../Deep Learning Agents/' + pool_name)
Path(agents_folder).mkdir(parents=True, exist_ok=True)
agent_paths = [f for f in os.listdir(agents_folder) if f[-3:] == '.h5']
if not agent_paths:
agent_paths = create_initial_random_agents(agents_folder, agent_config)
if record_videos_new_iteration:
utils.record_videos(os.path.join(agents_folder, agent_paths[0]),
agent_config['num_agents_per_game'],
agent_config['num_mirror_dim'])
# Order the agent models on their iteration ids
agent_paths = utils.decreasing_sort_on_iteration_id(agent_paths)
agent_paths = agent_paths[:max_pool_size]
# Load all agent models
agent_full_paths = [os.path.join(agents_folder, p) for p in agent_paths]
if best_iteration_opponents is None:
agents = utils.load_models(agent_full_paths)
this_agent = agents[0]
if exclude_current_from_opponents or len(
agents) <= max_pool_size_exclude_current_from_opponent:
agents = agents[1:]
else:
# Load the latest iteration of all opponent pools
this_agent = utils.load_models(agent_full_paths[:1])[0]
agents = []
for opponent_pool in best_iteration_opponents:
opponent_dir = os.path.join(
this_folder, '../Deep Learning Agents/' + opponent_pool)
opponent_paths = [
f for f in os.listdir(opponent_dir) if f[-3:] == '.h5'
]
opponent_paths = utils.decreasing_sort_on_iteration_id(
opponent_paths)
opponent_full_path = os.path.join(opponent_dir, opponent_paths[0])
agents.append(utils.load_models([opponent_full_path])[0])
return this_agent, agents, agent_full_paths
def segment_text(input_text):
with tempfile.TemporaryDirectory() as temp_dir:
input_dir = join(temp_dir, 'input')
output_dir = join(temp_dir, 'output')
for dirname in [input_dir, output_dir]:
os.makedirs(dirname)
with open(join(input_dir, 'input.txt'), 'w') as wf:
wf.write(input_text)
embeddings, vocabulary = utils.load_models()
segment.run_segmentation(input_dir, output_dir, embeddings, vocabulary)
segmented_text = open(join(output_dir, 'input.txt.seg')).readlines()
return segmented_text
def main():
"""Run NFQ."""
# Setup hyperparameters
parser = configargparse.ArgParser()
parser.add("-c", "--config", required=True, is_config_file=True)
parser.add("--EPOCH", type=int)
parser.add("--TRAIN_ENV_MAX_STEPS", type=int)
parser.add("--EVAL_ENV_MAX_STEPS", type=int)
parser.add("--DISCOUNT", type=float)
parser.add("--INIT_EXPERIENCE", type=int)
parser.add("--INCREMENT_EXPERIENCE", action="store_true")
parser.add("--HINT_TO_GOAL", action="store_true")
parser.add("--RANDOM_SEED", type=int)
parser.add("--TRAIN_RENDER", action="store_true")
parser.add("--EVAL_RENDER", action="store_true")
parser.add("--SAVE_PATH", type=str, default="")
parser.add("--LOAD_PATH", type=str, default="")
parser.add("--USE_TENSORBOARD", action="store_true")
parser.add("--USE_WANDB", action="store_true")
CONFIG = parser.parse_args()
if not hasattr(CONFIG, "INCREMENT_EXPERIENCE"):
CONFIG.INCREMENT_EXPERIENCE = False
if not hasattr(CONFIG, "HINT_TO_GOAL"):
CONFIG.HINT_TO_GOAL = False
if not hasattr(CONFIG, "TRAIN_RENDER"):
CONFIG.TRAIN_RENDER = False
if not hasattr(CONFIG, "EVAL_RENDER"):
CONFIG.EVAL_RENDER = False
if not hasattr(CONFIG, "USE_TENSORBOARD"):
CONFIG.USE_TENSORBOARD = False
if not hasattr(CONFIG, "USE_WANDB"):
CONFIG.USE_WANDB = False
print()
print(
"+--------------------------------+--------------------------------+")
print(
"| Hyperparameters | Value |")
print(
"+--------------------------------+--------------------------------+")
for arg in vars(CONFIG):
print("| {:30} | {:<30} |".format(
arg,
getattr(CONFIG, arg) if getattr(CONFIG, arg) is not None else ""))
print(
"+--------------------------------+--------------------------------+")
print()
# Log to File, Console, TensorBoard, W&B
logger = get_logger()
if CONFIG.USE_TENSORBOARD:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir="tensorboard_logs")
if CONFIG.USE_WANDB:
import wandb
wandb.init(project="implementations-nfq", config=CONFIG)
# Setup environment
train_env = CartPoleRegulatorEnv(mode="train")
eval_env = CartPoleRegulatorEnv(mode="eval")
# Fix random seeds
if CONFIG.RANDOM_SEED is not None:
make_reproducible(CONFIG.RANDOM_SEED, use_numpy=True, use_torch=True)
train_env.seed(CONFIG.RANDOM_SEED)
eval_env.seed(CONFIG.RANDOM_SEED)
else:
logger.warning(
"Running without a random seed: this run is NOT reproducible.")
# Setup agent
nfq_net = NFQNetwork()
optimizer = optim.Rprop(nfq_net.parameters())
nfq_agent = NFQAgent(nfq_net, optimizer)
# Load trained agent
if CONFIG.LOAD_PATH:
load_models(CONFIG.LOAD_PATH, nfq_net=nfq_net, optimizer=optimizer)
# NFQ Main loop
# A set of transition samples denoted as D
all_rollouts = []
total_cost = 0
if CONFIG.INIT_EXPERIENCE:
for _ in range(CONFIG.INIT_EXPERIENCE):
rollout, episode_cost = train_env.generate_rollout(
None, render=CONFIG.TRAIN_RENDER)
all_rollouts.extend(rollout)
total_cost += episode_cost
for epoch in range(CONFIG.EPOCH + 1):
# Variant 1: Incermentally add transitions (Section 3.4)
# TODO(seungjaeryanlee): Done before or after training?
if CONFIG.INCREMENT_EXPERIENCE:
new_rollout, episode_cost = train_env.generate_rollout(
nfq_agent.get_best_action, render=CONFIG.TRAIN_RENDER)
all_rollouts.extend(new_rollout)
total_cost += episode_cost
state_action_b, target_q_values = nfq_agent.generate_pattern_set(
all_rollouts)
# Variant 2: Clamp function to zero in goal region
# TODO(seungjaeryanlee): Since this is a regulator setting, should it
# not be clamped to zero?
if CONFIG.HINT_TO_GOAL:
goal_state_action_b, goal_target_q_values = train_env.get_goal_pattern_set(
)
goal_state_action_b = torch.FloatTensor(goal_state_action_b)
goal_target_q_values = torch.FloatTensor(goal_target_q_values)
state_action_b = torch.cat([state_action_b, goal_state_action_b],
dim=0)
target_q_values = torch.cat(
[target_q_values, goal_target_q_values], dim=0)
loss = nfq_agent.train((state_action_b, target_q_values))
# TODO(seungjaeryanlee): Evaluation should be done with 3000 episodes
eval_episode_length, eval_success, eval_episode_cost = nfq_agent.evaluate(
eval_env, CONFIG.EVAL_RENDER)
if CONFIG.INCREMENT_EXPERIENCE:
logger.info(
"Epoch {:4d} | Train {:3d} / {:4.2f} | Eval {:4d} / {:5.2f} | Train Loss {:.4f}"
.format( # noqa: B950
epoch,
len(new_rollout),
episode_cost,
eval_episode_length,
eval_episode_cost,
loss,
))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("train/episode_length", len(new_rollout),
epoch)
writer.add_scalar("train/episode_cost", episode_cost, epoch)
writer.add_scalar("train/loss", loss, epoch)
writer.add_scalar("eval/episode_length", eval_episode_length,
epoch)
writer.add_scalar("eval/episode_cost", eval_episode_cost,
epoch)
if CONFIG.USE_WANDB:
wandb.log({"Train Episode Length": len(new_rollout)},
step=epoch)
wandb.log({"Train Episode Cost": episode_cost}, step=epoch)
wandb.log({"Train Loss": loss}, step=epoch)
wandb.log({"Evaluation Episode Length": eval_episode_length},
step=epoch)
wandb.log({"Evaluation Episode Cost": eval_episode_cost},
step=epoch)
else:
logger.info(
"Epoch {:4d} | Eval {:4d} / {:5.2f} | Train Loss {:.4f}".
format(epoch, eval_episode_length, eval_episode_cost, loss))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("train/loss", loss, epoch)
writer.add_scalar("eval/episode_length", eval_episode_length,
epoch)
writer.add_scalar("eval/episode_cost", eval_episode_cost,
epoch)
if CONFIG.USE_WANDB:
wandb.log({"Train Loss": loss}, step=epoch)
wandb.log({"Evaluation Episode Length": eval_episode_length},
step=epoch)
wandb.log({"Evaluation Episode Cost": eval_episode_cost},
step=epoch)
if eval_success:
logger.info(
"Epoch {:4d} | Total Cycles {:6d} | Total Cost {:4.2f}".format(
epoch, len(all_rollouts), total_cost))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("summary/total_cycles", len(all_rollouts),
epoch)
writer.add_scalar("summary/total_cost", total_cost, epoch)
if CONFIG.USE_WANDB:
wandb.log({"Total Cycles": len(all_rollouts)}, step=epoch)
wandb.log({"Total Cost": total_cost}, step=epoch)
break
# Save trained agent
if CONFIG.SAVE_PATH:
save_models(CONFIG.SAVE_PATH, nfq_net=nfq_net, optimizer=optimizer)
train_env.close()
eval_env.close()
from collections import defaultdict
import os
from nltk.tag import pos_tag
from utils import get_related_forms, dump_json_set, load_json_set, clean_title
from utils import get_variants_and_derivatives, get_syn_thesaurus_net, get_syn_thesaurus_com
from utils import get_column_variants_per_pos, get_describe, get_related
from utils import get_common_words, load_models
from utils import get_ant_thesaurus_com, get_ant_antonymfor
from database import open_session, initialize_db
from database import ColumnD, ColumnI, ColumnJ, ColumnK, ColumnL, ColumnN
from tqdm import tqdm
MODELS = load_models()
def task_2():
if os.path.exists('output/task_2.json'):
print("Task 2 data found...")
return load_json_set('output/task_2.json')
else:
print("Loading data for Task 2...")
df = pd.read_excel('DATASET.xlsx', sheet_name='Task 2')
set1 = set(map(lambda x: x.strip(), df['Unnamed: 1'].dropna().values))
print("Getting related forms...")
nouns, adjs, advs, vrbs = get_related_forms(set1, should_ban=True)
print(len(nouns), len(adjs), len(advs), len(vrbs))
with Pool(5) as p:
f = p.map(get_related_forms, [nouns, adjs, advs, vrbs])
date = datetime.datetime.now().replace(microsecond=0)
train_id = "_".join(CONFIG.values())
wandb.init(project="SacadeDetect",
id=train_id,
name=train_id,
resume=LOAD_PREVIOUS,
config=CONFIG)
vectorized_persons, voc = load_data(ROOT_DATASET)
train_loader, valid_loader = get_data_loader(vectorized_persons,
workers=NUM_WORKERS)
check = Checkpoints()
(sacade_rnn, classifier,
autoencoder) = load_models(voc.num_user,
load_previous_state=LOAD_PREVIOUS,
load_classifier=False)
wandb.watch((sacade_rnn, classifier, autoencoder))
Cel = nn.CrossEntropyLoss()
optimizerGru = torch.optim.Adam(sacade_rnn.parameters(), lr=LEARNING_RATE)
optimizerClas = torch.optim.Adam(classifier.parameters(), lr=LEARNING_RATE)
optimizerAuto = torch.optim.Adam(autoencoder.parameters(), lr=LEARNING_RATE)
sacade_rnn = sacade_rnn.to(DEVICE)
classifier = classifier.to(DEVICE)
autoencoder = autoencoder.to(DEVICE)
Cel = Cel.to(DEVICE)
with SummaryWriter(comment=f'_{h.name}') as writer:
print(f'Run name: {h.name}')
writer.add_hparams(vars(h),{})
dataset = BasicDataset(model_dir=h.dataroot)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=h.batch_size, num_workers=8, shuffle=True)
print('Loading models')
gen = Generator(h.nz, h.nc, h.ngf, device)
discr = Discriminator(h.nc, h.ndf, device)
if h.load_name.lower() != 'none':
fname = load_models(os.path.join(h.modelroot,h.load_name),gen,discr,h.load_step)
print (f'Loaded model: {fname}')
gen_opt = optim.Adam(gen.parameters(), lr=h.lrG, betas=(h.beta1, h.beta2))
discr_opt = optim.Adam(discr.parameters(), lr=h.lrD, betas=(h.beta1, h.beta2))
# Create save dir
save_dir = os.path.join(h.modelroot,h.name)
if not os.path.exists(save_dir):
os.mkdir(save_dir)
print('Beginning training')
completed = train(
dataloader,
discr,
print("Stored.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Segmenting text documents from a given directory.")
parser.add_argument(
"input_dir",
type=str,
help="Path to the directory containing text documents to be segmented")
parser.add_argument(
"output_dir",
type=str,
help=
"Path to the directory where the serialized tfrecords will be saved.")
parser.add_argument(
"--scores",
type=int,
default=0,
help=
"Indicates whether to print segmentation prediction probabilities next to each sentence (default 0 = false)."
)
args = parser.parse_args()
embeddings, vocabulary = utils.load_models()
predict(args.input_dir,
args.output_dir,
embeddings,
vocabulary,
scores=False if args.scores == 0 else True)
kernel = 2
x1 = np.arange(0.1, 1, 0.1)
for i in tqdm(x1):
train_and_store(X_train, y_train, i, kernel)
x1 = np.arange(1, 31)
for i in tqdm(x1):
train_and_store(X_train, y_train, i, kernel)
error_random = []
error_last = []
error_avg = []
error_vote = []
kernel = 1
try:
for i in np.arange(0.1, 1, 0.1):
load_models(i, 1, 0)
load_models(i, 2, 0)
for i in np.arange(1, 11):
load_models(i, 1, 0)
load_models(i, 2, 0)
for i in np.arange(11, 31):
load_models(i, 2, 0)
except:
print("Error: are you sure you copied the files in the model dir?")
print("epoch: from 0.1 to 0.9 kernel:{}".format(kernel))
x1 = np.arange(0.1, 1, 0.1)
x2 = np.arange(1, 11)
for i in tqdm(x1):
e_r, e_l, e_a, e_v = load_and_test(X_train, X_test, y_test, i, kernel)
error_random.append(e_r)
utils.log_dirctory()
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir=PATH_LOG + "/log")
def init_weights(model):
for name, param in model.named_parameters():
nn.init.uniform_(param.data, -0.1, 0.1)
################################################
###
#model and dataset loading
if FINE_TUNING:
model, word2id, id2word = utils.load_models()
train_dataloader, test_dataloader, (word2id, id2word) = read_csv.table2dl(
w2i=word2id, i2w=id2word)
else:
train_dataloader, test_dataloader, (word2id, id2word) = read_csv.table2dl()
model = model_making.load_model(len(word2id))
model.apply(init_weights)
optimizer, scheduler = model_making.optimizer_scheduler(
model, len(train_dataloader))
criterion = nn.CrossEntropyLoss(ignore_index=0)
example, lengths = utils.str2id(word2id)
###
#train
history_kappa_valid = []
def main(args):
# hyperparameters
batch_size = args.batch_size
num_workers = 2
# Image Preprocessing
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
vocab = load_vocab()
loader = get_basic_loader(dir_path=os.path.join(args.image_path),
transform=transform,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers)
# Build the models
embed_size = args.embed_size
num_hiddens = args.num_hidden
checkpoint_path = 'checkpoints'
encoder = CNN(embed_size)
decoder = RNN(embed_size,
num_hiddens,
len(vocab),
1,
rec_unit=args.rec_unit)
encoder_state_dict, decoder_state_dict, optimizer, *meta = utils.load_models(
args.checkpoint_file)
encoder.load_state_dict(encoder_state_dict)
decoder.load_state_dict(decoder_state_dict)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Train the Models
try:
results = []
with torch.no_grad():
for step, (images, image_ids) in enumerate(tqdm(loader)):
images = utils.to_var(images)
features = encoder(images)
captions = beam_sample(decoder, features)
# captions = decoder.sample(features)
captions = captions.cpu().data.numpy()
captions = [
utils.convert_back_to_text(cap, vocab) for cap in captions
]
captions_formatted = [{
'image_id': int(img_id),
'caption': cap
} for img_id, cap in zip(image_ids, captions)]
results.extend(captions_formatted)
print('Sample:', captions_formatted[0])
except KeyboardInterrupt:
print('Ok bye!')
finally:
import json
file_name = 'captions_model.json'
with open(file_name, 'w') as f:
json.dump(results, f)
from pydantic import BaseModel
from os import path
from decouple import config as cfg #para variavel de ambiente
import argparse, joblib
from utils import load_models, check_inputs
from train import load_data2, transform
import numpy as np
from sklearn.model_selection import train_test_split
DIR_NAME = path.dirname(__file__)
DATA_FOLDER = path.join(DIR_NAME, 'models', 'exp_01_default')
DATA_CSV = cfg('DATA_CSV', cast=str)
#load models
model, tf_feature = load_models()
datas = pd.read_csv(path.join(DATA_FOLDER, DATA_CSV))
class Item(BaseModel):
#classe: Optional[str] = None
user: Optional[str] = 'None'
week: Optional[int] = 7
total_sessions: Optional[int] = 4
total_mediaids: Optional[int] = 7
total_days: Optional[int] = 3
total_played: Optional[float] = 130.450
max_played_time: Optional[float] = 58.280
age_without_access: Optional[int] = -285
sexo: Optional[str] = 'M'
idade: Optional[float] = 42
cidade: Optional[str] = 'None'
import argparse, joblib
from os import path
from train import load_data
from utils import load_models, check_inputs
from flask import Flask, request, jsonify
app = Flask(__name__)
# Load models
model, tf = load_models()
@app.route('/predict', methods=['POST'])
def predict():
if request.method == 'POST':
# Check inputs
x = check_inputs(request.json['features'])
y_hat = model.predict(tf.transform(x))
return jsonify(output={"y_hat": y_hat.tolist()},
status=200,
message='Model Working')
@app.route('/predict_test')
def predict_test():
X, y = load_data()
X_tf = tf.transform(X)
y_hat = model.predict(X_tf)
warnings.simplefilter(action = 'ignore')
#path_to_data = '/courses/cs342/Assignment2/'
path_to_data = ''
split_count = 3
columns = 64
start = time.time()
chunks = 5000000
test_meta = pd.read_csv(path_to_data+'test_set_metadata.csv')
g_wtable, eg_wtable = utils.get_wtables(path_to_data)
g_clfs = utils.load_models(split_count, True, g_wtable)
eg_clfs = utils.load_models(split_count, False, eg_wtable)
folds = StratifiedKFold(n_splits = split_count, shuffle = True)
straddler = None
for i_c, data_chunk in enumerate(pd.read_csv(path_to_data + 'test_set.csv', chunksize = chunks, iterator = True)):
if i_c != 0:
data_chunk = pd.concat([straddler, data_chunk], ignore_index=True)
arr = data_chunk['object_id'].unique()
straddler = data_chunk.loc[data_chunk['object_id'] == arr[len(arr)-1]]
data_chunk = data_chunk[data_chunk.object_id != arr[len(arr)-1]]
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if not os.path.isdir(args.log_dir):
os.makedirs(args.log_dir)
summary_writer = SummaryWriter(log_dir=args.log_dir) if args.log else None
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: (x > .5).float())
])),
batch_size=args.bsize,
shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: (x > .5).float())
])),
batch_size=args.bsize,
shuffle=True)
context_encoder = ContextEncoder()
context_to_dist = ContextToLatentDistribution()
decoder = Decoder()
if args.aggregator == "mean":
aggregator = MeanAgregator()
elif args.aggregator == "vector_attention":
aggregator = VectorAttentionAggregator(128)
else:
assert args.aggregator == "query_attention"
aggregator = QueryAttentionAggregator(128)
if args.resume_file is not None:
load_models(args.resume_file, context_encoder, context_to_dist,
decoder, aggregator)
context_encoder = context_encoder.to(device)
decoder = decoder.to(device)
context_to_dist = context_to_dist.to(device)
aggregator = aggregator.to(device)
full_model_params = list(context_encoder.parameters()) + list(
decoder.parameters()) + list(context_to_dist.parameters()) + list(
aggregator.parameters())
optimizer = optim.Adam(full_model_params, lr=args.lr)
train(context_encoder,
context_to_dist,
decoder,
aggregator,
train_loader,
test_loader,
optimizer,
args.epochs,
device,
args.models_path,
summary_writer=summary_writer,
save_every=args.save_every,
log=args.log)
def main(args):
# hyperparameters
batch_size = args.batch_size
num_workers = 1
# Image Preprocessing
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
# load COCOs dataset
IMAGES_PATH = 'data/train2014'
CAPTION_FILE_PATH = 'data/annotations/captions_train2014.json'
vocab = load_vocab()
train_loader = get_coco_data_loader(path=IMAGES_PATH,
json=CAPTION_FILE_PATH,
vocab=vocab,
transform=transform,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
IMAGES_PATH = 'data/val2014'
CAPTION_FILE_PATH = 'data/annotations/captions_val2014.json'
val_loader = get_coco_data_loader(path=IMAGES_PATH,
json=CAPTION_FILE_PATH,
vocab=vocab,
transform=transform,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
losses_val = []
losses_train = []
# Build the models
ngpu = 1
initial_step = initial_epoch = 0
embed_size = args.embed_size
num_hiddens = args.num_hidden
learning_rate = 5e-4
num_epochs = 2
log_step = args.log_step
save_step = 500
checkpoint_dir = args.checkpoint_dir
encoder = CNNEncoder()
decoder = DecoderRNN(embed_size, num_hiddens, len(vocab))
# Loss
criterion = nn.CrossEntropyLoss()
if args.checkpoint_file:
encoder_state_dict, decoder_state_dict, optimizer, *meta = utils.load_models(
args.checkpoint_file, args.sample)
initial_step, initial_epoch, losses_train, losses_val = meta
encoder.load_state_dict(encoder_state_dict)
decoder.load_state_dict(decoder_state_dict)
else:
params = list(decoder.parameters()) + list(
encoder.batchnorm.parameters())
optimizer = torch.optim.Adam(params, lr=learning_rate)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
if args.sample:
return utils.sample(encoder, decoder, vocab, val_loader)
# Train the Models
total_step = len(train_loader)
try:
for epoch in range(initial_epoch, num_epochs):
for step, (images, captions,
lengths) in enumerate(train_loader, start=initial_step):
# Set mini-batch dataset
images = utils.to_var(images, volatile=True)
captions = utils.to_var(captions)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
if ngpu > 1:
# run on multiple GPU
features = nn.parallel.data_parallel(
encoder, images, range(ngpu))
outputs, alphas = nn.parallel.data_parallel(
decoder, features, range(ngpu))
else:
# run on single GPU
features = encoder(images)
outputs, alphas = decoder(features, captions, lengths)
train_loss = criterion(outputs, targets.cpu())
train_loss += ((1. - alphas.sum(dim=1))**2).mean()
losses_train.append(train_loss.data)
train_loss.backward()
optimizer.step()
print('Epoch: {} - Step: {} - Train Loss: {}'.format(
epoch, step, losses_train[-1]))
# Run validation set and predict
if step % log_step == 404:
encoder.batchnorm.eval()
# run validation set
batch_loss_val = []
for val_step, (images, captions,
lengths) in enumerate(val_loader):
images = utils.to_var(images, volatile=True)
captions = utils.to_var(captions, volatile=True)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
features = encoder(images)
outputs, alphas = decoder(features, captions, lengths)
val_loss = criterion(outputs, targets.cpu())
val_loss += ((1. - alphas.sum(dim=1))**2).mean()
batch_loss_val.append(val_loss.data)
if val_step % 50 == 0:
print('Epoch: {} - Step: {} - Mini Eval Loss: {}'.
format(epoch, val_step, val_loss))
sampled_ids = decoder.sample(features)
sampled_ids = sampled_ids.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(
sampled_ids, vocab)
print('Sample:', sentence)
true_ids = captions.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(
true_ids, vocab)
print('Target:', sentence)
losses_val.append(np.mean(batch_loss_val))
# predict
print('Epoch: {} - Step: {} - Eval Loss: {}'.format(
epoch, step, losses_val[-1]))
encoder.batchnorm.train()
# Save the models
if (step + 1) % save_step == 0:
utils.save_models(encoder, decoder, optimizer, step, epoch,
losses_train, losses_val, checkpoint_dir)
utils.dump_losses(
losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
except KeyboardInterrupt:
pass
finally:
# Do final save
utils.save_models(encoder, decoder, optimizer, step, epoch,
losses_train, losses_val, checkpoint_dir)
utils.dump_losses(losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
label = type + str(mistake_factor)
#label = type + 'periodic'
ghost = LorenzModel(sigma=mistake_factor * 10)
#ghost = PeriodicApprox()
f = KnowledgeModel(hidden_dim=50, known_model=ghost, io_dim=data_dim)
model_dir = project_dir + '/FullStateNODE/models/M-NODE/' + str(
mistake_factor) + 'sigma/knowledge'
#model_dir = project_dir + '/FullStateNODE/models/M-NODE/periodic_model/knowledge'
elif type == 'pure':
label = "pure"
f = Net(hidden_dim=256, io_dim=data_dim)
model_dir = project_dir + '/FullStateNODE/models/PureNODE/3DLorenzmodel'
else:
print("type '{}' doesn't exist".format(type))
exit()
load_models(model_dir, f)
ic_state, ic_future = next(iter(dataloader))
ic_state = ic_state.view(batch_size, data_dim)
#vec = torch.tensor([[2, 10, 2]]).float()
t = torch.zeros(1)
dstatedt = f(t, ic_state)
dxdt_NODE = dstatedt[:, 0].detach().numpy()
if type == 'pure':
dxdt_NODE = dxdt_NODE / 10
dxdt = x_of_exact_lorenz(ic_state).detach().numpy()
comp = np.arange(-10, 10)
plt.plot(dxdt, dxdt_NODE, 'x', color='orange')
plt.xlabel(r'$(1-\alpha)\sigma(y-x)$')
def __init__(self, models_path):
self.models = utils.load_models(models_path)
LR = 0.01
HIDDEN_DIM = 500
num_rnn_layers = 2
# Construct model
encoder_rnn = CreationRNN(input_dim=1,
hidden_dim=HIDDEN_DIM,
num_layers=num_rnn_layers,
output_dim=latent_dim,
nbatch=batch_size)
print(encoder_rnn)
params = list(encoder_rnn.parameters())
optimizer = optim.Adam(params, lr=LR)
f = Net(hidden_dim=256)
load_models(ddd_model_dir, f)
if TRAIN_MODEL:
train_dataset = DDDLorenzData(dddtrain_data_dir,
lookahead=lookahead,
tau=tau,
k=8,
max_len=max_len)
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True)
val_dataset = DDDLorenzData(dddval_data_dir,
lookahead,
def main():
"""Run only when this file is called directly."""
# Setup hyperparameters
CONFIG = get_config()
# Log to File and Console
logger = get_logger(log_to_console=True, log_to_file=CONFIG.LOG_TO_FILE)
# Choose CPU and GPU
torch.set_num_threads(CONFIG.CPU_THREADS)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if not torch.cuda.is_available():
logger.warning("GPU not available: this run could be slow.")
# Setup environment
# v4 variant: No repeat action
env = gym.make(CONFIG.ENV_NAME)
eval_env = gym.make(CONFIG.ENV_NAME)
# AtariPreprocessing:
# - Max NOOP on start: 30
# - Frameskip: CONFIG.FRAME_SKIP
# - If Frameskip > 1, max pooling is done
# - Screen size: 84
# - Terminal on life loss: True
# - Grayscale obs: True
env = AtariPreprocessing(env,
frame_skip=CONFIG.FRAME_SKIP,
terminal_on_life_loss=True)
# NOTE(seungjaeryanlee): In evaluation, episode does not end in life loss
# https://github.com/deepmind/dqn/blob/9d9b1d1/dqn/train_agent.lua#L119
eval_env = AtariPreprocessing(eval_env,
frame_skip=CONFIG.FRAME_SKIP,
terminal_on_life_loss=False)
# Stack frames to create observation
env = FrameStack(env, stack_size=CONFIG.FRAME_STACK)
eval_env = FrameStack(eval_env, stack_size=CONFIG.FRAME_STACK)
# Fix random seeds
if CONFIG.RANDOM_SEED is not None:
set_global_random_seeds(CONFIG.RANDOM_SEED,
use_numpy=True,
use_torch=True)
# NOTE(seungjaeryanlee): Seed for env and eval_env are different for fair evaluation
set_env_random_seeds(env, CONFIG.RANDOM_SEED)
set_env_random_seeds(eval_env, CONFIG.RANDOM_SEED + 1)
else:
logger.warning(
"Running without a random seed: this run is NOT reproducible.")
# Setup agent and replay buffer
q_net = AtariQNetwork(CONFIG.FRAME_STACK, env.action_space.n).to(device)
optimizer = optim.RMSprop(
q_net.parameters(),
lr=CONFIG.RMSPROP_LR,
alpha=CONFIG.RMSPROP_DECAY,
eps=CONFIG.RMSPROP_EPSILON,
momentum=CONFIG.RMSPROP_MOMENTUM,
weight_decay=CONFIG.RMSPROP_WEIGHT_DECAY,
centered=CONFIG.RMSPROP_IS_CENTERED,
)
if CONFIG.LOAD_PATH:
# Load parameters if possible
load_models(CONFIG.LOAD_PATH, q_net=q_net, optimizer=optimizer)
dqn_agent = DQNAgent(env, q_net, optimizer, device)
replay_buffer = CircularReplayBuffer(
env,
maxlen=CONFIG.REPLAY_BUFFER_SIZE,
device=device,
preprocess_batch=NATUREDQN_ATARI_PREPROCESS_BATCH,
)
# Train and evaluate agent
train_eval(dqn_agent, replay_buffer, env, eval_env, device, logger, CONFIG)
raise
print("Start")
train_df = pd.read_pickle(args.data + '/' + Train_Data)
valid_df = pd.read_pickle(args.data + '/' + Valid_Data)
eval_df = pd.read_pickle(args.data + '/' + Eval_Data)
embed_df = pd.read_pickle(args.data + '/' + Embed_Data)
if (args.mode == 'train'):
print("Preparing to Train the model")
''' Load and vectorize data '''
if (args.load_existing):
print("Loading Existing !!! Loading Dictionary")
corpus = load_dictionary(args)
print("Loading Existing !!! Loading Models")
Encoder, Classifier = load_models(args, True)
if Encoder is None or Classifier is None:
print(
"Loading Existing !!! Models not loaded building from scratch"
)
Encoder, Classifier = build_model_from_scratch(
args, corpus, embed_df)
else:
if (args.build_dict):
print("Build Dictionary Given! Building dictionary now!")
corpus = build_dictionary_from_scratch(args, train_df,
valid_df, eval_df)
else:
print("Building dictionary now!")
corpus = load_dictionary(args)
print("Building models from scratch now!")
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 24 03:08:09 2019
@author: mustafa
"""
import numpy as np
import torch
from utils import get_obs, load_models
#Load model
net = load_models("py_src/models/REINGINE-checkpoint.dat")
a = 10
action = None
def step(state_dic):
"""
Parameters
----------
state_dic : dictionary
current state of agent coming from environment
Returns
-------
Action: angle
"""
global net, a, action
time.sleep(0.025)
print('No hand detected on this window')
# Draw the window
# clone = resized.copy()
# cv2.rectangle(clone, (x, y), (x + winW, y + winH), (0, 255, 0), 2)
# cv2.imshow('Window', clone)
# plt.imshow(clone)
# filename = 'test.png'
# cv2.imwrite(filename, clone)
# cv2.waitKey(1)
time.sleep(0.025)
return None
if __name__ == '__main__':
modes = load_models()
data_dir = 'data/sample/'
for filename in [f for f in os.listdir(data_dir) if '.jpg' in f]:
print('Analysing: %s' % (filename))
filename = os.path.join(data_dir, filename)
image = cv2.imread(filename)
print('Recogize:')
recognize(image, models)
print('Recognition done!')
time.sleep(2)
break
'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain',
'mouse', 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree',
'pear', 'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy',
'porcupine', 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket',
'rose', 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail',
'snake', 'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper',
'table', 'tank', 'telephone', 'television', 'tiger', 'tractor', 'train',
'trout', 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf',
'woman', 'worm'
] #for CIFAR100, fine names
# model = models.vgg19(pretrained=True)
files = list(
filter(lambda filename: filename.endswith('.npy'), os.listdir(args.path)))
model, loader, preprocess_image = load_models()
for _class in classes[:1]:
filtered_files = filter(lambda x: _class in x, files)
for inp, outp in (('conv5_3',
'conv5_4'), ): #('fc1', 'fc2'),('fc2', 'output')):
inmask = torch.from_numpy(
np.load(
os.path.join(args.path,
'class-{}-{}.npy'.format(_class,
inp))).mean(axis=0))
outmask = torch.from_numpy(
np.load(
os.path.join(args.path,
'class-{}-{}.npy'.format(_class,
outp))).mean(axis=0))
"input_dir",
type=str,
help="Path to the directory containing text documents to be segmented")
parser.add_argument(
"output_dir",
type=str,
help=
"Path to the directory where the serialized tfrecords will be saved.")
parser.add_argument(
"--train",
type=int,
default=0,
help=
"Indicates if you're preparing tfrecords for training (value 1) the model or instances on which to predict (value 0) the segmentation scores."
)
parser.add_argument(
"--ssplit",
type=int,
default=1,
help=
"Indicates whether the texts are already sentence split, one sentence per line in the text files (value 1), or the texts need to be first split for sentences (value 0)"
)
args = parser.parse_args()
_, vocabulary = utils.load_models()
preprocess(args.input_dir,
args.output_dir,
vocabulary,
train=True if args.train == 1 else False,
ssplit=True if args.ssplit == 1 else False)
