def __init__(self, model_name, dataset_name, losswise_api_key, tensorboard=False):
"""
:param model_name: ``str``, model name
:param dataset_name: ``str``, dataset name
:param losswise_api_key: ``str``, losswise API key
:param tensorboard: if True uses tensorboard
"""
self.model_name = model_name
self.dataset_name = dataset_name
self.ls_api_key = losswise_api_key
self.tensorboard = tensorboard
self.data_subdir = "{}/{}".format(model_name, dataset_name)
self.comment = "{}/{}".format(model_name, dataset_name)
self.step = []
self.loss_g = []
self.loss_d = []
self.acc_real = []
self.acc_fake = []
if self.ls_api_key:
logger.info('Init losswise session')
losswise.set_api_key(self.ls_api_key)
self.session = losswise.Session(
tag=model_name,
max_iter=cfg.NUM_EPOCHS,
track_git=False
)
self.graph_loss = self.session.graph('loss', kind='min')
self.graph_acc = self.session.graph('accuracy', kind='max')
if self.tensorboard:
self.tf_logger = SummaryWriter(comment=self.comment)
def __init__(self,
model_name,
dataset_name,
ls_api_key,
tensorboard=False):
"""
:param model_name: ``str``, model name
:param dataset_name: ``str``, dataset name
:param ls_api_key: ``str``, losswise API key
:param tensorboard: if True - save tensorboard metrics
"""
self.model_name = model_name
self.dataset_name = dataset_name
self.data_subdir = '{}/{}'.format(model_name, dataset_name)
self.comment = '{}_{}'.format(self.model_name, self.dataset_name)
self.ls_api_key = ls_api_key
self.tensorboard = tensorboard
if self.ls_api_key:
losswise.set_api_key(ls_api_key)
self.session = losswise.Session(tag='egan',
max_iter=cfg.NUM_EPOCH,
track_git=False)
self.graph_loss = self.session.graph('loss', kind='min')
self.graph_acc = self.session.graph('accuracy', kind='max')
if self.tensorboard:
self.metric_logger = SummaryWriter(comment=self.comment)
def __init__(self,
api_key,
tag,
params={},
min_graphs={
'loss': {
'training_loss': 'Training Loss',
'test_loss': 'Test Loss',
},
'lr': {
'learning_rate': 'Learning Rate',
}
},
max_graphs={},
name='losswise_monitor'):
super().__init__(name=name)
losswise.set_api_key(api_key)
self._session = losswise.Session(tag=tag,
params=params,
track_git=False)
self._min_graphs = min_graphs
self._max_graphs = max_graphs
self._graphs = {}
for tracker in min_graphs:
self._graphs[tracker] = self._session.graph(tracker, kind='min')
for tracker in max_graphs:
self._graphs[tracker] = self._session.graph(tracker, kind='max')
self.previous_session_steps = 0
self.last_step = 0
def __init__(self, api_key, tag="", params=None):
import losswise
losswise.set_api_key(api_key)
self._session = losswise.Session(
tag=tag,
params=params if params is not None else {},
track_git=False)
self._graphs = dict()
def __init__(self, key=None, tag=None, epochs=300):
self.key = key
print(self.key)
if len(self.key)>0:
losswise.set_api_key(self.key)
session = losswise.Session(tag=tag, max_iter=epochs)
self.error = session.graph('Error', kind='min', display_interval=1)
self.loss_total = session.graph('Loss', kind='min', display_interval=1)
self.delta = session.graph('Delta', kind='min', display_interval=1)
self.session=session
def __init__(self, patterns=None, ignore_patterns=None, ignore_directories=False, case_sensitive=False):
super().__init__(patterns, ignore_patterns, ignore_directories, case_sensitive)
losswise.set_api_key('W5BWWF7RP') # api_key for 'NC'
# tracking stuff
session = losswise.Session(tag='test_set_performance')
self.graph = session.graph('reward', kind='max')
self.it_counter = 0
max_episode_steps = 4500
register_all(max_episode_steps=max_episode_steps)
def initialize_settings(args):
cc = ConfigurationContainer.instance()
cc.settings = read_settings(args.configuration_file)
if 'logging' in cc.settings['general'] and cc.settings['general']['logging']['enabled']:
log_dir = os.path.join(cc.settings['general']['output_dir'], 'log')
LogHelper.setup(cc.settings['general']['logging']['log_level'], log_dir)
if cc.is_losswise_enabled:
losswise.set_api_key(cc.settings['general']['losswise']['api_key'])
return cc
def __init__(self,
api_key,
metric_name,
model_name=None,
model_params=None):
super().__init__()
self.metric_name = metric_name
losswise.set_api_key(api_key)
self._session = losswise.Session(tag=model_name, params=model_params)
self._graph = self._session.graph('loss', kind='min')
def losswise(self, keyfile, model_to_json, epochs, steps_per_epoch):
keys = json.load(open(keyfile))
api_key = keys["losswise_api_key"]
tag = keys["losswise_tag"]
losswise.set_api_key(api_key)
params = json.loads(model_to_json)
params['steps_per_epoch'] = steps_per_epoch
params['epochs'] = epochs
self.losswise_callback = LosswiseKerasCallback(tag=tag,
params=params
)
from keras.models import load_model
import numpy as np
import tensorflow as tf
import keras.backend as K
from keras.callbacks import EarlyStopping, ModelCheckpoint
import losswise
from losswise.libs import LosswiseKerasCallback
import tables
from config import test_set_portion, dataset_name, lr_init, first_layer_filters, batch_size, use_cpu, training_epochs, kernel_size, use_sample_weights, sample_weight_lims, network_structure
from datetime import datetime
import os
import pickle
from glob import glob
from tqdm import tqdm
losswise.set_api_key('9BDAXRBWA') # set up losswise.com visualization
# prepare sample weights
if use_sample_weights:
angles = np.genfromtxt(os.path.join('data', 'raw', 'frame_angles.csv'))[1:,
3]
bin_counts, bin_edges = np.histogram(angles, bins=10)
frame_bins = np.digitize(angles, bin_edges[:-1])
sample_weights = (1 / bin_counts) / np.mean(
1 / bin_counts) # weights are inverse of bins
sample_weights = np.clip(
sample_weights, sample_weight_lims[0],
sample_weight_lims[1]) # make sure the weights aren't too big or small
sample_weights = sample_weights[frame_bins - 1] # give each frame a weight
from keras.models import load_model
import tensorflow as tf
import keras.backend as K
from keras.callbacks import ModelCheckpoint, EarlyStopping, LambdaCallback
import losswise
from losswise.libs import LosswiseKerasCallback
import config as cfg
from datetime import datetime
import os
import pickle
from glob import glob
import models
import utils
if cfg.losswise_api_key:
losswise.set_api_key(
cfg.losswise_api_key) # set up losswise.com visualization
# create model data generators
train_generator = DataGenerator(cfg.train_datasets,
batch_size=cfg.batch_size,
subframe_size=cfg.subframe_size,
normalize_subframes=cfg.normalize_subframes,
epoch_size=cfg.epoch_size // cfg.batch_size,
rotation=cfg.aug_rotation,
scaling=cfg.aug_scaling)
test_generator = DataGenerator(cfg.test_datasets,
batch_size=cfg.batch_size,
subframe_size=cfg.subframe_size,
normalize_subframes=cfg.normalize_subframes,
epoch_size=cfg.epoch_size // cfg.batch_size,
rotation=cfg.aug_rotation,
c_format = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
f_format = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format)
logger.addHandler(c_handler)
logger.addHandler(f_handler)
logger.setLevel(logging.INFO)
logger.info('XrayPneumoniaCNN starts training {}'.format(time.ctime()))
logger.info('Called with args: {}'.format(args.__dict__))
logger.info('Config params:{}'.format(cfg.__dict__))
losswise.set_api_key(args.api_key)
session = losswise.Session(tag='x-ray-test',
max_iter=cfg.NUM_EPOCHS,
track_git=False)
if torch.cuda.is_available() and not args.use_gpu:
logger.info(
'You have a GPU device so you should probably run with --use_gpu')
device = torch.device('cpu')
elif torch.cuda.is_available() and args.use_gpu:
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
else:
device = torch.device('cpu')
logger.info('Running with device %s', device)
logger.info('Creates datasets')
#Creating 32 batches of hxw 64x64 pixels
test_set = test_datagen.flow_from_directory('dataset/test_set',
target_size=(100, 100),
batch_size=32,
class_mode='binary')
"""
Tensorboard uses scalars, while Keras uses validation_data for evaluation.
Tensorboard; in terminal:
tensorboard --logdir==training:model_dir --host=127.0.0.1
tensorboard --host 0.0.0.0 --logdir=/dataset/logs
"""
###Visualise model progression
import losswise
from losswise.libs import LosswiseKerasCallback
losswise.set_api_key('your_api_here') #www.losswise.com
#Run session
session = losswise.Session(tag='Convolutional Classifier',
max_iter=4000,
params={
'cnn_size': 256,
'dropout': 0.25
},
track_git=True)
#Graph of loss & accuracy
graph_loss = session.graph('loss', kind='min', display_interval=1)
graph_accuracy = session.graph('accuracy', kind='max', display_interval=50)
#Running the CNN on the training set
classifier.fit_generator(
training_set,
import losswise
import time
import random
losswise.set_api_key("B4N1RQR1K")
session = losswise.Session(tag='my_special_lstm',
params={'rnn_size': 512},
max_iter=10)
graph = session.graph('loss', kind='min')
for x in range(10):
train_loss = 1. / (0.1 + x + 0.1 * random.random())
test_loss = 1.5 / (0.1 + x + 0.2 * random.random())
graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss})
time.sleep(1.)
session.done()
def run_master(master_redis_cfg, log_dir, exp):
losswise.set_api_key('W5BWWF7RP') # api_key for 'NC'
# tracking stuff
session = losswise.Session(tag='online_performance')
graph = session.graph('reward', kind='max')
logger.info('run_master: {}'.format(locals()))
from . import tabular_logger as tlogger
logger.info('Tabular logging to {}'.format(log_dir))
tlogger.start(log_dir)
config, env, sess, policy = setup(exp, single_threaded=False)
master = MasterClient(master_redis_cfg)
noise = SharedNoiseTable()
rs = np.random.RandomState()
if isinstance(config.episode_cutoff_mode, int):
tslimit, incr_tslimit_threshold, tslimit_incr_ratio = config.episode_cutoff_mode, None, None
adaptive_tslimit = False
elif config.episode_cutoff_mode.startswith('adaptive:'):
_, args = config.episode_cutoff_mode.split(':')
arg0, arg1, arg2 = args.split(',')
tslimit, incr_tslimit_threshold, tslimit_incr_ratio = int(arg0), float(
arg1), float(arg2)
adaptive_tslimit = True
logger.info(
'Starting timestep limit set to {}. When {}% of rollouts hit the limit, it will be increased by {}'
.format(tslimit, incr_tslimit_threshold * 100, tslimit_incr_ratio))
elif config.episode_cutoff_mode == 'env_default':
tslimit, incr_tslimit_threshold, tslimit_incr_ratio = None, None, None
adaptive_tslimit = False
else:
raise NotImplementedError(config.episode_cutoff_mode)
episodes_so_far = 0
timesteps_so_far = 0
tstart = time.time()
master.declare_experiment(exp)
best_score = float('-inf')
population = []
population_size = exp['population_size']
num_elites = exp['num_elites']
population_score = np.array([])
time_to_stop = datetime.now() + timedelta(hours=24)
# keep doling out tasks till you're out of time
while datetime.now() < time_to_stop:
step_tstart = time.time()
theta = policy.get_trainable_flat()
assert theta.dtype == np.float32
if policy.needs_ob_stat:
ob_stat = RunningStat(
env.observation_space.shape,
eps=
1e-2 # eps to prevent dividing by zero at the beginning when computing mean/stdev
)
curr_task_id = master.declare_task(
GATask(params=theta,
population=population,
ob_mean=ob_stat.mean if policy.needs_ob_stat else None,
ob_std=ob_stat.std if policy.needs_ob_stat else None,
timestep_limit=tslimit))
tlogger.log('********** Iteration {} **********'.format(curr_task_id))
# Pop off results for the current task
curr_task_results, eval_rets, eval_lens, worker_ids = [], [], [], []
num_results_skipped, num_episodes_popped, num_timesteps_popped, ob_count_this_batch = 0, 0, 0, 0
# break out of the inner loop as well if time's up
while datetime.now() < time_to_stop and (
num_episodes_popped < config.episodes_per_batch
or num_timesteps_popped < config.timesteps_per_batch):
# Wait for a result
task_id, result = master.pop_result()
assert isinstance(task_id, int) and isinstance(result, Result)
assert (result.eval_return is None) == (result.eval_length is None)
worker_ids.append(result.worker_id)
if result.eval_length is not None:
# This was an eval job
episodes_so_far += 1
timesteps_so_far += result.eval_length
# Store the result only for current tasks
if task_id == curr_task_id:
eval_rets.append(result.eval_return)
eval_lens.append(result.eval_length)
else:
assert result.returns_n2.dtype == np.float32
# Store results only for current tasks
if task_id == curr_task_id:
# Update counts
result_num_eps = result.lengths_n2.size
result_num_timesteps = result.lengths_n2.sum()
episodes_so_far += result_num_eps
timesteps_so_far += result_num_timesteps
curr_task_results.append(result)
num_episodes_popped += result_num_eps
num_timesteps_popped += result_num_timesteps
# Update ob stats
if policy.needs_ob_stat and result.ob_count > 0:
ob_stat.increment(result.ob_sum, result.ob_sumsq,
result.ob_count)
ob_count_this_batch += result.ob_count
else:
num_results_skipped += 1
# compute the results only if the time hasn't been exceeded
if datetime.now() < time_to_stop:
# Compute skip fraction
frac_results_skipped = num_results_skipped / (
num_results_skipped + len(curr_task_results))
if num_results_skipped > 0:
logger.warning(
'Skipped {} out of date results ({:.2f}%)'.format(
num_results_skipped, 100. * frac_results_skipped))
# Assemble results + elite
noise_inds_n = list(population[:num_elites])
returns_n2 = list(population_score[:num_elites])
for r in curr_task_results:
noise_inds_n.extend(r.noise_inds_n)
returns_n2.extend(r.returns_n2)
noise_inds_n = np.array(noise_inds_n)
returns_n2 = np.array(returns_n2)
lengths_n2 = np.array([r.lengths_n2 for r in curr_task_results])
# Process returns
idx = np.argpartition(
returns_n2, (-population_size, -1))[-1:-population_size - 1:-1]
population = noise_inds_n[idx]
population_score = returns_n2[idx]
assert len(population) == population_size
assert np.max(returns_n2) == population_score[0]
print('Elite: {} score: {}'.format(population[0],
population_score[0]))
policy.set_trainable_flat(
noise.get(population[0][0], policy.num_params))
policy.reinitialize()
v = policy.get_trainable_flat()
for seed in population[0][1:]:
v += config.noise_stdev * noise.get(seed, policy.num_params)
policy.set_trainable_flat(v)
# Update number of steps to take
if adaptive_tslimit and (
lengths_n2 == tslimit).mean() >= incr_tslimit_threshold:
old_tslimit = tslimit
tslimit = int(tslimit_incr_ratio * tslimit)
logger.info('Increased timestep limit from {} to {}'.format(
old_tslimit, tslimit))
step_tend = time.time()
tlogger.record_tabular("EpRewMax", returns_n2.max())
tlogger.record_tabular("EpRewMean", returns_n2.mean())
tlogger.record_tabular("EpRewStd", returns_n2.std())
tlogger.record_tabular("EpLenMean", lengths_n2.mean())
graph.append(curr_task_id, {
'meanReward': returns_n2.mean(),
'maxReward': returns_n2.max()
})
tlogger.record_tabular(
"EvalEpRewMean",
np.nan if not eval_rets else np.mean(eval_rets))
tlogger.record_tabular(
"EvalEpRewMedian",
np.nan if not eval_rets else np.median(eval_rets))
tlogger.record_tabular(
"EvalEpRewStd", np.nan if not eval_rets else np.std(eval_rets))
tlogger.record_tabular(
"EvalEpLenMean",
np.nan if not eval_rets else np.mean(eval_lens))
tlogger.record_tabular(
"EvalPopRank", np.nan if not eval_rets else
(np.searchsorted(np.sort(returns_n2.ravel()),
eval_rets).mean() / returns_n2.size))
tlogger.record_tabular("EvalEpCount", len(eval_rets))
tlogger.record_tabular(
"Norm", float(np.square(policy.get_trainable_flat()).sum()))
tlogger.record_tabular("EpisodesThisIter", lengths_n2.size)
tlogger.record_tabular("EpisodesSoFar", episodes_so_far)
tlogger.record_tabular("TimestepsThisIter", lengths_n2.sum())
tlogger.record_tabular("TimestepsSoFar", timesteps_so_far)
num_unique_workers = len(set(worker_ids))
tlogger.record_tabular("UniqueWorkers", num_unique_workers)
tlogger.record_tabular("UniqueWorkersFrac",
num_unique_workers / len(worker_ids))
tlogger.record_tabular("ResultsSkippedFrac", frac_results_skipped)
tlogger.record_tabular("ObCount", ob_count_this_batch)
tlogger.record_tabular("TimeElapsedThisIter",
step_tend - step_tstart)
tlogger.record_tabular("TimeElapsed", step_tend - tstart)
tlogger.dump_tabular()
# if config.snapshot_freq != 0 and curr_task_id % config.snapshot_freq == 0:
if config.snapshot_freq != 0:
import os.path as osp
filename = 'snapshot_iter{:05d}_rew{}.h5'.format(
curr_task_id,
np.nan if not eval_rets else int(np.mean(eval_rets)))
assert not osp.exists(filename)
policy.save(filename)
tlogger.log('Saved snapshot {}'.format(filename))
# thread = Thread(target=evaluate_policy_on_test, args=(policy,curr_task_id,graph))
# thread.start()
# hammer out this empty task to stop the workers
curr_task_id = master.declare_task(EmptyTask())
session.done()
import os
from tqdm import tqdm
from dumbrain.lib.download import mkdirp
import GPUtil as GPU
import numpy as np
import sys
import tensorflow as tf
import losswise
losswise.set_api_key('HMR4DB5IE')
class TFSchedule():
time = 0
def add_time(self, sess, additional_time):
self.time += additional_time
self.tick(sess, self.time)
def tick(self, sess, time):
pass
class ScheduledSaver(TFSchedule):
def __init__(self, save_interval=1000, save_dir='data/model/'):
import tensorflow as tf
self.saver = tf.train.Saver()
self.save_interval = save_interval
import time
import random
import losswise
import numpy as np
from PIL import Image
losswise.set_api_key("IE11WZOSM")
max_iter = 200
session = losswise.Session(max_iter=max_iter,
params={
'max_iter': max_iter,
'dropout': 0.3,
'lr': 0.01,
'rnn_sizes': [256, 512]
})
graph = session.graph('loss', kind='min')
for x in range(max_iter):
train_loss = 1. / (0.1 + x + 0.1 * random.random())
test_loss = 1.5 / (0.1 + x + 0.2 * random.random())
graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss})
time.sleep(0.5)
if x % 5 == 0:
seq = session.image_sequence(x=x, name="Test")
for img_id in range(5):
pil_image = Image.open("./image.png")
seq.append(pil_image,
metrics={'accuracy': 1},
image_id=str(img_id) + "_img")
session.done()
#
from losswise.libs import LosswiseKerasCallback
from datetime import datetime
import os
import pickle
from glob import glob
import yaml
import shutil
# load configurations
with open('config.yaml', 'r') as f:
cfg_global = yaml.safe_load(f)
with open(os.path.join('region_proposal', 'config.yaml'), 'r') as f:
cfg = yaml.safe_load(f)
if cfg['losswise_api_key']:
losswise.set_api_key(
cfg['losswise_api_key']) # set up losswise.com visualization
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
# create model data generators
train_generator = dg.DataGenerator(cfg['train_datasets'],
batch_size=cfg['batch_size'],
subframe_size=cfg['subframe_size'],
epoch_size=cfg['epoch_size'],
rotation=cfg['aug_rotation'],
scaling=cfg['aug_scaling'])
test_generator = dg.DataGenerator(cfg['test_datasets'],
batch_size=cfg['batch_size'],
subframe_size=cfg['subframe_size'],
epoch_size=cfg['epoch_size'],
rotation=cfg['aug_rotation'],
scaling=cfg['aug_scaling'])
import random
import losswise
import time
losswise.set_api_key('DJOOF0FUN')
session = losswise.Session(tag='my_dilated_convnet',
max_iter=10,
params={'cnn_size': 20})
graph = session.graph('loss', kind='min')
session.set_values({'key5': 'eee1'})
for x in range(10):
train_loss = 1. / (0.1 + x + 0.1 * random.random())
test_loss = 1.5 / (0.1 + x + 0.2 * random.random())
graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss})
time.sleep(1.)
session.done()
import random
import losswise
import time
losswise.set_api_key('WY92FQJ03') # api_key for 'Curadiv blog'
session = losswise.Session(max_iter=10)
graph = session.graph('loss', kind='min')
for x in range(10):
train_loss = 2. / (0.1 + x + 0.1 * random.random())
test_loss = 1.5 / (0.1 + x + 0.2 * random.random())
graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss})
time.sleep(1.)
session.done()
# Argument validation
if args['timesteps'] == 0:
raise ValueError("Timesteps cannot be 0")
if args['batch_size'] == 0:
raise ValueError("Batch size cannot be 0")
if args['epochs'] == 0:
raise ValueError("Epoch count cannot be 0")
if args['std'] == 0:
raise ValueError("STD cannot be 0")
if args['workers'] == 0:
raise ValueError("Cannot have 0 workers")
# Losswise compatibility
if args['losswise_api']:
losswise.set_api_key(args['losswise_api'])
# Load whisker images
print('[INFO] Loading whisker images...')
test_gen = WhiskerGenerator(os.path.join(args['data_dir'], 'test'),
args['timesteps'],
args['batch_size'],
mean=args['mean'],
std=args['std'])
train_gen = WhiskerGenerator(os.path.join(args['data_dir'], 'train'),
args['timesteps'],
args['batch_size'],
width_shift_range=args['width_shift'],
height_shift_range=args['height_shift'],
zoom_range=args['zoom_range'],
rotation_range=args['rotation_range'],
patience=30,
verbose=1,
mode='min')
tboard = TensorBoard(log_dir=basedir,
histogram_freq=1,
batch_size=self.bsize,
write_graph=True,
write_grads=True,
write_images=True)
losswise_ = LosswiseKerasCallback(tag=lw_sess_tag, display_interval=1)
history = History()
if eval_cb == 'tb':
callbacks = [modelcheckpoint, lrop, earlystopping, history, tboard]
elif eval_cb == 'lw':
callbacks = [
modelcheckpoint, lrop, earlystopping, history, losswise_
]
elif eval_cb is None:
callbacks = [modelcheckpoint, lrop, earlystopping, history]
return callbacks
if __name__ == '__main__':
import losswise
losswise.set_api_key("5FPT5QCHE")
import random
import losswise
import time
losswise.set_api_key('ZEGNIL0YY') # api_key for 'Test drive'
session = losswise.Session(tag='my_dilated_convnet',
max_iter=10,
params={'cnn_size': 20})
graph = session.graph('loss', kind='min')
for x in range(10):
train_loss = 2. / (0.1 + x + 0.1 * random.random())
test_loss = 1.5 / (0.1 + x + 0.2 * random.random())
graph.append(x, {'train_loss': train_loss, 'test_loss': test_loss})
time.sleep(1.)
session.done()
def train(params):
num_epochs = params["epochs"] # into json file
train_batch_size = params["train_batchsize"]
train_workers = 8
test_batch_size = 8
test_workers = 8
num_classes = 197
data_dir = params["data_dir"]
checkpoint_dir = "./checkpoint"
save_freq = params["save_freq"]
enable_cuda = params["enable_cuda"]
learning_rate = params["learning_rate"]
lr_updates = params["lr_updates"]
lr_gamma = params["lr_gamma"]
weight_decay = params["weight_decay"]
cars_data = CarsDataset(os.path.join(data_dir,
'devkit/cars_train_annos.mat'),
os.path.join(data_dir, 'cars_train'),
os.path.join(data_dir, 'devkit/cars_meta.mat'),
cleaned=os.path.join(data_dir, 'cleaned.dat'),
transform=transforms.Compose([
transforms.Scale(250),
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(
(0.4706145, 0.46000465, 0.45479808),
(0.26668432, 0.26578658, 0.2706199))
]))
cars_data_val = CarsDataset(
os.path.join(data_dir, 'devkit/cars_test_annos_withlabels.mat'),
os.path.join(data_dir, 'cars_test'),
os.path.join(data_dir, 'devkit/cars_meta.mat'),
cleaned=os.path.join(data_dir, 'cleaned_test.dat'),
transform=transforms.Compose([
transforms.Scale(224),
transforms.RandomSizedCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.46905602, 0.45872932, 0.4539325),
(0.26603131, 0.26460057, 0.26935185))
]))
dataloader = DataLoader(cars_data,
batch_size=train_batch_size,
shuffle=True,
num_workers=train_workers)
print("Train data set length:", len(cars_data))
valloader = DataLoader(cars_data_val,
batch_size=test_batch_size,
shuffle=True,
num_workers=test_workers)
print("Validation data set length:", len(cars_data_val))
losswise.set_api_key(params["key"])
# for i, batch in enumerate(dataloader):
# plt.figure(figsize=(15,5))
# img, classes = batch
# cars_data.show_batch(img.numpy(), classes.numpy())
#
# plt.show()
# plt.pause(4)
# plt.clf()
session = losswise.Session(tag='Densnet161_cars', max_iter=num_epochs)
graph_tloss = session.graph('loss', kind='min')
graph_acc = session.graph('accuracy', kind='max')
model_ft = models.resnet18(pretrained=True)
# for param in model_ft.parameters():
# param.requires_grad = False
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
model_val = models.resnet18(pretrained=True)
# for param in model_ft.parameters():
# param.requires_grad = False
num_val = model_val.fc.in_features
model_val.fc = nn.Linear(num_val, num_classes)
# model_ft = Densenet161(drop_rate=0.5)
# model_val = Densenet161()
if enable_cuda:
model_ft = model_ft.cuda()
model_val = model_val.cuda()
criterion = nn.CrossEntropyLoss().cuda()
else:
criterion = nn.CrossEntropyLoss()
# optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4)
optimizer_ft = optim.Adam(model_ft.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
step_size=lr_updates,
gamma=lr_gamma)
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
exp_lr_scheduler.step()
model_ft.train(True) # Set model to training mode
running_loss = 0.0
c = 0
correct = 0
total = 0
for batch in tqdm(dataloader):
c += 1
inputs, labels = batch
labels = labels.type(torch.LongTensor)
if enable_cuda:
inputs, labels = Variable(inputs).cuda(), Variable(
labels).cuda()
else:
inputs, labels = Variable(inputs), Variable(labels)
optimizer_ft.zero_grad()
outputs = model_ft(inputs)
_, predicted = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer_ft.step()
running_loss += loss.data[0]
total += labels.size(0)
correct += (predicted == labels.data).sum()
epoch_loss = running_loss / c
train_acc = 100 * correct / total
print('Train epoch: {} || Loss: {:.4f} || Acc: {:.2f} %%'.format(
epoch, epoch_loss, train_acc))
graph_tloss.append(epoch, {'train_loss': running_loss / c})
graph_acc.append(epoch, {'train_acc': train_acc})
if epoch % save_freq == 0:
save_model(epoch, model_ft, optimizer_ft,
os.path.join(checkpoint_dir, 'model_%03d.pth' % epoch))
model_val.train(False)
model_val.load_state_dict(model_ft.state_dict())
correct = 0
total = 0
for data in tqdm(valloader):
images, labels = data
if enable_cuda:
labels = labels.type(torch.LongTensor).cuda()
images = Variable(images).cuda()
else:
images = Variable(images)
outputs = model_val(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
val_acc = 100 * correct / total
print('Validation Acc: {:.2f} %%'.format(val_acc))
graph_acc.append(epoch, {'val_acc': val_acc})
session.done()
seed_val = int(parameters["general"]["seed"])
else:
seed_val = random.randint(1, 10000)
random.seed(seed_val)
torch.manual_seed(seed_val)
np.random.seed(seed_val)
if is_losswise:
losswise_key = parameters['general']['losswise_api_key']
if losswise_key == 'None':
raise Exception(
"Must set API key in the parameters file to use losswise")
losswise.set_api_key(losswise_key)
session = losswise.Session(tag=experiment_name, max_iter=200)
graph_loss = session.graph("loss", kind="min")
graph_evasion_loss = session.graph("evasion_loss", kind="min")
graph_accuracy = session.graph("accuracy", kind="max")
graph_coverage = session.graph("coverage", kind="max")
graph_evasion = session.graph("evasion", kind="min")
graph_saving_metric = session.graph(
"metric_for_model_saving (ben_loss + max(mal_loss, evade_loss)",
kind="min")
evasion_iterations = int(parameters['hyperparam']['evasion_iterations'])
train_model_from_scratch = eval(
parameters['general']['train_model_from_scratch'])
import numpy as np
import keras
import losswise
from losswise.libs import LosswiseKerasCallback
from keras import backend as K
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers import Activation
from keras.layers.core import Dense
from keras.layers import LeakyReLU
from keras.layers import Dropout
from keras.optimizers import SGD
from keras.metrics import categorical_crossentropy
losswise.set_api_key('HK5Q1JWEY')
def extract_data(filename):
file = open(filename, 'r')
samples = []
labels = []
for line in file:
line_data = line.strip().split(sep=',')
line_data = [int(x) for x in line_data]
for i in range(10):
if i % 2 == 0:
from keras.layers import Input, Conv2D, Dropout, merge, Dense, Flatten, MaxPooling2D, GlobalAveragePooling2D, InputLayer
from keras.models import Model, Sequential
from keras import backend as K
from keras.models import load_model, model_from_json
from keras.optimizers import Adam
K.image_data_format() == 'channels_last'
from keras.utils import generic_utils
from keras.preprocessing.image import ImageDataGenerator
from keras import regularizers
from keras.callbacks import TensorBoard, Callback
from keras.losses import categorical_hinge, mean_squared_error, mean_absolute_error, categorical_crossentropy
import keras
import losswise
from losswise.libs import LosswiseKerasCallback
losswise.set_api_key('JWN8A6X96')
module_path = os.path.abspath(os.path.join('..'))
sys.path.append(module_path)
from utils.data_util import save_obj, load_obj, make_ids_list, load_model_from_file
from utils.pred_util import add_unknown_imgs, pred_generator, split_seq, square_error, gap
from conf.configure import *
from conf.generatorConf import *
from conf.predConf import *
if __name__ == "__main__":
train_lim = 0#2**15
pieces = 20
whole_ids_list = make_ids_list(val_images_folder)
if train_lim:
whole_ids_list = whole_ids_list[:train_lim]
class Char_LSTM(Model):
losswise.set_api_key('EFW1YNI7H')
def __init__(self, name=None, **parameters):
# max length of sentence to use, first n - 1 tokens --> input, last token --> output
super().__init__(**parameters)
# learning rate
self.learning_rate = parameters.get('learning_rate', 0.1)
self.max_sentence_len = parameters.get('max_sentence_len', 40)
self.step_size = parameters.get('step_size', 3)
# percentage of inputs to use for validation
self.validation_percentage = parameters.get('validation_perc', 0.05)
self.batch_size = parameters.get('batch_size', 128)
self.n_epochs = parameters.get('n_epochs', 40)
if (name is None):
import time
self.name = time.strftime("%Y%m%d-%H%M%S")
else:
self.name = name
self.stored_params = parameters
if not os.path.exists('../logs/' + self.name):
os.makedirs('../logs/' + self.name)
def load_data(self, path):
with io.open(path, encoding='utf-8') as f:
text = f.read().lower()
import re
self.text = re.sub("[A-Za-z0-9 _.,!?'\"#)(\-$&;:“”’%/…—‘ +–*=\n]+",
"", text)
# self.text = re.sub("\n", " ", self.text)
print('corpus length:', len(self.text))
self.corpus_size = len(self.text)
chars = sorted(list(set(self.text)))
self.charset_size = len(chars)
print('total chars:', self.charset_size)
self.char_indices = dict((c, i) for i, c in enumerate(chars))
self.indices_char = dict((i, c) for i, c in enumerate(chars))
# cut the text in semi-redundant sequences of maxlen characters
sentences = []
next_chars = []
for i in range(0,
len(self.text) - self.max_sentence_len, self.step_size):
sentences.append(self.text[i:i + self.max_sentence_len])
next_chars.append(self.text[i + self.max_sentence_len])
print('nb sequences:', len(sentences))
print('Vectorization...')
x = np.zeros((len(sentences), self.max_sentence_len, len(chars)),
dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
for t, char in enumerate(sentence):
x[i, t, self.char_indices[char]] = 1
y[i, self.char_indices[next_chars[i]]] = 1
nb_validation_samples = int(self.validation_percentage * x.shape[0])
x_train = x[:-nb_validation_samples]
y_train = y[:-nb_validation_samples]
x_val = x[-nb_validation_samples:]
y_val = y[-nb_validation_samples:]
return x_train, y_train, x_val, y_val
# build the model: a single LSTM
def generate_text(self, char_limit, **parameters):
# Function invoked at end of each epoch. Prints generated text.
start_index = 1
while (self.text[start_index] is not "\n"):
start_index = random.randint(
0, self.corpus_size - self.max_sentence_len - 1)
generated = ''
sentence = self.text[start_index:start_index + self.max_sentence_len]
generated += sentence
while (len(generated) < char_limit):
x_pred = np.zeros((1, self.max_sentence_len, self.charset_size))
for t, char in enumerate(sentence):
x_pred[0, t, self.char_indices[char]] = 1.
preds = self.model.predict(x_pred, verbose=0)[0]
next_index = sample(preds, 0.1)
next_char = self.indices_char[next_index]
generated += next_char
sentence = sentence[1:] + next_char
return generated
def save(self, path):
self.model.save(path + self.name + '.hd5')
with open(path + self.name + '.pkl', 'wb') as output:
pickle.dump(self.corpus_size, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.max_sentence_len, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.charset_size, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.text, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.indices_char, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.char_indices, output, pickle.HIGHEST_PROTOCOL)
# and loading
def load(self, model_file):
self.model = load_model(model_file)
if model_file.endswith('.hd5'):
model_file = model_file[:-4]
model_file = model_file + '.pkl'
else:
raise Exception()
with open(model_file, 'rb') as input:
self.corpus_size = pickle.load(input)
self.max_sentence_len = pickle.load(input)
self.charset_size = pickle.load(input)
self.text = pickle.load(input)
self.indices_char = pickle.load(input)
self.char_indices = pickle.load(input)
def train(self, data_path, **parameters):
x_train, y_train, x_val, y_val = self.load_data(data_path)
def on_epoch_end():
test = self.generate_text(280)
print(test)
def build_model(learning_rate=0.01, n_units=128):
print('Build model...')
model = Sequential()
model.add(
LSTM(n_units,
input_shape=(self.max_sentence_len, self.charset_size)))
model.add(Dense(self.charset_size))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
best_param = {}
do_gridsearch = False
if (do_gridsearch):
# wrap the model in a scikit-learn regressor
sci_model = KerasRegressor(build_fn=build_model)
# define the grid search parameters
learning_rate = [0.01, 0.05, 0.1]
n_units = [64, 128, 256]
epochs = [5]
param_grid = dict(learning_rate=learning_rate,
epochs=epochs,
n_units=n_units)
# fix random seed for reproducibility
seed = 42
numpy.random.seed(seed)
random.seed(seed)
# downsample randomly for gridsearch
if (len(x_train) > 10000):
train_x_d, train_y_d = downsample(x_train, y_train, 10000)
else:
train_x_d = x_train
train_y_d = y_train
grid = GridSearchCV(estimator=sci_model,
param_grid=param_grid,
n_jobs=1,
verbose=100,
cv=3)
grid_result = grid.fit(train_x_d, train_y_d)
# summarize results
best_param = grid_result.best_params_
print("Best: %f using %s" % (grid_result.best_score_, best_param))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
learning_rate = best_param[
'learning_rate'] if 'learning_rate' in best_param else self.learning_rate
n_units = best_param['n_units'] if 'n_units' in best_param else 128
self.model = build_model(learning_rate=learning_rate, n_units=n_units)
all_params = merge_two_dicts(best_param, self.stored_params)
# get the training history after fitting the model
history = self.model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.n_epochs,
validation_data=(x_val, y_val),
callbacks=[
LambdaCallback(on_epoch_end=on_epoch_end),
LosswiseKerasCallback(params=all_params),
EarlyStopping(patience=2)
])
plot_history(history, self.name)
