def is_derailed(model, load_model_for_check=False):
from cvae import ClassificationVariationalNetwork
if isinstance(model, dict):
directory = model['dir']
elif isinstance(model, str):
directory = model
else:
directory = model.saved_dir
if os.path.exists(os.path.join(directory, 'derailed')):
return True
elif load_model_for_check:
try:
model = ClassificationVariationalNetwork.load(directory)
if torch.cuda.is_available():
model.to('cuda')
x = torch.zeros(1, *model.input_shape, device=model.device)
model.evaluate(x)
except ValueError:
return True
return False
def collect_models(directory,
wanted_epoch='last',
load_state=True, tpr_for_max=0.95, **default_load_paramaters):
from cvae import ClassificationVariationalNetwork
if 'dump' in directory:
return
assert wanted_epoch == 'last' or not load_state
try:
logging.debug(f'Loading net in: {directory}')
model = ClassificationVariationalNetwork.load(directory,
load_state=load_state,
**default_load_paramaters)
return make_dict_from_model(model, directory, tpr=tpr_for_max, wanted_epoch=wanted_epoch)
except (FileNotFoundError, PermissionError, NoModelError) as e:
pass
except RuntimeError as e:
logging.warning(f'Load error in {directory} see log file')
logging.debug(f'Load error: {e}')
def needed_remote_files(*mdirs, epoch='last', which_rec='all', state=False):
r""" list missing recorders to be fetched on a remote
-- mdirs: list of directories
-- epoch: last or min-loss or int
-- which_rec: either 'none' 'ind' or 'all'
-- state: wehter to include state.pth
returns generator of needed files paths
"""
assert not state or epoch == 'last'
from cvae import ClassificationVariationalNetwork as M
for d in mdirs:
m = M.load(d, load_net=False)
epoch_ = epoch
if epoch_ == 'min-loss':
epoch_ = m.training_parameters.get('early-min-loss', 'last')
if epoch_ == 'last':
epoch_ = max(m.testing)
if isinstance(epoch_, int):
epoch_ = '{:04d}'.format(epoch_)
testset = m.training_parameters['set']
sets = []
recs_to_exclude = which_rec.split('-')[1:]
which_rec_ = which_rec.split('-')[0]
if which_rec_ in ('all', 'ind'):
sets.append(testset)
if which_rec_ == 'all':
sets += get_same_size_by_name(testset)
for _ in [_ for _ in recs_to_exclude if _ in sets]:
sets.remove(_)
for s in sets:
sdir = os.path.join(d, 'samples', epoch_, 'record-{}.pth'.format(s))
if not os.path.exists(sdir):
yield d, sdir
if state:
sdir = os.path.join(d, 'state.pth')
if not os.path.exists(sdir):
yield d, sdir
def gather_registered_models(mdict, filter, tpr_for_max=0.95, wanted_epoch='last', **kw):
from cvae import ClassificationVariationalNetwork
mlist = []
for _ in mdict:
if filter is None or filter.filter(mdict[_]):
m = ClassificationVariationalNetwork.load(_, **kw)
mlist.append(make_dict_from_model(m, _, tpr=tpr_for_max, wanted_epoch=wanted_epoch))
return mlist
def load(cls, dir_name, *a, **kw):
architecture = save_load.load_json(dir_name, 'params.json')
models = [architecture[str(_)] for _ in range(len(architecture))]
m = cls(*[M.load(_, *a, **kw) for _ in models])
try:
m.testing = save_load.load_json(dir_name,
'test.json',
presumed_type=int)
except (FileNotFoundError):
pass
try:
m.ood_results = save_load.load_json(dir_name,
'ood.json',
presumed_type=int)
except (FileNotFoundError):
pass
m.saved_dir = dir_name
return m
logging.info('{} model{} over {}'.format(len(mdirs),
's' if len(mdirs) > 1 else '',
total_models))
if removed:
logging.error('Exiting, load files')
logging.error(
'E.g: %s',
'$ rsync -avP --files-from=/tmp/files remote:dir/joint-vae .')
logging.error(' Or: %s', '$ . /tmp/rsync-files remote:dir/joint-vae')
with open('/tmp/rsync-files', 'w') as f:
f.write('#!/bin/bash\n')
f.write('rsync -avP --files-from=/tmp/files $1 .\n')
sys.exit(1)
models = [M.load(d, load_state=True) for d in mdirs]
model = IteratedModels(*models)
device = args.device
model.to(device)
logging.debug('Model sent to {} (device wanted: {})'.format(
next(iter(model.parameters())), device))
testset = model.training_parameters['set']
allsets = [testset]
allsets.extend(get_same_size_by_name(testset))
transformer = model.training_parameters['transformer']
n_by_rep = dict(hsv=0, rgb=0)
as_in = {'ind': {}, 'correct': {}}
pr = {'ind': {}, 'correct': {}}
y_classif = {}
classif_acc = {}
agreement = {}
distribution = {}
y_true = None
for mdir in mdirs:
model = M.load(mdir, load_net=False)
rep = model.architecture['representation']
name = rep.upper() + str(n_by_rep[rep])
name = str(model.job_number)
n_by_rep[rep] += 1
current_testset = model.training_parameters['set']
if testset and current_testset != testset:
continue
else:
testset = current_testset
if n_by_rep[rep] > max_per_rep:
continue
if args.when == 'min-loss':
from cvae import ClassificationVariationalNetwork as M
from utils.save_load import available_results, make_dict_from_model, LossRecorder, find_by_job_number, load_json
import logging
logging.getLogger().setLevel(logging.DEBUG)
mdir = '/tmp/000033'
mdir = '/tmp/000186'
mdir = '/tmp/151320'
mdir = '/tmp/151024'
mdir = '/tmp/151409'
mdir = '/tmp/151020'
mdir = '/tmp/148722'
print('Loading')
model = M.load(mdir, load_state=False)
new = M((3, 32, 32), 10, type_of_net='vib')
print('Loaded')
"""
acc = {}
acc = {_: model.accuracy(wygiwyu=True, wanted_epoch=_) for _ in (0, 10, 200, 'last')}
print(acc)
"""
# model.trained = 2000
# model.testing[2000].pop('iws')
# model.ood_results.pop(2000)
args.add_argument('--soft', choices=['kl', 'iws'], default='default')
args.add_argument('--hard', choices=['kl', 'iws'])
args.add_argument('--entropy', '-H', action='store_true')
args.add_argument('--elbo', action='store_true')
args.add_argument('--baseline', action='store_true')
args.add_argument('--2s', action='store_true', dest='two_sided')
args.add_argument('--print', action='store_true')
a = args.parse_args()
j = a.j
ood_tpr = a.ood_tpr / 100
mis_tpr = a.mis_tpr / 100
reload = False
if a.direct_load:
net = Model.load(a.direct_load, load_state=False)
print(net.job_number, 'loaded')
try:
reload = net.job_number != j and not a.direct_load
except NameError:
reload = True
if reload:
net = find_by_job_number(j, load_state=False)['net']
dir_path = os.path.join(net.saved_dir, 'samples', 'last')
testset = net.training_parameters['set']
if net.type == 'vib':
pass # a.plot = False
from matplotlib import pyplot as plt
logging.getLogger().setLevel(logging.WARNING)
compute = False
compute = True
if compute:
j = 107495
j = 108183
j = 37
load_dir = find_by_job_number('./jobs', j, load_net=False)[j]['dir']
print('Load net', end='')
net = ClassificationVariationalNetwork.load(load_dir, load_state=True)
print(' to gpu')
net.to('cuda')
net.latent_sampling = 16
print('Getting sets')
trainset_name = net.training['set']
trainset, testset = dl.get_dataset(trainset_name,
transformer=net.training['transformer'])
oodsets = [dl.get_dataset(n)[1] for n in testset.same_size]
oodset = oodsets[0]
batch_size = 200
n_batch = 1000 // batch_size
loader = torch.utils.data.DataLoader(testset,
for job_number in job_numbers:
jobs[job_number]
except (NameError, KeyError):
print('Loading jobs')
reload = True
recompute = True
if reload:
jobs = find_by_job_number(search_dir, *job_numbers, load_state=False)
#, json_file='networks-lss.json')
to_be_removed = []
for job_number in jobs:
try:
jobs[job_number]['net'] = ClassificationVariationalNetwork.load(
jobs[job_number]['dir'])
except RuntimeError:
print(f'Error loading {job_number}')
to_be_removed.append(job_number)
for job_number in to_be_removed:
jobs.pop(job_number)
fgrid = {}
fexamples = {}
food = {}
foodexamples = {}
fmuvar = {}
fhist = {}
fx_ = {}
oodsets = test_dataset.same_size
for o in oodsets:
_, ood_dataset = tl.get_dataset(o, transformer=transformer)
x[o], y[o] = tl.get_batch(ood_dataset,
device=device,
batch_size=max(z_sample, N))
if not L:
L = args.total_width // (1 + len(x))
for n in list_of_nets:
logging.info('loading state of %s', n['job'])
model = Net.load(n['dir'])
model.to(device)
logging.info('done')
logging.info('Compute max batch size')
batch_size = min(
m, model.compute_max_batch_size(batch_size=m, which='test'))
logging.info(f'done ({batch_size})')
for s in x:
logging.info('sampling %s', s)
if N:
list_of_images = sample(model,
x[s][:N],
for s in archs:
archs[s] = {
n['model']['arch']
for n in models_to_be_kept if n['model']['set'] == s
}
for m_ in models_to_be_kept:
m = m_['model']
epoch = m_['epoch']
plan = m_['plan']
if plan['recorders'] or plan['compute']:
print('Computing rates of job {} of type {} at epoch {}'.format(
m['job'], m['type'], epoch))
logging.debug('Plan for {}; {}'.format(m['job'], plan))
model = CVNet.load(m['dir'], load_state=plan['compute'])
if plan['compute']:
device = args.device or 'cuda'
else:
device = args.device or 'cpu'
logging.debug('Will work on {}'.format(device))
model.to(device)
with torch.no_grad():
print('OOD')
model.ood_detection_rates(
epoch=epoch,
from_where=where,
sample_dirs=[
os.path.join(m['dir'], 'samples',
'{:4d}'.format(epoch))
if done_epochs == 0:
verb = 'will start from scratch.'
elif done_epochs < args.epochs:
verb = f'will resume from {done_epochs}.'
else:
verb = 'is already done.'
log.info(f'Training {verb}')
except (NoModelError):
log.error(f'model #{job_TBR_num} not found!')
sys.exit(1)
else:
try:
resumed_from = resume
log.info('Loading network in %s', resume)
jvae = CVNet.load(args.resume, load_state=True)
log.debug('Network loaded in {}'.format(resumed_from))
done_epochs = jvae.trained
if done_epochs == 0:
verb = 'will start from scratch.'
elif done_epochs < args.epochs:
verb = f'will resume from {done_epochs}.'
else:
verb = 'is already done.'
log.info(f'Training {verb}')
except (FileNotFoundError, NameError):
log.error(f'network not found in {resume}')
sys.exit(1)
else: