def spectrumToArrays(device, params, img_path, output_folder, file_name):
g = tf.Graph()
content_image = ut.read_image(img_path)
with g.device(device), g.as_default(), tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True)) as sess:
print "Load content values..."
image = tf.constant(content_image)
model = models.getModel(image, params)
content_image_y_val = [sess.run(y_l) for y_l in model.y()] # sess.run(y_l) is a constant numpy array
for i in range(len(content_image_y_val)):
output_path = output_folder + layers_names[i] + "/"
if not os.path.exists(output_path):
os.makedirs(output_path)
dirr = os.path.dirname(output_path + file_name)
if not os.path.exists(dirr):
os.makedirs(dirr)
np.save(output_path + file_name, content_image_y_val[i])
#cleanUp
del image
del content_image
del model
del content_image_y_val
del g
def __init__(self, mdlParams):
super(CNN_GRU, self).__init__()
# Some necessary vars
self.crop_number = mdlParams['multiCropTrain']
self.cell_type = mdlParams['cell_type']
self.cnn_output_point = mdlParams['CNN_Output_Point']
self.aux_classifier = mdlParams['aux_classifier']
# CNN first,up to feature vector
self.cnn = models.getModel(mdlParams['model_type_cnn'])()
if 'Dense' in mdlParams['model_type_cnn']:
if self.cnn_output_point == 'end':
self.cnn_features = self.cnn.features
elif self.cnn_output_point == 'transition3':
self.cnn_features = nn.Sequential(
*list(self.cnn.features.children())[:10])
elif 'InceptionV3' in mdlParams['model_type_cnn']:
if self.cnn_output_point == 'end':
self.cnn_features = nn.Sequential(
self.cnn.Conv2d_1a_3x3, self.cnn.Conv2d_2a_3x3,
self.cnn.Conv2d_2b_3x3,
nn.MaxPool2d(kernel_size=3, stride=2),
self.cnn.Conv2d_3b_1x1, self.cnn.Conv2d_4a_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Mixed_5b,
self.cnn.Mixed_5c, self.cnn.Mixed_5d, self.cnn.Mixed_6a,
self.cnn.Mixed_6b, self.cnn.Mixed_6c, self.cnn.Mixed_6d,
self.cnn.Mixed_6e, self.cnn.Mixed_7a, self.cnn.Mixed_7b,
self.cnn.Mixed_7c)
else:
if self.cnn_output_point == 'end':
self.cnn_features = nn.Sequential(self.cnn.layer0,
self.cnn.layer1,
self.cnn.layer2,
self.cnn.layer3,
self.cnn.layer4)
# The GRU
if self.cell_type == 'A':
self.gru = nn.GRU(mdlParams['CNN_Features'],
mdlParams['GRU_FM_Hidden'],
mdlParams['GRU_num_layers'],
batch_first=True,
bidirectional=mdlParams['bidirectional'])
# The classifier
if mdlParams['bidirectional']:
self.classifier = nn.Linear(2 * mdlParams['GRU_FM_Hidden'],
mdlParams['numClasses'])
else:
self.classifier = nn.Linear(mdlParams['GRU_FM_Hidden'],
mdlParams['numClasses'])
if self.aux_classifier:
num_ftrs = self.cnn.classifier.in_features
self.cnn.classifier = nn.Linear(num_ftrs, mdlParams['numClasses'])
def main():
train = pd.read_json("train.json")
#test = pd.read_json("test.json")
X_band_1 = np.array([
np.array(band).astype(np.float32).reshape(75, 75)
for band in train["band_1"]
])
X_band_2 = np.array([
np.array(band).astype(np.float32).reshape(75, 75)
for band in train["band_2"]
])
X_train = np.concatenate([
X_band_1[:, :, :, np.newaxis], X_band_2[:, :, :, np.newaxis],
((X_band_1 + X_band_2) / 2)[:, :, :, np.newaxis]
],
axis=-1)
target_train = train['is_iceberg']
file_path = ".model_weights.hdf5"
#X_valid, y_train_cv, y_valid = train_test_split(X_train, target_train, random_state=1, train_size=0.75)
X_train_cv, X_valid, y_train_cv, y_valid = GroupShuffleStratifiedS_angle(
X_train, target_train, train.inc_angle)
#create and set model
gmodel = getModel()
mypotim = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
gmodel.compile(loss='binary_crossentropy',
optimizer=mypotim,
metrics=['accuracy'])
#train our model
gmodel = train_model_(gmodel, X_train_cv, y_train_cv, X_valid, y_valid,
file_path)
#download our best weights from file, evaluate our model on 25% train data
gmodel.load_weights(filepath=file_path)
score = gmodel.evaluate(X_valid, y_valid, verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
'''
def main(start_epoch=0, end_epoch=1000, save_frequency=50, mode='train'):
print(f'Using device {device}')
assert mode in ('train', 'test')
assert save_frequency > 0
ds = data.RubikDataset(60000, maxScrambles)
dl = DataLoader(ds, batch_size=64, num_workers=16)
if start_epoch == 0:
net = models.getModel(mul=2).to(device)
else:
net = torch.load(getModelPath(start_epoch), map_location=device)
if mode == 'train':
train(net, dl, start_epoch, end_epoch, save_frequency)
if mode in ('train', 'test'):
sum, numSolves = 0, 50
for i in range(numSolves):
sum += testSolve(net, scrambles=random.randint(1000, 1099))
print(f'Average solution steps: {sum / numSolves}')
def __init__(self, mdlParams):
super(CNN_GRU_TP, self).__init__()
# Some necessary vars
self.crop_number = mdlParams['multiCropTrain']
self.cell_type = mdlParams['cell_type']
self.cnn_output_point = mdlParams['CNN_Output_Point']
# CNN first,up to feature vector
self.cnn = models.getModel(mdlParams['model_type_cnn'])()
if self.cnn_output_point == 'end':
self.cnn_features = self.cnn.features
elif self.cnn_output_point == 'transition3':
self.cnn_features = nn.Sequential(
*list(self.cnn.features.children())[:10])
# The GRU
if self.cell_type == 'A':
self.gru = nn.GRU(mdlParams['CNN_Features'],
mdlParams['GRU_FM_Hidden'],
mdlParams['GRU_num_layers'],
batch_first=True,
bidirectional=mdlParams['bidirectional'])
# The second CNN for global, make this better
self.cnn_global = tvmodels.densenet121(pretrained=True)
if self.cnn_output_point == 'end':
self.cnn_features_global = self.cnn_global.features
elif self.cnn_output_point == 'transition3':
self.cnn_features_global = nn.Sequential(
*list(self.cnn_global.features.children())[:10])
# The GRU
# The classifier
if mdlParams['bidirectional']:
self.classifier = nn.Linear(
2 * mdlParams['GRU_FM_Hidden'] + mdlParams['CNN_Features'],
mdlParams['numClasses'])
else:
self.classifier = nn.Linear(
mdlParams['GRU_FM_Hidden'] + mdlParams['CNN_Features'],
mdlParams['numClasses'])
with tf.device('gpu:%d' % mdlParams['numGPUs'][i]):
with tf.name_scope('tower_%d' % i) as scope:
# Batches for tower
modelVars['X_' +
str(i)], modelVars['Tar_' + str(i)], modelVars[
'Inds_' +
str(i)] = modelVars['iterator'].get_next()
#summaries.append(tf.summary.image('model input',modelVars['X_'+str(i)][:,:,:,0:],max_outputs=5))
print("Input", modelVars['X_' + str(i)],
modelVars['X_' + str(i)].get_shape(), "Output",
modelVars['Tar_' + str(i)],
modelVars['Tar_' + str(i)].get_shape())
# Build graph, put all variables on CPU
with slim.arg_scope([slim.model_variable, slim.variable],
device='/cpu:0'):
model_function = models.getModel(
mdlParams, placeholders)
modelVars['pred_' + str(i)] = model_function(
modelVars['X_' + str(i)])
print("Pred", modelVars['pred_' + str(i)],
modelVars['pred_' + str(i)].get_shape())
# Build loss
#modelVars['loss_'+str(i)] = tf.losses.mean_squared_error(labels=modelVars['Tar_'+str(i)], predictions=modelVars['pred_'+str(i)], scope=scope, loss_collection=tf.GraphKeys.LOSSES)
modelVars['loss_' + str(i)] = tf.reduce_mean(
tf.square(
tf.subtract(modelVars['Tar_' + str(i)],
modelVars['pred_' + str(i)])))
tf.add_to_collection(tf.GraphKeys.LOSSES,
modelVars['loss_' + str(i)])
# Reuse
tf.get_variable_scope().reuse_variables()
# Get relevant variables for training
def trainModel(modelname, X_train, y_train, config):
info("Training a {0} estimator".format(modelname), ind=0)
info("X data is {0}".format(getDim(X_train)), ind=2)
info("y data is {0}".format(getDim(y_train)), ind=2)
problemType = config['problem']
info("This is a {0} problem".format(problemType), ind=2)
modelData = getModelData(config, modelname)
tuneForParams = True
refitModel = False
goodModel = True
if modelData is not None:
if modelData.get('tune') is False:
tuneForParams = False
if modelData.get('fit') is True:
tuneForParams = False
if modelData.get('cv') is True:
tuneForParams = False
if modelData.get('refit') is True:
refitModel = True
if modelData.get('error') is True:
goodModel = False
else:
info("No model parameters were given. Using default {0} estimator".format(modelname), ind=4)
tuneForParams = False
if goodModel is False:
error("Model {0} is no good and will not run it.".format(modelname))
return None
#################################################################
# Get Model
#################################################################
retval = getModel(config, modelname)
#################################################################
# Tune Parameters
#################################################################
estimator = retval['estimator']
params = retval['params']
if tuneForParams:
tuneResults = tuneModel(modelname, estimator, params, X_train, y_train, config)
estimator = tuneResults['estimator']
params = tuneResults['params']
if refitModel:
try:
estimator.set_params(probability=True)
info("Set probability to True for model refit", ind=4)
except:
info("Could not set probability to True for model refit")
info("Re-fitting for {0} model parameters with probability".format(modelname), ind=4)
estimator = estimator.fit(X_train, y_train)
info("Finished re-fitting {0} model parameters with probability".format(modelname), ind=4)
else:
if estimator is not None:
info("Fitting for {0} model parameters".format(modelname), ind=2)
estimator = estimator.fit(X_train, y_train)
info("Finished fitting {0} model parameters".format(modelname), ind=4)
else:
error("No model with name {0} was trained".format(modelname))
return estimator
def main():
# add configuration file
# Dictionary for model configuration
mdlParams = {}
# Import machine config
pc_cfg = importlib.import_module('pc_cfgs.' + sys.argv[1])
mdlParams.update(pc_cfg.mdlParams)
# Import model config
model_cfg = importlib.import_module('cfgs.' + sys.argv[2])
mdlParams_model = model_cfg.init(mdlParams)
mdlParams.update(mdlParams_model)
# Indicate training
mdlParams['trainSetState'] = 'train'
# Path name from filename
mdlParams['saveDirBase'] = mdlParams['saveDir'] + sys.argv[2]
# Set visible devices
if 'gpu' in sys.argv[3]:
mdlParams['numGPUs'] = [[
int(s) for s in re.findall(r'\d+', sys.argv[3])
][-1]]
cuda_str = ""
for i in range(len(mdlParams['numGPUs'])):
cuda_str = cuda_str + str(mdlParams['numGPUs'][i])
if i is not len(mdlParams['numGPUs']) - 1:
cuda_str = cuda_str + ","
print("Devices to use:", cuda_str)
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_str
# Specify val set to train for
if len(sys.argv) > 4:
mdlParams['cv_subset'] = [
int(s) for s in re.findall(r'\d+', sys.argv[4])
]
print("Training validation sets", mdlParams['cv_subset'])
# Check if there is a validation set, if not, evaluate train error instead
if 'valIndCV' in mdlParams or 'valInd' in mdlParams:
eval_set = 'valInd'
print("Evaluating on validation set during training.")
else:
eval_set = 'trainInd'
print("No validation set, evaluating on training set during training.")
# Check if there were previous ones that have alreary bin learned
# prevFile = Path(mdlParams['saveDirBase'] + '/CV.pkl')
prevFile = Path(mdlParams['saveDirBase'] + '\CV.pkl')
# print(prevFile)
if prevFile.exists():
print("Part of CV already done")
# with open(mdlParams['saveDirBase'] + '/CV.pkl', 'rb') as f:
with open(mdlParams['saveDirBase'] + '\CV.pkl', 'rb') as f:
allData = pickle.load(f)
else:
allData = {}
allData['f1Best'] = {}
allData['sensBest'] = {}
allData['specBest'] = {}
allData['accBest'] = {}
allData['waccBest'] = {}
allData['aucBest'] = {}
allData['convergeTime'] = {}
allData['bestPred'] = {}
allData['targets'] = {}
# Take care of CV
if mdlParams.get('cv_subset', None) is not None:
cv_set = mdlParams['cv_subset']
else:
cv_set = range(mdlParams['numCV'])
for cv in cv_set:
# Check if this fold was already trained
already_trained = False
if 'valIndCV' in mdlParams:
mdlParams['saveDir'] = mdlParams['saveDirBase'] + '/CVSet' + str(
cv)
# mdlParams['saveDir'] = mdlParams['saveDirBase'] + '\CVSet' + str(cv)
if os.path.isdir(mdlParams['saveDirBase']):
if os.path.isdir(mdlParams['saveDir']):
all_max_iter = []
for name in os.listdir(mdlParams['saveDir']):
int_list = [int(s) for s in re.findall(r'\d+', name)]
if len(int_list) > 0:
all_max_iter.append(int_list[-1])
# if '-' + str(mdlParams['training_steps'])+ '.pt' in name:
# print("Fold %d already fully trained"%(cv))
# already_trained = True
all_max_iter = np.array(all_max_iter)
if len(all_max_iter) > 0 and np.max(
all_max_iter) >= mdlParams['training_steps']:
print(
"Fold %d already fully trained with %d iterations"
% (cv, np.max(all_max_iter)))
already_trained = True
if already_trained:
continue
print("CV set", cv)
# Reset model graph
importlib.reload(models)
# importlib.reload(torchvision)
# Collect model variables
modelVars = {}
# print("here")
modelVars['device'] = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
print(modelVars['device'])
# Def current CV set
mdlParams['trainInd'] = mdlParams['trainIndCV']
if 'valIndCV' in mdlParams:
mdlParams['valInd'] = mdlParams['valIndCV']
# Def current path for saving stuff
if 'valIndCV' in mdlParams:
mdlParams['saveDir'] = mdlParams['saveDirBase'] + '/CVSet' + str(
cv)
# mdlParams['saveDir'] = mdlParams['saveDirBase'] + '\CVSet' + str(cv)
else:
mdlParams['saveDir'] = mdlParams['saveDirBase']
# Create basepath if it doesnt exist yet
print(mdlParams['saveDir'])
if not os.path.isdir(mdlParams['saveDirBase']):
os.mkdir(mdlParams['saveDirBase'])
# Check if there is something to load
load_old = 0
if os.path.isdir(mdlParams['saveDir']):
# Check if a checkpoint is in there
print("i got here" + str(147))
if len([name for name in os.listdir(mdlParams['saveDir'])]) > 0:
load_old = 1
print("Loading old model")
else:
# Delete whatever is in there (nothing happens)
filelist = [
os.remove(mdlParams['saveDir'] + '/' + f)
for f in os.listdir(mdlParams['saveDir'])
]
# filelist = [os.remove(mdlParams['saveDir'] + '\\' + f) for f in os.listdir(mdlParams['saveDir'])]
else:
os.mkdir(mdlParams['saveDir'])
# Save training progress in here
save_dict = {}
save_dict['acc'] = []
save_dict['loss'] = []
save_dict['wacc'] = []
save_dict['auc'] = []
save_dict['sens'] = []
save_dict['spec'] = []
save_dict['f1'] = []
save_dict['step_num'] = []
if mdlParams['print_trainerr']:
save_dict_train = {}
save_dict_train['acc'] = []
save_dict_train['loss'] = []
save_dict_train['wacc'] = []
save_dict_train['auc'] = []
save_dict_train['sens'] = []
save_dict_train['spec'] = []
save_dict_train['f1'] = []
save_dict_train['step_num'] = []
# Potentially calculate setMean to subtract
if mdlParams['subtract_set_mean'] == 1:
mdlParams['setMean'] = np.mean(
mdlParams['images_means'][mdlParams['trainInd'], :], (0))
print("Set Mean", mdlParams['setMean'])
if mdlParams['balance_classes'] == 9:
# Only use official indicies for calculation
print("Balance 9")
indices_ham = mdlParams['trainInd'][mdlParams['trainInd'] < 25331]
if mdlParams['numClasses'] == 9:
class_weights_ = 1.0 / np.mean(
mdlParams['labels_array'][indices_ham, :8], axis=0)
# print("class before",class_weights_)
class_weights = np.zeros([mdlParams['numClasses']])
class_weights[:8] = class_weights_
class_weights[-1] = np.max(class_weights_)
else:
class_weights = 1.0 / np.mean(
mdlParams['labels_array'][indices_ham, :], axis=0)
print("Current class weights", class_weights)
if isinstance(mdlParams['extra_fac'], float):
class_weights = np.power(class_weights, mdlParams['extra_fac'])
else:
class_weights = class_weights * mdlParams['extra_fac']
print("Current class weights with extra", class_weights)
# Meta scaler
if mdlParams.get('meta_features',
None) is not None and mdlParams['scale_features']:
mdlParams[
'feature_scaler_meta'] = sklearn.preprocessing.StandardScaler(
).fit(mdlParams['meta_array'][mdlParams['trainInd'], :])
print("scaler mean", mdlParams['feature_scaler_meta'].mean_, "var",
mdlParams['feature_scaler_meta'].var_)
# Set up dataloaders
num_workers = psutil.cpu_count(logical=False)
# For train
dataset_train = utils.ISICDataset(mdlParams, 'trainInd')
# For val
dataset_val = utils.ISICDataset(mdlParams, 'valInd')
if mdlParams['multiCropEval'] > 0:
modelVars['dataloader_valInd'] = DataLoader(
dataset_val,
batch_size=mdlParams['multiCropEval'],
shuffle=False,
num_workers=num_workers,
pin_memory=True)
else:
modelVars['dataloader_valInd'] = DataLoader(
dataset_val,
batch_size=mdlParams['batchSize'],
shuffle=False,
num_workers=num_workers,
pin_memory=True)
modelVars['dataloader_trainInd'] = DataLoader(
dataset_train,
batch_size=mdlParams['batchSize'],
shuffle=True,
num_workers=num_workers,
pin_memory=True,
drop_last=True)
# print("Setdiff",np.setdiff1d(mdlParams['trainInd'],mdlParams['trainInd']))
# Define model
modelVars['model'] = models.getModel(mdlParams)()
# Load trained model
if mdlParams.get('meta_features', None) is not None:
# Find best checkpoint
files = glob(mdlParams['model_load_path'] + '/CVSet' + str(cv) +
'/*')
# files = glob(mdlParams['model_load_path'] + '\CVSet' + str(cv) + '\*')
global_steps = np.zeros([len(files)])
# print("files",files)
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' not in files[i]:
continue
if 'checkpoint' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+', files[i])]
global_steps[i] = nums[-1]
# Create path with maximum global step found
chkPath = mdlParams['model_load_path'] + '/CVSet' + str(
cv) + '/checkpoint_best-' + str(int(
np.max(global_steps))) + '.pt'
# chkPath = mdlParams['model_load_path'] + '\CVSet' + str(cv) + '\checkpoint_best-' + str(
# int(np.max(global_steps))) + '.pt'
print("Restoring lesion-trained CNN for meta data training: ",
chkPath)
# Load
state = torch.load(chkPath)
# Initialize model
curr_model_dict = modelVars['model'].state_dict()
for name, param in state['state_dict'].items():
# print(name,param.shape)
if isinstance(param, nn.Parameter):
# backwards compatibility for serialized parameters
param = param.data
if curr_model_dict[name].shape == param.shape:
curr_model_dict[name].copy_(param)
else:
print("not restored", name, param.shape)
# Define classifier layer
if 'Dense' in mdlParams['model_type']:
if mdlParams['input_size'][0] != 224:
modelVars['model'] = utils.modify_densenet_avg_pool(
modelVars['model'])
# print(modelVars['model'])
num_ftrs = modelVars['model'].classifier.in_features
modelVars['model'].classifier = nn.Linear(num_ftrs,
mdlParams['numClasses'])
# print(modelVars['model'])
elif 'dpn' in mdlParams['model_type']:
num_ftrs = modelVars['model'].classifier.in_channels
modelVars['model'].classifier = nn.Conv2d(num_ftrs,
mdlParams['numClasses'],
[1, 1])
# modelVars['model'].add_module('real_classifier',nn.Linear(num_ftrs, mdlParams['numClasses']))
# print(modelVars['model'])
elif 'efficient' in mdlParams['model_type'] and ('0' in mdlParams['model_type'] or '1' in mdlParams['model_type'] \
or '2' in mdlParams['model_type'] or '3' in mdlParams['model_type']):
num_ftrs = modelVars['model'].classifier.in_features
modelVars['model'].classifier = nn.Linear(num_ftrs,
mdlParams['numClasses'])
elif 'efficient' in mdlParams['model_type']:
num_ftrs = modelVars['model']._fc.in_features
modelVars['model'].classifier = nn.Linear(num_ftrs,
mdlParams['numClasses'])
elif 'wsl' in mdlParams['model_type'] or 'Resnet' in mdlParams[
'model_type'] or 'Inception' in mdlParams['model_type']:
# Do nothing, output is prepared
num_ftrs = modelVars['model'].fc.in_features
modelVars['model'].fc = nn.Linear(num_ftrs,
mdlParams['numClasses'])
else:
num_ftrs = modelVars['model'].last_linear.in_features
modelVars['model'].last_linear = nn.Linear(num_ftrs,
mdlParams['numClasses'])
# Take care of meta case
if mdlParams.get('meta_features', None) is not None:
# freeze cnn first
if mdlParams['freeze_cnn']:
# deactivate all
for param in modelVars['model'].parameters():
param.requires_grad = False
if 'wsl' in mdlParams['model_type'] or 'Resnet' in mdlParams[
'model_type'] or 'Inception' in mdlParams['model_type']:
# Activate classifier layer
for param in modelVars['model'].fc.parameters():
param.requires_grad = True
elif ('efficient' in mdlParams['model_type'] and ('0' in mdlParams['model_type'] or '1' in mdlParams['model_type'] \
or '2' in mdlParams['model_type'] or '3' in mdlParams['model_type'])) or 'Dense' in mdlParams['model_type']:
# Activate classifier layer
for param in modelVars['model'].classifier.parameters():
param.requires_grad = True
elif 'efficient' in mdlParams['model_type']:
#Activate classifier layer
for param in modelVars['model']._fc.parameters():
param.requires_grad = True
else:
# Activate classifier layer
for param in modelVars['model'].last_linear.parameters():
param.requires_grad = True
else:
# mark cnn parameters
for param in modelVars['model'].parameters():
param.is_cnn_param = True
# unmark fc
for param in modelVars['model']._fc.parameters():
param.is_cnn_param = False
# modify model
modelVars['model'] = models.modify_meta(mdlParams,
modelVars['model'])
# Mark new parameters
for param in modelVars['model'].parameters():
if not hasattr(param, 'is_cnn_param'):
param.is_cnn_param = False
# multi gpu support
if len(mdlParams['numGPUs']) > 1:
modelVars['model'] = nn.DataParallel(modelVars['model'])
modelVars['model'] = modelVars['model'].cuda()
# summary(modelVars['model'], modelVars['model'].input_size)# (mdlParams['input_size'][2], mdlParams['input_size'][0], mdlParams['input_size'][1]))
modelVars['criterion'] = nn.CrossEntropyLoss(
weight=torch.cuda.FloatTensor(class_weights.astype(np.float32)))
if mdlParams.get('meta_features', None) is not None:
#True for meta data
if mdlParams['freeze_cnn']:
modelVars['optimizer'] = optim.Adam(
filter(lambda p: p.requires_grad,
modelVars['model'].parameters()),
lr=mdlParams['learning_rate_meta'])
# sanity check
for param in filter(lambda p: p.requires_grad,
modelVars['model'].parameters()):
print(param.name, param.shape)
else:
modelVars['optimizer'] = optim.Adam(
[{
'params':
filter(lambda p: not p.is_cnn_param,
modelVars['model'].parameters()),
'lr':
mdlParams['learning_rate_meta']
}, {
'params':
filter(lambda p: p.is_cnn_param,
modelVars['model'].parameters()),
'lr':
mdlParams['learning_rate']
}],
lr=mdlParams['learning_rate'])
else:
modelVars['optimizer'] = optim.Adam(
modelVars['model'].parameters(), lr=mdlParams['learning_rate'])
# Decay LR by a factor of 0.2 every 25 epochs
modelVars['scheduler'] = lr_scheduler.StepLR(
modelVars['optimizer'],
step_size=mdlParams['lowerLRAfter'],
gamma=1 / np.float32(mdlParams['LRstep']))
# Define softmax
modelVars['softmax'] = nn.Softmax(dim=1)
# Set up training
# loading from checkpoint
if load_old:
# Find last, not last best checkpoint
files = glob(mdlParams['saveDir'] + '/*')
# files = glob(mdlParams['saveDir'] + '\*')
global_steps = np.zeros([len(files)])
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' in files[i]:
continue
if 'checkpoint-' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+', files[i])]
global_steps[i] = nums[-1]
# Create path with maximum global step found
chkPath = mdlParams['saveDir'] + '/checkpoint-' + str(
int(np.max(global_steps))) + '.pt'
# chkPath = mdlParams['saveDir'] + '\checkpoint-' + str(int(np.max(global_steps))) + '.pt'
print("Restoring: ", chkPath)
# Load
state = torch.load(chkPath)
# Initialize model and optimizer
modelVars['model'].load_state_dict(state['state_dict'])
modelVars['optimizer'].load_state_dict(state['optimizer'])
start_epoch = state['epoch'] + 1
mdlParams['valBest'] = state.get('valBest', 1000)
mdlParams['lastBestInd'] = state.get('lastBestInd',
int(np.max(global_steps)))
else:
start_epoch = 1
mdlParams['lastBestInd'] = -1
# Track metrics for saving best model
mdlParams['valBest'] = 1000
# Num batches
numBatchesTrain = int(
math.floor(len(mdlParams['trainInd']) / mdlParams['batchSize']))
print("Train batches", numBatchesTrain)
# Run training
start_time = time.time()
print("Start training...")
for step in range(start_epoch, mdlParams['training_steps'] + 1):
# One Epoch of training
print(step)
if step >= mdlParams['lowerLRat'] - mdlParams['lowerLRAfter']:
modelVars['scheduler'].step()
modelVars['model'].train()
for j, (inputs, labels,
indices) in enumerate(modelVars['dataloader_trainInd']):
# Run optimization
if mdlParams.get('meta_features', None) is not None:
inputs[0] = inputs[0].cuda()
inputs[1] = inputs[1].cuda()
else:
inputs = inputs.cuda()
labels = labels.cuda()
# zero the parameter gradients
modelVars['optimizer'].zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(True):
if mdlParams.get('aux_classifier', False):
outputs, outputs_aux = modelVars['model'](inputs)
loss1 = modelVars['criterion'](outputs, labels)
labels_aux = labels.repeat(mdlParams['multiCropTrain'])
loss2 = modelVars['criterion'](outputs_aux, labels_aux)
loss = loss1 + mdlParams[
'aux_classifier_loss_fac'] * loss2
else:
outputs = modelVars['model'](inputs)
loss = modelVars['criterion'](outputs, labels)
# backward + optimize only if in training phase
loss.backward()
modelVars['optimizer'].step()
if step % mdlParams[
'display_step'] == 0 or step == 1 or step == 101:
# Calculate evaluation metrics
if mdlParams['classification']:
# Adjust model state
modelVars['model'].eval()
# Get metrics
loss, accuracy, sensitivity, specificity, conf_matrix, f1, auc, waccuracy, predictions, targets, _ = utils.getErrClassification_mgpu(
mdlParams, eval_set, modelVars)
# Save in mat
save_dict['loss'].append(loss)
save_dict['acc'].append(accuracy)
save_dict['wacc'].append(waccuracy)
save_dict['auc'].append(auc)
save_dict['sens'].append(sensitivity)
save_dict['spec'].append(specificity)
save_dict['f1'].append(f1)
save_dict['step_num'].append(step)
if os.path.isfile(mdlParams['saveDir'] + '/progression_' +
eval_set + '.mat'):
os.remove(mdlParams['saveDir'] + '/progression_' +
eval_set + '.mat')
io.savemat(
mdlParams['saveDir'] + '/progression_' + eval_set +
'.mat', save_dict)
# if os.path.isfile(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat'):
# os.remove(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat')
# io.savemat(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat', save_dict)
eval_metric = -np.mean(waccuracy)
# Check if we have a new best value
if eval_metric < mdlParams['valBest']:
mdlParams['valBest'] = eval_metric
if mdlParams['classification']:
allData['f1Best'][cv] = f1
allData['sensBest'][cv] = sensitivity
allData['specBest'][cv] = specificity
allData['accBest'][cv] = accuracy
allData['waccBest'][cv] = waccuracy
allData['aucBest'][cv] = auc
oldBestInd = mdlParams['lastBestInd']
mdlParams['lastBestInd'] = step
allData['convergeTime'][cv] = step
# Save best predictions
allData['bestPred'][cv] = predictions
allData['targets'][cv] = targets
# Write to File
with open(mdlParams['saveDirBase'] + '/CV.pkl', 'wb') as f:
# with open(mdlParams['saveDirBase'] + '\CV.pkl', 'wb') as f:
pickle.dump(allData, f, pickle.HIGHEST_PROTOCOL)
# Delte previously best model
if os.path.isfile(mdlParams['saveDir'] +
'/checkpoint_best-' + str(oldBestInd) +
'.pt'):
os.remove(mdlParams['saveDir'] + '/checkpoint_best-' +
str(oldBestInd) + '.pt')
# if os.path.isfile(mdlParams['saveDir'] + '\checkpoint_best-' + str(oldBestInd) + '.pt'):
# os.remove(mdlParams['saveDir'] + '\checkpoint_best-' + str(oldBestInd) + '.pt')
# Save currently best model
state = {
'epoch': step,
'valBest': mdlParams['valBest'],
'lastBestInd': mdlParams['lastBestInd'],
'state_dict': modelVars['model'].state_dict(),
'optimizer': modelVars['optimizer'].state_dict()
}
torch.save(
state, mdlParams['saveDir'] + '/checkpoint_best-' +
str(step) + '.pt')
# torch.save(state, mdlParams['saveDir'] + '\checkpoint_best-' + str(step) + '.pt')
# If its not better, just save it delete the last checkpoint if it is not current best one
# Save current model
state = {
'epoch': step,
'valBest': mdlParams['valBest'],
'lastBestInd': mdlParams['lastBestInd'],
'state_dict': modelVars['model'].state_dict(),
'optimizer': modelVars['optimizer'].state_dict()
}
torch.save(
state,
mdlParams['saveDir'] + '/checkpoint-' + str(step) + '.pt')
# torch.save(state, mdlParams['saveDir'] + '\checkpoint-' + str(step) + '.pt')
# Delete last one
if step == mdlParams['display_step']:
lastInd = 1
else:
lastInd = step - mdlParams['display_step']
if os.path.isfile(mdlParams['saveDir'] + '/checkpoint-' +
str(lastInd) + '.pt'):
os.remove(mdlParams['saveDir'] + '/checkpoint-' +
str(lastInd) + '.pt')
# if os.path.isfile(mdlParams['saveDir'] + '\checkpoint-' + str(lastInd) + '.pt'):
# os.remove(mdlParams['saveDir'] + '\checkpoint-' + str(lastInd) + '.pt')
# Duration so far
duration = time.time() - start_time
# Print
if mdlParams['classification']:
print("\n")
print("Config:", sys.argv[2])
print('Fold: %d Epoch: %d/%d (%d h %d m %d s)' %
(cv, step, mdlParams['training_steps'],
int(duration / 3600),
int(np.mod(duration, 3600) / 60),
int(np.mod(np.mod(duration, 3600), 60))) +
time.strftime("%d.%m.-%H:%M:%S", time.localtime()))
print("Loss on ", eval_set, "set: ", loss, " Accuracy: ",
accuracy, " F1: ", f1, " (best WACC: ",
-mdlParams['valBest'], " at Epoch ",
mdlParams['lastBestInd'], ")")
print("Auc", auc, "Mean AUC", np.mean(auc))
print("Per Class Acc", waccuracy, "Weighted Accuracy",
np.mean(waccuracy))
print("Sensitivity: ", sensitivity, "Specificity",
specificity)
print("Confusion Matrix")
print(conf_matrix)
# Potentially print train err
if mdlParams['print_trainerr'] and 'train' not in eval_set:
print("Evaluating training error")
loss, accuracy, sensitivity, specificity, conf_matrix, f1, auc, waccuracy, predictions, targets, _ = utils.getErrClassification_mgpu(
mdlParams, 'trainIndEval', modelVars)
# Save in mat
save_dict_train['loss'].append(loss)
save_dict_train['acc'].append(accuracy)
save_dict_train['wacc'].append(waccuracy)
save_dict_train['auc'].append(auc)
save_dict_train['sens'].append(sensitivity)
save_dict_train['spec'].append(specificity)
save_dict_train['f1'].append(f1)
save_dict_train['step_num'].append(step)
if os.path.isfile(mdlParams['saveDir'] +
'/progression_trainInd.mat'):
os.remove(mdlParams['saveDir'] +
'/progression_trainInd.mat')
# if os.path.isfile(mdlParams['saveDir'] + '\progression_trainInd.mat'):
# os.remove(mdlParams['saveDir'] + '\progression_trainInd.mat')
scipy.io.savemat(
mdlParams['saveDir'] + '/progression_trainInd.mat',
save_dict_train)
# scipy.io.savemat(mdlParams['saveDir'] + '\progression_trainInd.mat', save_dict_train)
print("Train loss: ", loss, "Accuracy:", accuracy,
" F1: ", f1)
print("Per Class Acc", waccuracy, "Weighted Accuracy",
np.mean(waccuracy))
print("Sensitivity: ", sensitivity, "Specificity",
specificity)
print("Confusion Matrix")
print(conf_matrix)
# Free everything in modelvars
modelVars.clear()
# After CV Training: print CV results and save them
print("Best F1:", allData['f1Best'][cv])
print("Best Sens:", allData['sensBest'][cv])
print("Best Spec:", allData['specBest'][cv])
print("Best Acc:", allData['accBest'][cv])
print("Best Per Class Accuracy:", allData['waccBest'][cv])
print("Best Weighted Acc:", np.mean(allData['waccBest'][cv]))
print("Best AUC:", allData['aucBest'][cv])
print("Best Mean AUC:", np.mean(allData['aucBest'][cv]))
print("Convergence Steps:", allData['convergeTime'][cv])
def __init__(self, config_path):
self.image_config, self.model_config, self.run_config = LoadConfig(
config_path=config_path).train_config()
self.device = torch.device('cuda:%d' %
self.run_config['device_ids'][0] if torch.
cuda.is_available else 'cpu')
self.model = getModel(self.model_config)
os.makedirs(self.run_config['model_save_path'], exist_ok=True)
self.run_config['num_workers'] = self.run_config['num_workers'] * len(
self.run_config['device_ids'])
self.train_set = Data(root=self.image_config['image_path'],
phase='train',
data_name=self.image_config['data_name'],
img_mode=self.image_config['image_mode'],
n_classes=self.model_config['num_classes'],
size=self.image_config['image_size'],
scale=self.image_config['image_scale'])
self.valid_set = Data(root=self.image_config['image_path'],
phase='valid',
data_name=self.image_config['data_name'],
img_mode=self.image_config['image_mode'],
n_classes=self.model_config['num_classes'],
size=self.image_config['image_size'],
scale=self.image_config['image_scale'])
self.className = self.valid_set.className
self.train_loader = DataLoader(
self.train_set,
batch_size=self.run_config['batch_size'],
shuffle=True,
num_workers=self.run_config['num_workers'],
pin_memory=True,
drop_last=False)
self.valid_loader = DataLoader(
self.valid_set,
batch_size=self.run_config['batch_size'],
shuffle=True,
num_workers=self.run_config['num_workers'],
pin_memory=True,
drop_last=False)
train_params = self.model.parameters()
self.optimizer = RAdam(train_params,
lr=eval(self.run_config['lr']),
weight_decay=eval(
self.run_config['weight_decay']))
if self.run_config['swa']:
self.optimizer = SWA(self.optimizer,
swa_start=10,
swa_freq=5,
swa_lr=0.005)
# 设置学习率调节策略
self.lr_scheduler = utils.adjustLR.AdjustLr(self.optimizer)
if self.run_config['use_weight_balance']:
weight = utils.weight_balance.getWeight(
self.run_config['weights_file'])
else:
weight = None
self.Criterion = SegmentationLosses(weight=weight,
cuda=True,
device=self.device,
batch_average=False)
self.metric = utils.metrics.MetricMeter(
self.model_config['num_classes'])
paramslstm = {'BATCH_SIZE': 512,
'GRAD_CLIP': 100,
'LEARNING_RATE': 0.01,
'NAME': 'LSTM',
'NUM_CLUST': 3,
'NUM_EPOCHS': 10,
'NUM_FEATURES': 8,
'N_HIDDEN': 64,
'SEQ_LENGTH': 4,
'TYPE': '2nd classifier'}
# here in this example we are fetching a pre-trained model. Note
# that all the saved models are in the ./models directory
try:
# print paramsold
rnnModelold = getModel(paramsold, "rnnmodel-old")
except:
print "couldn't create the model... please correct the error"
try:
# print paramsold
rnnModel = getModel(params, "newest")
except:
print "couldn't create the model... enter a valid filename"
try:
# print paramslstm
lstmmodel = getModel(paramslstm, "lstmodel-old")
except:
print "couldn't create the model... please correct the error"
parser.add_argument('--numexs', help='Number of examples', required=True, type=int)
parser.add_argument('--dataname', help='Data file name prefix', required=True)
parser.add_argument('--modelname', help='modelname used in output', required=True)
parser.add_argument('--bkgrd', help='sequence background global/feature', required=False, default='global')
parser.add_argument('--task', help='task name', required=False, default='transmembrane-region')
args = parser.parse_args()
input_dim = len(dataset.AAs) + 1
#
# Model Specific Settings - need to change these based on the model
#
NUMEXS = args.numexs
DATA_NAME = args.dataname
MODEL_NAME = args.modelname
model = models.getModel(args.model, input_dim)
background = args.bkgrd
# path to save figures
fig_base_path = '/home/gene245/cprobert/deep-psp/keras/figures/' + DATA_NAME
# load training/test data
X_train, X_test, y_train, y_test = dataset.getSplitDataset(
args.task, num_exs=NUMEXS, bkgrd=background,
max_len=100, min_len=10, test_size=0.1)
model.compile(loss='categorical_crossentropy', optimizer='adagrad')
weights_path = '/home/gene245/cprobert/deep-psp/keras/output/' + DATA_NAME + '_model_weights.hdf5'
assert (args.bkgrd in ['global', 'feature'])
print 'using parameters:\n', vars(args)
print '--loading dataset--'
# load the input data
X_train, X_test, y_train, y_test = dataset.getSplitDataset(task_name=args.task,
num_exs=args.numexs,
bkgrd=args.bkgrd,
max_len=100,
min_len=10,
test_size=0.1)
feature_dim = len(dataset.AAs) + 1
model = models.getModel(args.model, feature_dim)
print '--compiling model--'
model.compile(loss='categorical_crossentropy', optimizer='adagrad')
# fit the model
print '--fitting model--'
fit_results = model.fit(X_train,
y_train,
nb_epoch=args.numepochs,
batch_size=32,
validation_split=0.1,
shuffle=True,
show_accuracy=True,
verbose=2)
def initialize_model(cv):
# Define model
modelVars['model'] = models.getModel(params)()
# Load trained model
if params.get('meta_features', None) is not None:
# Find best checkpoint
files = (params['model_load_path'] / f'CVSet{cv}').glob('/*')
global_steps = np.zeros([len(files)])
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' not in files[i]:
continue
if 'checkpoint' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+', str(files[i]))]
global_steps[i] = nums[-1]
# Create path with maximum global step found
chkPath = params[
'model_load_path'] / f'CVSet{cv}' / 'checkpoint_best-{int(np.max(global_steps))}.pt'
print("Restoring lesion-trained CNN for meta data training: ", chkPath)
# Load
state = torch.load(chkPath)
# Initialize model
curr_model_dict = modelVars['model'].state_dict()
for name, param in state['state_dict'].items():
if isinstance(param, nn.Parameter):
# backwards compatibility for serialized parameters
param = param.data
if curr_model_dict[name].shape == param.shape:
curr_model_dict[name].copy_(param)
else:
print("not restored", name, param.shape)
if 'Dense' in params['model_type']:
if params['input_size'][0] != 224:
modelVars['model'] = utils.modify_densenet_avg_pool(
modelVars['model'])
num_ftrs = modelVars['model'].classifier.in_features
modelVars['model'].classifier = nn.Linear(num_ftrs,
params['numClasses'])
#print(modelVars['model'])
elif 'dpn' in params['model_type']:
num_ftrs = modelVars['model'].classifier.in_channels
modelVars['model'].classifier = nn.Conv2d(num_ftrs,
params['numClasses'], [1, 1])
elif 'efficient' in params['model_type']:
# Do nothing, output is prepared
num_ftrs = modelVars['model']._fc.in_features
modelVars['model']._fc = nn.Linear(num_ftrs, params['numClasses'])
elif 'wsl' in params['model_type']:
num_ftrs = modelVars['model'].fc.in_features
modelVars['model'].fc = nn.Linear(num_ftrs, params['numClasses'])
else:
num_ftrs = modelVars['model'].last_linear.in_features
modelVars['model'].last_linear = nn.Linear(num_ftrs,
params['numClasses'])
# Take care of meta case
if params.get('meta_features', None) is not None:
# freeze cnn first
if params['freeze_cnn']:
# deactivate all
for param in modelVars['model'].parameters():
param.requires_grad = False
if 'efficient' in params['model_type']:
# Activate fc
for param in modelVars['model']._fc.parameters():
param.requires_grad = True
elif 'wsl' in params['model_type']:
# Activate fc
for param in modelVars['model'].fc.parameters():
param.requires_grad = True
else:
# Activate fc
for param in modelVars['model'].last_linear.parameters():
param.requires_grad = True
else:
# mark cnn parameters
for param in modelVars['model'].parameters():
param.is_cnn_param = True
# unmark fc
for param in modelVars['model']._fc.parameters():
param.is_cnn_param = False
# modify model
modelVars['model'] = models.modify_meta(params, modelVars['model'])
# Mark new parameters
for param in modelVars['model'].parameters():
if not hasattr(param, 'is_cnn_param'):
param.is_cnn_param = False
# multi gpu support
if params['numGPUs'] > 1:
modelVars['model'] = nn.DataParallel(modelVars['model'])
modelVars['model'] = modelVars['model'].cuda()
g = tf.Graph()
content_image = ut.process_image(img)
wanted_style = np.array([[0,1.0]])
with g.device(device), g.as_default(), tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True)) as sess:
print "Load content values and calculate style and content softmaxes"
#content
cW = tf.constant(content_weights)
cB = tf.constant(content_bias)
#style
sW = tf.constant(style_weights)
sB = tf.constant(style_bias)
wanted_style = tf.constant(wanted_style, tf.float32)
image = tf.constant(content_image)
model = models.getModel(image, params)
pool2_image_val = sess.run(model.y())
#fc7_image_val = sess.run(model.y())
reshaped_pool2_image_val = tf.reshape(pool2_image_val, [-1, 401408])
#ReshapeToVector(pool2_image_val)
content_values= tf.nn.softmax(tf.matmul(reshaped_pool2_image_val,cW) + cB)
##style_values= tf.nn.softmax(tf.matmul(pool2_image_val,sW) + sB)
print "Generate noise"
##gen_image = tf.Variable(tf.truncated_normal(content_image.shape, stddev=20), trainable=True, name='gen_image')
gen_image = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=True, name='gen_image')
##gen_image = tf.Variable(tf.constant(np.array(style_image, dtype=np.float32)), trainable=True, name='gen_image')
model_gen = models.getModel(gen_image, params)
pool2_gen_image_val = model_gen.y()
def get_activates(args):
if args.gpu >= 0:
cuda.check_cuda_available()
print('use GPU')
else:
print('use CPU only')
xp = cuda.cupy if args.gpu >= 0 else np
# cuda.cudnn_enabled = False
# Prepare dataset
val_list = dataio.load_image_list(args.val, args.root)
val_size = len(val_list)
assert val_size % args.val_batchsize == 0
# Prepare model
model = models.getModel(args.arch)
#from chainer.functions import caffe
#model = caffe.CaffeFunction('./models/bvlc_alexnet.caffemodel')
#model.insize=227
#model = caffe.CaffeFunction('./models/bvlc_googlenet.caffemodel')
#model.insize=224
if model is None:
raise ValueError('Invalid architecture name')
if args.finetune and hasattr(model, 'load_param'):
print('finetune')
model.load_param()
mean_image = None
if hasattr(model, 'getMean'):
mean_image = model.getMean()
else:
#mean_image = pickle.load(open(args.mean, 'rb'))
mean_image = np.load(args.mean)
#mean_image = np.ndarray((3, 256, 256), dtype=np.float32)
#mean_image[0] = 104
#mean_image[1] = 117
#mean_image[2] = 123
assert mean_image is not None
print(model.__class__.__name__)
print('batchsize (validation) : ' + str(args.val_batchsize))
nowt = datetime.datetime.today()
outdir = './' + args.arch
if not os.path.exists(outdir):
os.makedirs(outdir)
#outdir = './results/' + args.arch + '_bs' + str(args.batchsize) + '_' + nowt.strftime("%Y%m%d-%H%M")
#os.makedirs(outdir)
#args.out = outdir + '/' + args.out + '_' + args.arch
#args.outstate = outdir + '/' + args.outstate
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
# Init/Resume
if args.initmodel:
print('Load model from', args.initmodel)
serializers.load_hdf5(args.initmodel, model)
if args.resume:
print('Load optimizer state from', args.resume)
serializers.load_hdf5(args.resume, optimizer)
# ------------------------------------------------------------------------------
# This example consists of three threads: data feeder, logger and trainer.
# These communicate with each other via Queue.
data_q = queue.Queue(maxsize=1)
res_q = queue.Queue()
ret = TrainResult()
def feed_data():
# Data feeder
i = 0
count = 0
val_x_batch = np.ndarray(
(args.val_batchsize, 3, model.insize, model.insize),
dtype=np.float32)
val_y_batch = np.ndarray((args.val_batchsize, ), dtype=np.int32)
val_batch_pool = [None] * args.val_batchsize
pool = multiprocessing.Pool(args.loaderjob)
data_q.put('val')
j = 0
for path, label in val_list:
val_batch_pool[j] = pool.apply_async(
dataio.read_image,
(path, model.insize, mean_image, True, False))
val_y_batch[j] = label
j += 1
if j == args.val_batchsize:
for k, x in enumerate(val_batch_pool):
val_x_batch[k] = x.get()
data_q.put((val_x_batch.copy(), val_y_batch.copy()))
j = 0
pool.close()
pool.join()
data_q.put('end')
def log_result():
# Logger
testlogfilename = outdir + '/val.log'
activatescsvfilename = outdir + '/acts_' + args.layer + '.csv'
activatesnpyfilename = outdir + '/acts_' + args.layer + '.npy'
train_count = 0
train_cur_loss = 0
train_cur_accuracy = 0
begin_at = time.time()
val_begin_at = None
result = None
Ret = [ret]
activates = None
while True:
result = res_q.get()
if result == 'end':
print(file=sys.stderr)
break
elif result == 'val':
print(file=sys.stderr)
train = False
val_count = val_loss = val_accuracy = 0
val_begin_at = time.time()
continue
loss, accuracy, layer_activates = result
if activates is None:
activates = cuda.to_cpu(layer_activates)
else:
activates = np.r_[activates, cuda.to_cpu(layer_activates)]
val_count += args.val_batchsize
duration = time.time() - val_begin_at
throughput = val_count / duration
sys.stderr.write(
'\rval {} batches ({} samples) time: {} ({} images/sec)'.
format(val_count / args.val_batchsize, val_count,
datetime.timedelta(seconds=duration), throughput))
val_loss += loss
val_accuracy += accuracy
#print('accuacy', accuracy)
if val_count == val_size:
mean_loss = val_loss * args.val_batchsize / val_size
mean_error = 1 - val_accuracy * args.val_batchsize / val_size
print(file=sys.stderr)
print(
json.dumps({
'type': 'val',
'iteration': train_count,
'error': mean_error,
'loss': mean_loss
}))
with open(testlogfilename, 'a') as f:
f.write(
json.dumps({
'type': 'val',
'iteration': train_count,
'error': mean_error,
'loss': mean_loss
}) + '\n')
Ret[0].val_loss = mean_loss
Ret[0].val_error = mean_error
sys.stdout.flush()
print(activates.shape)
#np.savetxt(activatescsvfilename, activates, delimiter=",")
np.save(activatesnpyfilename, activates)
Ret[0].activates = activates
def train_loop():
# Trainer
while True:
while data_q.empty():
time.sleep(0.1)
inp = data_q.get()
if inp == 'end': # quit
res_q.put('end')
break
elif inp == 'val': # start validation
res_q.put('val')
model.train = False
continue
model.train = False
volatile = 'off' #if model.train else 'on'
x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)
model(x, t)
#fc8, = model(inputs={'data': x}, outputs=['fc8'], train=False)
#model.loss = F.softmax_cross_entropy(fc8, t)
#model.accuracy = F.accuracy(fc8, t)
#y, = model(inputs={'data': x}, outputs=['loss3/classifier'], disable=['loss1/ave_pool', 'loss2/ave_pool'], train=False)
#model.loss = F.softmax_cross_entropy(y, t)
#model.accuracy = F.accuracy(y, t)
variable = model.getLayerVariableFromLoss(args.layer)
#print(model.layer2rank(args.layer))
#print(variable)
ax = (2, 3) if len(variable.data.shape) == 4 else 1
layer_activates = variable.data.max(axis=ax)
#layer_activates = np.arange(args.val_batchsize * 10).reshape((args.val_batchsize, 10))
#data = cuda.to_cpu(variable.data)
#argmax = data.argmax(axis=(1))
#print(data.shape)
#print(argmax)
res_q.put((float(model.loss.data), float(model.accuracy.data),
layer_activates))
del x, t,
# Invoke threads
feeder = threading.Thread(target=feed_data)
feeder.daemon = True
feeder.start()
logger = threading.Thread(target=log_result)
logger.daemon = True
logger.start()
train_loop()
feeder.join()
logger.join()
# Save final model
ret.outdir = outdir
ret.valid = True
return ret
kernel = np.ones((2, 2), np.uint8)
kernel_1 = np.ones((5, 5), np.uint8)
opening = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_OPEN, kernel)
opening_1 = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_OPEN, kernel_1)
closing = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_CLOSE, kernel)
opening_1 = np.abs(255 - opening_1)
plt.imshow(opening_1)
y_l, x = np.unique(np.where(opening_1 > 0)[0]), np.unique(
np.where(opening_1 > 0)[1])
CONFIG_FILE = './config.cfg'
settings = parseTrainingOptions(CONFIG_FILE)
print settings
model = getModel(settings)
model.load_weights('./model.h5')
print 'Loading model: ./model.h5'
patch_size = 224
patches = []
flag = False
heatmap = np.zeros((slide.level_dimensions[slide_level][1],
slide.level_dimensions[slide_level][0]))
hfp = open('heatmapFile.csv', 'w')
batch_size = settings['batch_size'] #should be 32
counter = 0
batch = []
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--directory',
type=str,
required=True,
help="base directory")
args = parser.parse_args()
nns = []
for root, dirs, files in os.walk(args.directory):
for f in files:
if f == "configs.txt":
data = yaml.load(open(join(root, f)))
data["version"] = root.split("/")[-1]
#analyse model
m = models.getModel(data["model_schema"], 34)
nodes = []
for l in [i for i in m.layers if not "dropout" in i.name]:
nodes.append(int(l.output.shape[1]))
data["nodes"] = "-".join([str(i) for i in nodes])
data["n_weights"] = m.count_params()
nns.append(data)
nns = sorted(nns, key=lambda d: d["AUC"], reverse=True)
for d in nns:
print("-----------")
print(d)
def __init__(self, mdlParams):
super(CNN_FC, self).__init__()
# Some necessary vars
self.crop_number = mdlParams['multiCropTrain']
self.combine_features = mdlParams['combine_features']
self.cnn_output_point = mdlParams['CNN_Output_Point']
self.initial_attention = mdlParams['initial_attention']
if mdlParams.get('end_pool') is not None:
self.end_pool = mdlParams['end_pool']
else:
self.end_pool = False
self.end_attention = mdlParams['end_attention']
# CNN first,up to feature vector
self.cnn = models.getModel(mdlParams['model_type_cnn'])()
#print(self.cnn)
if 'Dense' in mdlParams['model_type_cnn']:
if self.cnn_output_point == 'end':
self.cnn_features = self.cnn.features
elif self.cnn_output_point == 'transition3':
self.cnn_features = nn.Sequential(
*list(self.cnn.features.children())[:10])
elif 'InceptionV3' in mdlParams['model_type_cnn']:
if self.cnn_output_point == 'end':
self.cnn_features = nn.Sequential(
self.cnn.Conv2d_1a_3x3, self.cnn.Conv2d_2a_3x3,
self.cnn.Conv2d_2b_3x3,
nn.MaxPool2d(kernel_size=3, stride=2),
self.cnn.Conv2d_3b_1x1, self.cnn.Conv2d_4a_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Mixed_5b,
self.cnn.Mixed_5c, self.cnn.Mixed_5d, self.cnn.Mixed_6a,
self.cnn.Mixed_6b, self.cnn.Mixed_6c, self.cnn.Mixed_6d,
self.cnn.Mixed_6e, self.cnn.Mixed_7a, self.cnn.Mixed_7b,
self.cnn.Mixed_7c)
else:
if self.cnn_output_point == 'end':
self.cnn_features = nn.Sequential(self.cnn.layer0,
self.cnn.layer1,
self.cnn.layer2,
self.cnn.layer3,
self.cnn.layer4)
#print(self.cnn_features)
# The classifier
if self.combine_features == 'add' or self.combine_features == 'conv1':
self.classifier = nn.Linear(mdlParams['CNN_Features'],
mdlParams['numClasses'])
else:
self.classifier = nn.Linear(
self.crop_number * mdlParams['CNN_Features'],
mdlParams['numClasses'])
if self.combine_features == 'conv1':
self.conv1x1 = nn.Conv1d(self.crop_number, 1, 1, 1)
if self.initial_attention:
if 'attention_size' in mdlParams:
self.initial_attention_layer = SEPatchLayer_Pool(
mdlParams['multiCropTrain'],
mdlParams['attention_size'][0])
else:
self.initial_attention_layer = SEPatchLayer(
mdlParams['multiCropTrain'])
if self.end_attention:
if 'attention_size' in mdlParams:
self.end_attention_layer = SEPatchLayer_Pool(
mdlParams['multiCropTrain'],
mdlParams['attention_size'][1])
else:
self.end_attention_layer = SEPatchLayer(
mdlParams['multiCropTrain'])
elif mdlParams['balance_classes'] == 9:
# Only use HAM indicies for calculation
indices_ham = mdlParams['trainInd'][mdlParams['trainInd'] < 10015]
class_weights = 1.0/np.mean(mdlParams['labels_array'][indices_ham,:],axis=0)
print("Current class weights",class_weights)
# Set up dataloaders
# For train
dataset_train = utils.ISICDataset(mdlParams, 'trainInd')
modelVars['dataloader_train'] = DataLoader(dataset_train, batch_size=mdlParams['batchSize'], shuffle=True, num_workers=8, pin_memory=True)
# For val
dataset_val = utils.ISICDataset(mdlParams, 'valInd')
modelVars['dataloader_val'] = DataLoader(dataset_val, batch_size=mdlParams['multiCropEval'], shuffle=False, num_workers=8, pin_memory=True)
#print("Setdiff",np.setdiff1d(mdlParams['trainInd'],mdlParams['trainInd']))
# Define model
modelVars['model'] = models.getModel(mdlParams['model_type'])()
# Original input size
if 'Dense' not in mdlParams['model_type']:
print("Original input size",modelVars['model'].input_size)
#print(modelVars['model'])
if 'Dense' in mdlParams['model_type']:
num_ftrs = modelVars['model'].classifier.in_features
modelVars['model'].classifier = nn.Linear(num_ftrs, mdlParams['numClasses'])
print(modelVars['model'])
elif 'dpn' in mdlParams['model_type']:
num_ftrs = modelVars['model'].classifier.in_channels
modelVars['model'].classifier = nn.Conv2d(num_ftrs,mdlParams['numClasses'],[1,1])
print(modelVars['model'])
else:
num_ftrs = modelVars['model'].last_linear.in_features
modelVars['model'].last_linear = nn.Linear(num_ftrs, mdlParams['numClasses'])
'GRAD_CLIP': 100,
'LEARNING_RATE': 0.01,
'NAME': 'LSTM',
'NUM_CLUST': 3,
'NUM_EPOCHS': 10,
'NUM_FEATURES': 8,
'N_HIDDEN': 64,
'SEQ_LENGTH': 4,
'TYPE': '2nd classifier'
}
# here in this example we are fetching a pre-trained model. Note
# that all the saved models are in the ./models directory
try:
# print paramsold
rnnModelold = getModel(paramsold, "rnnmodel-old")
except:
print("couldn't create the model... please correct the error")
try:
# print paramsold
rnnModel = getModel(params, "newest")
except:
print("couldn't create the model... enter a valid filename")
try:
# print paramslstm
lstmmodel = getModel(paramslstm, "lstmodel-old")
except:
print("couldn't create the model... please correct the error")
host)
args = vars(opt)
print('------------ Options -------------')
for k, v in sorted(args.items()):
print('%s: %s' % (str(k), str(v)))
print('-------------- End ----------------')
expr_dir = os.path.join(opt.checkpoints_dir, opt.name)
util.mkdirs(expr_dir)
opt.file_name = os.path.join(expr_dir, 'log.txt')
with open(opt.file_name, 'wt') as log_file:
log_file.write('------------ Options -------------\n')
for k, v in sorted(args.items()):
log_file.write('%s: %s\n' % (str(k), str(v)))
log_file.write('-------------- End ----------------\n')
Model = getModel(opt)
model = Model()
'''
model.initialize(opt)
print("model [%s] was created" % (model.name()))
'''
MRFDataset = getDataset(opt)
opt.set_type = 'train'
dataset_train = MRFDataset()
dataset_train.initialize(opt)
dataloader_train = torch.utils.data.DataLoader(dataset_train,
batch_size=opt.batchSize,
shuffle=True,
num_workers=int(opt.nThreads))
dataloader_train.dataset.patchSize = opt.patchSize
parser = argparse.ArgumentParser(description='Train a model')
parser.add_argument('--model', help='The model to use. Either RNN or LSTM', required=True)
parser.add_argument('--numexs', help='Number of examples', required=True, type=int)
parser.add_argument('--dataname', help='Data file name prefix', required=True)
parser.add_argument('--bkgrd', help='sequence background global/feature', required=False, default='global')
parser.add_argument('--task', help='task name', required=False, default='transmembrane-region')
args = parser.parse_args()
input_dim = len(dataset.AAs) + 1
#
# Model Specific Settings - need to change these based on the model
#
NUMEXS = args.numexs
DATA_NAME = args.dataname
model = models.getModel(args.model, input_dim)
background = args.bkgrd
# load training/test data
X_train, X_test, y_train, y_test = dataset.getSplitDataset(
args.task, num_exs=NUMEXS, bkgrd=background,
max_len=100, min_len=10, test_size=0.1)
model.compile(loss='categorical_crossentropy', optimizer='adagrad')
weights_path = '/home/gene245/cprobert/deep-psp/keras/output/' + DATA_NAME + '_model_weights.hdf5'
model.load_weights(weights_path)
test_preds = model.predict(X_test)
train_preds = model.predict(X_train)
def get_activated_patch(args):
if args.gpu >= 0:
cuda.check_cuda_available()
print('use GPU')
else:
print('use CPU only')
xp = cuda.cupy if args.gpu >= 0 else np
# cuda.cudnn_enabled = False
# Prepare dataset
val_list = dataio.load_image_list(args.val, args.root)
val_size = len(val_list)
assert val_size % args.val_batchsize == 0
categories = pd.read_csv(args.categories, header=None)
assert args.layeractivates is not None
layeractivates = np.load(args.layeractivates)
assert layeractivates.shape[0] == val_size
map_size = layeractivates.shape[1]
#indexes = np.argsort(a)
df = pd.DataFrame(layeractivates)
val_path_list = [v[0] for v in val_list]
val_labelnum_list = [v[1] for v in val_list]
val_label_list = [categories.ix[i[1],0] for i in val_list]
df['path'] = val_path_list
df['label'] = val_label_list
df['labelnum'] = val_labelnum_list
outdir = './' + args.arch + '/' + args.layer
if not os.path.isdir(outdir):
os.makedirs(outdir)
# Prepare model
model = models.getModel(args.arch)
if model is None:
raise ValueError('Invalid architecture name')
if args.finetune and hasattr(model, 'load_param'):
print ('finetune')
model.load_param()
mean_image = None
if hasattr(model, 'getMean'):
mean_image = model.getMean()
else:
#mean_image = pickle.load(open(args.mean, 'rb'))
mean_image = np.load(args.mean)
assert mean_image is not None
print(model.__class__.__name__)
print('batchsize (validation) : ' + str(args.val_batchsize))
val_image_pool = [None] * args.top
impool = multiprocessing.Pool(args.loaderjob)
nowt = datetime.datetime.today()
#outdir = './results/' + args.arch + '_bs' + str(args.batchsize) + '_' + nowt.strftime("%Y%m%d-%H%M")
#os.makedirs(outdir)
#args.out = outdir + '/' + args.out + '_' + args.arch
#args.outstate = outdir + '/' + args.outstate
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
# Init/Resume
if args.initmodel:
print('Load model from', args.initmodel)
serializers.load_hdf5(args.initmodel, model)
if args.resume:
print('Load optimizer state from', args.resume)
serializers.load_hdf5(args.resume, optimizer)
for c in six.moves.range(map_size):
#topacts = df.sort(i, ascending=False)[['path', 'label', i]].iloc[:args.top]
topacts = df.sort_values(by=[c], ascending=False)[['path', 'label', 'labelnum', c]].iloc[:args.top]
topacts.to_csv(outdir + '/' + "{0:0>4}".format(c) + '.csv', header=None, index=None)
images = np.zeros((3, model.insize, model.insize * args.top))
#for n, path in enumerate(topacts['path']):
# images[:,:,n*model.insize:(n+1)*model.insize] = \
# dataio.read_image(path, model.insize, None, True, False)
val_x_batch = np.ndarray(
(args.top, 3, model.insize, model.insize), dtype=np.float32)
val_y_batch = np.ndarray((args.top,), dtype=np.int32)
for n, path in enumerate(topacts['path']):
val_image_pool[n] = impool.apply_async(
dataio.read_image, (path, model.insize, mean_image, True, False))
for n, labelnum in enumerate(topacts['labelnum']):
val_y_batch[n] = labelnum
for k, im in enumerate(val_image_pool):
val_x_batch[k] = im.get()
# train
model.train = False
volatile = 'off' #if model.train else 'on'
x = chainer.Variable(xp.asarray(val_x_batch), volatile=volatile)
t = chainer.Variable(xp.asarray(val_y_batch), volatile=volatile)
model(x, t)
variable = model.getLayerVariableFromLoss(args.layer)
ax = (2,3) if len(variable.data.shape) == 4 else 1
maxloc= variable.data.argmax(axis=ax)
if c == 0:
print(variable.data[0])
print(maxloc.shape)
print(maxloc)
for n, path in enumerate(topacts['path']):
xmax, ymax = idx2loc(variable, maxloc[n, c])
pl, pr, pt, pb = get_patch_bounds(variable, xmax, ymax)
if c == 0 and n == 0:
print(xmax, ymax, ':', pl, pr, pt, pb)
patch = dataio.read_image_patch(path, model.insize, pl, pr, pt, pb)
if c == 0 and n == 0:
print(patch, patch.shape)
patchimg = Image.fromarray(np.uint8(patch[::-1].transpose(1, 2, 0)))
patchimg.save(outdir + '/' + "{0:0>4}".format(c) + '_' + "{0:0>2}".format(n) + args.ext)
#pilImg = Image.fromarray(np.uint8(images[::-1].transpose(1, 2, 0)))
#pilImg.save(outdir + '/' + "{0:0>4}".format(i) + '.jpg', 'JPEG', quality=100, optimize=True)
del x, t
impool.close()
impool.join()
return
# ------------------------------------------------------------------------------
# This example consists of three threads: data feeder, logger and trainer.
# These communicate with each other via Queue.
data_q = queue.Queue(maxsize=1)
res_q = queue.Queue()
ret = TrainResult()
def feed_data():
# Data feeder
i = 0
count = 0
val_x_batch = np.ndarray(
(args.val_batchsize, 3, model.insize, model.insize), dtype=np.float32)
val_y_batch = np.ndarray((args.val_batchsize,), dtype=np.int32)
val_batch_pool = [None] * args.val_batchsize
pool = multiprocessing.Pool(args.loaderjob)
data_q.put('val')
j = 0
for path, label in val_list:
val_batch_pool[j] = pool.apply_async(
dataio.read_image, (path, model.insize, mean_image, True, False))
val_y_batch[j] = label
j += 1
if j == args.val_batchsize:
for k, x in enumerate(val_batch_pool):
val_x_batch[k] = x.get()
data_q.put((val_x_batch.copy(), val_y_batch.copy()))
j = 0
pool.close()
pool.join()
data_q.put('end')
def log_result():
# Logger
testlogfilename=args.out+'/val.log'
activatescsvfilename=args.out+'/acts_'+args.layer+'.csv'
activatesnpyfilename=args.out+'/acts_'+args.layer+'.npy'
train_count = 0
train_cur_loss = 0
train_cur_accuracy = 0
begin_at = time.time()
val_begin_at = None
result = None
Ret = [ret]
activates=None
while True:
result = res_q.get()
if result == 'end':
print(file=sys.stderr)
break
elif result == 'val':
print(file=sys.stderr)
train = False
val_count = val_loss = val_accuracy = 0
val_begin_at = time.time()
continue
loss, accuracy, max_activates = result
if activates is None:
activates = cuda.to_cpu(max_activates)
else:
activates = np.r_[activates, cuda.to_cpu(max_activates)]
val_count += args.val_batchsize
duration = time.time() - val_begin_at
throughput = val_count / duration
sys.stderr.write(
'\rval {} batches ({} samples) time: {} ({} images/sec)'
.format(val_count / args.val_batchsize, val_count,
datetime.timedelta(seconds=duration), throughput))
val_loss += loss
val_accuracy += accuracy
#print('accuacy', accuracy)
if val_count == val_size:
mean_loss = val_loss * args.val_batchsize / val_size
mean_error = 1 - val_accuracy * args.val_batchsize / val_size
print(file=sys.stderr)
print(json.dumps({'type': 'val', 'iteration': train_count,
'error': mean_error, 'loss': mean_loss}))
with open(testlogfilename, 'a') as f:
f.write(json.dumps({'type': 'val', 'iteration': train_count,
'error': mean_error, 'loss': mean_loss})+'\n')
Ret[0].val_loss = mean_loss
Ret[0].val_error = mean_error
sys.stdout.flush()
print(activates.shape)
np.savetxt(activatescsvfilename, activates, delimiter=",")
np.save(activatesnpyfilename, activates)
Ret[0].activates = activates
def train_loop():
# Trainer
while True:
while data_q.empty():
time.sleep(0.1)
inp = data_q.get()
if inp == 'end': # quit
res_q.put('end')
break
elif inp == 'val': # start validation
res_q.put('val')
model.train = False
continue
model.train = False
volatile = 'off' #if model.train else 'on'
x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)
model(x, t)
#fc8, = model(inputs={'data': x}, outputs=['fc8'], train=False)
#model.loss = F.softmax_cross_entropy(fc8, t)
#model.accuracy = F.accuracy(fc8, t)
#y, = model(inputs={'data': x}, outputs=['loss3/classifier'], disable=['loss1/ave_pool', 'loss2/ave_pool'], train=False)
#model.loss = F.softmax_cross_entropy(y, t)
#model.accuracy = F.accuracy(y, t)
variable = model.getLayerVariableFromLoss(args.layer)
#print(model.layer2rank(args.layer))
#print(variable)
ax = (2,3) if len(variable.data.shape) == 4 else 1
maxloc= variable.data.argmax(axis=ax)
print(maxloc)
#max_activates = np.arange(args.val_batchsize * 10).reshape((args.val_batchsize, 10))
#data = cuda.to_cpu(variable.data)
#argmax = data.argmax(axis=(1))
#print(data.shape)
#print(argmax)
res_q.put((float(model.loss.data), float(model.accuracy.data), maxloc))
del x, t,
# Invoke threads
feeder = threading.Thread(target=feed_data)
feeder.daemon = True
feeder.start()
logger = threading.Thread(target=log_result)
logger.daemon = True
logger.start()
train_loop()
feeder.join()
logger.join()
# Save final model
ret.outdir = args.out
ret.valid = True
return ret
# For val
dataset_val = utils.ISICDataset(params, 'valInd')
if params['multiCropEval'] > 0:
modelVars['dataloader_valInd'] = DataLoader(dataset_val, batch_size=params['multiCropEval'], shuffle=False, num_workers=num_workers, pin_memory=True)
else:
modelVars['dataloader_valInd'] = DataLoader(dataset_val, batch_size=params['batchSize'], shuffle=False, num_workers=num_workers, pin_memory=True)
if params['balance_classes'] == 12 or params['balance_classes'] == 13:
#print(np.argmax(params['labels_array'][params['trainInd'],:],1).size(0))
strat_sampler = utils.StratifiedSampler(params)
modelVars['dataloader_trainInd'] = DataLoader(dataset_train, batch_size=params['batchSize'], sampler=strat_sampler, num_workers=num_workers, pin_memory=True)
else:
modelVars['dataloader_trainInd'] = DataLoader(dataset_train, batch_size=params['batchSize'], shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True)
#print("Setdiff",np.setdiff1d(params['trainInd'],params['trainInd']))
# Define model
modelVars['model'] = models.getModel(params)()
# Load trained model
if params.get('meta_features',None) is not None:
# Find best checkpoint
files = glob(params['model_load_path'] + '/CVSet' + str(cv) + '/*')
global_steps = np.zeros([len(files)])
#print("files",files)
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' not in files[i]:
continue
if 'checkpoint' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+',files[i])]
global_steps[i] = nums[-1]
modelVars = {}
modelVars['device'] = torch.device("cuda:" + cuda_str.strip())
# For train
dataset_train = utils.MetaAugDataset(mdlParams,
'trainInd',
index=train_index)
modelVars['dataloader_trainInd'] = DataLoader(
dataset_train, batch_size=mdlParams['batchSize'], shuffle=True)
# For val
dataset_val = utils.MetaAugDataset(mdlParams, 'valInd', index=valid_index)
modelVars['dataloader_valInd'] = DataLoader(
dataset_val, batch_size=mdlParams['batchSize'], shuffle=False)
# Define model
modelVars['model'] = models.getModel(mdlParams)()
# Load trained model
if mdlParams.get('load_previous', False):
# Find best checkpoint
files = glob(mdlParams['model_load_path'] + '/*')
global_steps = np.zeros([len(files)])
print("files", files)
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' not in files[i]:
continue
if 'checkpoint' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+', files[i])]
global_steps[i] = nums[-1]
#!/usr/bin/env python
import keras, models, losses
m = models.getModel("CascadeNet-5", (8, 9, 1))
m.summary()
def train(model_name,
optimizer_name,
scheduler_name,
lr,
img_path,
mask_path,
names_path,
epochs=10):
model = models.getModel(model_name)
model.build((None, None, None, 3))
print(model.summary())
scheduler = schedulers.getScheduler(scheduler_name, lr)
optimizer = optimizers.getOptimizer(optimizer_name, scheduler)
cce = tf.keras.losses.CategoricalCrossentropy()
train_loss_metric = tf.keras.metrics.Mean()
train_accuracy_metric = tf.keras.metrics.CategoricalAccuracy()
test_loss_metric = tf.keras.metrics.Mean()
test_accuracy_metric = tf.keras.metrics.CategoricalAccuracy()
file_list = open(names_path, 'r')
names = file_list.read().splitlines()
file_list.close()
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/gradient_tape/' + current_time + '/train'
test_log_dir = 'logs/gradient_tape/' + current_time + '/test'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir)
trainset, testval = ttsplit(names, train_size=0.9)
test, val = ttsplit(testval, train_size=0.5)
trainset = names
print(names)
total_step = 0
with tf.device('/device:GPU:0'):
for epoch in range(epochs):
for step_, batch in enumerate(trainset):
total_step += 1
print(total_step)
img, mask = utils.genData(batch, mask_path, img_path)
with tf.GradientTape() as tape:
mask_pred = model(img)
loss = cce(mask, mask_pred)
train_loss_metric.update_state(loss)
train_accuracy_metric.update_state(mask, mask_pred)
grads = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(grads, model.trainable_weights))
print(total_step)
if step_ % 150 == 0:
with train_summary_writer.as_default():
tf.summary.scalar('Training Loss',
train_loss_metric.result(),
step=total_step)
tf.summary.scalar('Training Accuracy',
train_accuracy_metric.result(),
step=total_step)
for step, batch in enumerate(val):
img_val, mask_val = utils.genData(
batch, mask_path, img_path)
mask_pred_val = model(img_val)
loss_val = cce(mask_val, mask_pred_val)
print(loss_val)
test_loss_metric.update_state(loss_val)
test_accuracy_metric.update_state(
mask_val, mask_pred_val)
with test_summary_writer.as_default():
tf.summary.scalar('Validation Loss',
test_loss_metric.result(),
step=total_step)
tf.summary.scalar('Validation Accuracy',
test_accuracy_metric.result(),
step=total_step)
print('Epoch: ' + str(epoch) + ' | Batch: ' + str(step) +
' | Training Loss: ' +
str(train_loss_metric.result().numpy()) +
' | Training Accuracy: ' +
str(train_accuracy_metric.result().numpy()))
print('Epoch: ' + str(epoch) + ' | Batch: ' + str(step) +
' | Validation Loss: ' +
str(test_loss_metric.result().numpy()) +
' | Validation Accuracy: ' +
str(test_accuracy_metric.result().numpy()))
train_loss_metric.reset_states()
train_accuracy_metric.reset_states()
test_loss_metric.reset_states()
test_accuracy_metric.reset_states()
batch_size = 64
epochs = 6
y_target = np.zeros((y_train.shape[0], len(ALL), y_train.shape[1]))
y_test_pred = np.zeros((X_test.shape[0], len(ALL) * fold, y_train.shape[1]))
n = 0
for name in DL:
# print('='*80)
seed = SEED * (n + 1)
kfold = list(
KFold(n_splits=fold, random_state=seed,
shuffle=True).split(X_train, y_train))
for i, (train_index, val_index) in enumerate(kfold):
X, y, val_X, val_y = X_train[train_index], y_train[
train_index], X_train[val_index], y_train[val_index]
util.seed_everything(seed + i)
model = models.getModel(param, name)
# if i == 0: print(model.summary())
filepath = param['subject_ckp_path'] + name + '-' + str(i + 1)
if not os.path.exists(filepath):
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=1,
min_lr=0.0001,
verbose=2)
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=2,
save_best_only=True,
mode='min')
model.fit(X,
y,
