def get_example_dataloaders():
df_path = r'data/processed_dataframe.csv'
img_col = 'filename'
cont_cols = ['followers', 'following', 'engagement_factor_std', 'month', 'year', 'day_name', 'hour']
cat_cols = []
target_col = 'engagement_factor_moving_avg'
image_path = Path(r'data/Images')
tfms = Transforms([Resize(256), ToTorch()])
df = pd.read_csv(r'data/processed_dataframe.csv')
train_idx = df.sample(frac=0.8).index
valid_idx = df.loc[~df.index.isin(train_idx)].index
ds_train = Dataset(df_path,
img_col = img_col,
cont_cols = cont_cols,
cat_cols = cat_cols,
target_col = target_col,
image_path = image_path,
transforms = tfms,
indices = train_idx)
ds_valid = Dataset(df_path,
img_col = img_col,
cont_cols = cont_cols,
cat_cols = cat_cols,
target_col = target_col,
image_path = image_path,
transforms = tfms,
indices = valid_idx)
dl_train = DataLoader(dataset = ds_train,
sampler = Sampler(ds_train, bs = 16),
collate_func = collate)
dl_valid = DataLoader(dataset = ds_valid,
sampler = Sampler(ds_valid, bs = 16),
collate_func = collate)
return dl_train, dl_valid
args = parser.parse_args()
#Main Code
base_data_dir='data/'
if args.htune:
base_model_dir='htune/model-approx/'
else:
base_model_dir='models/'
dataset_name=args.dataset_name
#Dataset
if dataset_name=='bn1':
dataset= pd.read_csv(base_data_dir+dataset_name+'.csv')
dataset.drop(['Unnamed: 0'], axis=1, inplace=True)
params= {'dataframe':dataset.copy(), 'continuous_features':['x1','x2','x3'], 'outcome_name':'y'}
d = DataLoader(params)
path= base_model_dir + 'bn1-64-50-auto-encoder.pth'
elif dataset_name=='adult':
dataset = load_adult_income_dataset()
params= {'dataframe':dataset.copy(), 'continuous_features':['age','hours_per_week'], 'outcome_name':'income'}
d = DataLoader(params)
elif dataset_name=='sangiovese':
dataset = pd.read_csv( base_data_dir + dataset_name + '.csv', index_col=None )
dataset= dataset.drop(columns= ['Unnamed: 0'])
outcome=[]
for i in range(dataset.shape[0]):
if dataset['GrapeW'][i] > 0:
outcome.append( 1 )
else:
def main():
"""Create the model and start the training."""
args = get_arguments()
logs_dir = os.path.join(LOGS_ROOT, args.exper_name, LOGS_DIR_SUFFIX)
snap_dir = os.path.join(LOGS_ROOT, args.exper_name, SNAPSHOT_DIR_SUFFIX)
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
c_h, c_w = map(int, args.crop_size.split(','))
crop_size = (c_h, c_w)
tf.set_random_seed(args.random_seed)
# Coordinator for threads
coord = tf.train.Coordinator()
with tf.name_scope("create_inputs"):
reader = DataLoader(args.data_dir, input_size, crop_size, True, False,
False, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
addNameToTensor(image_batch, 'input')
with tf.name_scope("validating_input"):
validate_reader = DataLoader(args.data_dir, input_size, crop_size,
False, True, False, coord)
image_validate_batch, label_validate_batch = validate_reader.dequeue(
args.batch_size)
# for training
with tf.variable_scope(''):
net = G_UNetResidual_Ext2({'data': image_batch},
is_training=True,
num_classes=args.num_classes)
for layer in net.layers:
print(layer)
print(net.layers[layer].shape)
# tensors = [n.name for n in tf.get_default_graph().as_graph_def().node]
# for tensor in tensors[:2000]:
# print(tensor)
total_params = 0
for variable in tf.trainable_variables():
shape = variable.get_shape()
# print(variable)
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_params += variable_parameters
print("Number of trainable parameters: %d" % (total_params))
# return
# for validation
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
net_val = G_UNetResidual_Ext2({'data': image_validate_batch},
is_training=False,
num_classes=args.num_classes)
with tf.variable_scope('training_output'):
# output training
logits = net.getOutput()
addNameToTensor(logits, 'output')
with tf.variable_scope('validation_output'):
# output validation
logits_validation = net_val.getOutput()
restore_var = [v for v in tf.global_variables()]
with tf.variable_scope('training_loss'):
train_weights = tf.gather(CLASS_WEIGHTS, label_batch)
# loss for training
ce_loss = tf.losses.sparse_softmax_cross_entropy(
label_batch, logits, train_weights)
ce_reduced_loss = tf.reduce_mean(ce_loss)
dice_loss = generalised_dice_loss(tf.nn.softmax(logits), label_batch)
train_loss = (ce_reduced_loss + dice_loss)
with tf.variable_scope('validation_loss'):
valid_weights = tf.gather(CLASS_WEIGHTS, label_validate_batch)
# loss for validation
ce_loss_validation = tf.losses.sparse_softmax_cross_entropy(
label_validate_batch, logits_validation, valid_weights)
ce_reduced_loss_validation = tf.reduce_mean(ce_loss_validation)
dice_loss_validation = generalised_dice_loss(
tf.nn.softmax(logits_validation), label_validate_batch)
valid_loss = (ce_reduced_loss_validation + dice_loss_validation)
with tf.variable_scope('accuracy'):
# accuracy
preds = tf.argmax(logits_validation, axis=-1)
acc = tf.reduce_mean(
tf.cast(tf.equal(preds, label_validate_batch), tf.double))
with tf.variable_scope('probability'):
probs = tf.nn.softmax(logits_validation)
with tf.variable_scope('train'):
batch_ops = tf.get_collection(UPDATE_OPS_COLLECTION)
old_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# print(batch_ops)
# print(old_update_ops)
update_ops = batch_ops + old_update_ops
with tf.control_dependencies(update_ops):
# training optimizer
optimizer = tf.train.AdamOptimizer(
learning_rate=args.learning_rate)
train_op = optimizer.minimize(
train_loss, global_step=tf.train.get_global_step())
init = tf.global_variables_initializer()
sess = tf.keras.backend.get_session()
K.set_session(sess)
K.set_learning_phase(1)
sess.run(init)
train_logger = Logger(logs_dir + '/train')
valid_logger = Logger(logs_dir + '/valid')
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1)
# initialize weight
ckpt = tf.train.get_checkpoint_state(snap_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(
os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
load_step = 0
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
min_val_loss = 1e9
best_acc = 0
print(K.learning_phase())
for step in range(args.num_steps):
realStep = step + load_step
start_time = time.time()
loss_value, _ = sess.run([train_loss, train_op],
feed_dict={net.update_bn: True})
train_logger.log_scalar('loss', loss_value, realStep)
if realStep % args.save_pred_every == 0:
# train_output = np.uint8(batch_probs[0] * 255)
# img = Image.fromarray(train_output)
# img.save('temp_output.png')
print('Validation')
numValidateImg = len(
os.listdir(os.path.join(args.data_dir, 'valid', 'img')))
numStep = int(numValidateImg / args.batch_size)
loss_validation_test = 0
accuracy = 0
for step_val in range(numStep):
predicted, batch_prob, gt, l, ac = sess.run([
logits_validation, probs, label_validate_batch, valid_loss,
acc
])
if step_val == 0:
valid_output = np.uint8(batch_prob[0] * 255)
img = Image.fromarray(valid_output)
img.save('temp_valid_output.png')
gt_output = np.uint8(toOneHot(gt[0]) * 255)
img = Image.fromarray(gt_output)
img.save('temp_gt_output.png')
loss_validation_test = loss_validation_test + l / numStep
# accuracy = accuracy + ac / numStep
accuracy = accuracy + cal_AJI_batch(predicted, gt,
args.batch_size) / numStep
print('Validation result: loss = %.5f, acc = %.5f' %
(loss_validation_test, accuracy))
if accuracy > best_acc:
print(
'Update model: previous loss: %4.lf, new loss: %.4lf, step: %d'
% (min_val_loss, loss_validation_test, realStep))
min_val_loss = loss_validation_test
best_acc = accuracy
save(saver, sess, snap_dir, realStep)
valid_logger.log_scalar('loss', loss_validation_test, realStep)
valid_logger.log_scalar('acc', accuracy, realStep)
duration = time.time() - start_time
print('step %d \t loss = %.5f, (%.3f sec/step)' %
(realStep, loss_value, duration))
coord.request_stop()
coord.join(threads)
base_model_dir = 'models/'
dataset_name = args.dataset_name
if not os.path.exists(base_model_dir + args.oracle_data[:-5]):
os.mkdir(base_model_dir + args.oracle_data[:-5])
#Dataset
if dataset_name == 'bn1':
dataset = pd.read_csv(base_data_dir + dataset_name + '.csv')
dataset.drop(['Unnamed: 0'], axis=1, inplace=True)
params = {
'dataframe': dataset.copy(),
'continuous_features': ['x1', 'x2', 'x3'],
'outcome_name': 'y'
}
d = DataLoader(params)
temp = d.train_df.copy()
temp.drop('y', axis=1, inplace=True)
elif dataset_name == 'adult':
dataset = load_adult_income_dataset()
params = {
'dataframe': dataset.copy(),
'continuous_features': ['age', 'hours_per_week'],
'outcome_name': 'income'
}
d = DataLoader(params)
temp = d.train_df.copy()
temp.drop('income', axis=1, inplace=True)
elif dataset_name == 'sangiovese':