def main():
seed = 1
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
cuda = True
train, test = prep_data(pca=True, pca_scale=True, inputation=True,
strategy='median', remove_low_variance=False)
columns_to_drop = ['Response']
test = test.drop(columns_to_drop, axis=1)
test_data = CustData(test)
test_loader = DataLoader(test_data, batch_size=len(test_data), num_workers=6,
worker_init_fn=worker_init_fn)
#with open('../5Others/config.txt', 'rb') as fp:
with open('../4TrainingWeights/tuning_2/num_node_layer_num/num_nodes_256_layer_number_2/2019-01-15_16_49_59.215415\\2019-01-15_18_54_05.071129.txt', 'rb') as fp:
#with open('../4TrainingWeights/2019-01-06_09_45_38.867660/2019-01-06_11_28_41.798519.txt', 'rb') as fp:
param = json.load(fp)
input_dim = len(test.columns)
model = Ordinal_regression(create_module, config=param, input_dim=input_dim)
#state_dic = torch.load('../4TrainingWeights/2019-01-06_20_43_56.362198/2019-01-06_21_04_05.995207.pth')
#model.load_state_dict(state_dic)
if cuda:
model.cuda()
model.eval()
final_prediction = prediction(model, test_loader, cuda)
# test_loader = DataLoader(test_data, batch_size=len(test_data), num_workers=6,
# worker_init_fn=worker_init_fn)
# y, final_prediction = prediction(model, test_loader, cuda, label= True)
# len(y[abs(y-final_prediction) <= 2]) / len(y)
# y = list(map(int, y))
# accuracy_score(y, final_prediction)
submission = pd.read_csv('../1TestData/sample_submission.csv', index_col=0)
submission['Response'] = final_prediction.astype('int32')
submission.to_csv('submit.csv')
def main():
THRESHOLD = 0.0
print(os.getcwd())
dream_model = load_dream_model()
# Image preprocessing
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
test_files, test_labels = utils.prep_data(cfg.test_path)
# test_files, test_labels = load_data_set_test(cfg.test_path)
test_labels = list(map(int, test_labels))
test_res = []
for im_name in test_files:
# Prepare an image
image = load_image(im_name, transform)
image_tensor = image.to(device)
# test image
predicted_label = dream_model(image_tensor)
m = nn.Softmax()
norm_labels = m(predicted_label)
norm_labels_np = norm_labels.cpu().data.numpy(
) # convert to numpy array
predicted_label = np.argmax(norm_labels_np)
if norm_labels_np[0][predicted_label] >= THRESHOLD:
test_res.append(predicted_label)
else:
test_res.append(-1)
print("Image Name: " + im_name + " Test Res: " +
cfg.CLASS_LIST[predicted_label] + " Confidence: " +
str(norm_labels_np[0][predicted_label]))
# calculate metrics
score = metrics.accuracy_score(test_labels, test_res)
cls_report = metrics.classification_report(test_labels, test_res)
conf_mat = metrics.confusion_matrix(test_labels, test_res)
print(("Accuracy = " + str(score)))
print(cls_report)
print(conf_mat)
def post(self):
args = self.reqparse.parse_args()
rabbit = utils.RabbitClient(queue=PUBLISH, host='rabbitmq')
logger.info('Received data...')
data = utils.prep_data(args['data'])
key = hashlib.sha1(''.join(data['sents'])).hexdigest()
data['pipeline_key'] = key
logger.info('Sending downstream with key {}...'.format(key))
rabbit.send(data, PUBLISH)
logging.info('Sent {}'.format(key))
return key
def main():
THRESHOLD = 0.0
# Image preprocessing
transform = transforms.Compose([
transforms.Resize((cfg.RESIZE, cfg.RESIZE)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
test_files, test_labels = utils.prep_data(cfg.test_path)
# test_files, test_labels = load_data_set_test(test_path)
test_labels = list(map(int, test_labels))
for i in range(399, 4, -5):
test_res = []
model_name = "art_model-{}.pkl".format(i)
dream_model = load_dream_model(model_name)
print "Model Name: " + model_name
for im_name in test_files:
# Prepare an image
image = load_image(im_name, transform)
image_tensor = image.to(device)
# test image
predicted_label = dream_model(
image_tensor) # convert to numpy array
m = nn.Softmax()
norm_labels = m(predicted_label)
norm_labels_np = norm_labels.cpu().data.numpy()
predicted_label = np.argmax(norm_labels_np)
if norm_labels_np[0][predicted_label] >= THRESHOLD:
test_res.append(predicted_label)
else:
test_res.append(-1)
# print "Image Name: " + im_name + " Test Res: " + cfg.CLASS_LIST[predicted_label] + " Confidence: " + str(norm_labels_np[0][predicted_label])
score = metrics.accuracy_score(test_labels, test_res)
cls_report = metrics.classification_report(test_labels, test_res)
conf_mat = metrics.confusion_matrix(test_labels, test_res)
print "Accuracy = " + str(score)
print cls_report
print conf_mat
def post(self):
args = self.reqparse.parse_args()
rabbit = utils.RabbitClient(queue=PUBLISH, host='rabbitmq')
logger.info('Received data...')
data = args['data']
data = utils.prep_data(data)
pipeline_key = str(uuid.uuid4())
data['pipeline_key'] = pipeline_key
logger.info('Sending to the downstream with key {}...'.format(pipeline_key))
rabbit.send(data, PUBLISH)
logging.info('Sent {}'.format(pipeline_key))
return pipeline_key
def init_from_spec(data_path, model_spec_path, **kwargs):
# load data
if 'npy' in data_path:
data = np.load(data_path).astype(DTYPE)
else:
raise NotImplementedError(
'Only datasets with the .npy extension are supported.')
# Occasionally JSON files accidentally
# get troublesome whitespace added which
# can break the JSON loading
with open(model_spec_path, 'r') as spec_src:
spec_as_str = spec_src.read().strip()
spec = json.loads(spec_as_str)
# If any special arguments have been passed,
# use them to update the specification for the
# model fitting.
for key, value in kwargs.items():
if key in spec.keys():
spec[key] = value
spec.update({'image_dims': data.shape[1:], 'channels': data.shape[-1]})
describe_spec(spec)
dataset = prep_data(data, spec)
model = init_model(spec, dataset)
#try:
# model.train()
#except KeyboardInterrupt:
# return model
#toggle_training_layers(model)
#if save:
# model.save(SAVED_MODELS_DIR + spec['name'])
return model
worker_init_fn=worker_init_fn)
# Start training
train(model, optimizer, train_loader, test_loader, scorer_list, cuda,
optimizer_name.lower())
if __name__ == '__main__':
for seed in range(1, 2):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
logging.basicConfig(level=logging.DEBUG,
format="[%(asctime)s %(filename)s] %(message)s")
cuda = True
train_set, test_set = prep_data()
columns_to_drop = ['Id', 'Response']
x = train_set.drop(columns_to_drop, axis=1)
y = train_set.Response - 1
input_dim = len(x.columns)
# with open('../5Others/config.txt', 'w') as fp:
# fp.write(json.dumps(param, indent=4))
with open('../5Others/2019-01-07_00_57_25.027646.txt', 'rb') as fp:
# with open('../5Others/config.txt', 'rb') as fp:
# with open('../4TrainingWeights/2019-01-06_00_40_22.960024/2019-01-06_02_39_53.326797.txt', 'rb') as fp:
param = json.load(fp)
model = Ordinal_regression(create_module,
config=param,
input_dim=input_dim)
# if param['pretrain_snapshot']:
# state_dic = torch.load(param['pretrain_snapshot'])
min_num = -128
n_dim = 160
# just load train data
df_train = pd.read_csv(utils.TRAIN_META)
# set index, it makes the data access much faster
df_train = df_train.set_index(['id_measurement', 'phase'])
# this code is very simple, divide the total size of the df_train into two sets and process it
X = []
y = []
total_size = len(df_train)
#for ini, end in [(0, int(total_size/2)), (int(total_size/2), total_size)]:
#X_temp, y_temp = utils.prep_data(df_train, 0, tatal_size, n_dim, min_num, max_num, sample_size)
X, y = utils.prep_data(df_train, 0, total_size, n_dim, min_num, max_num,
sample_size)
# X.append(X_temp)
# y.append(y_temp)
# X = np.concatenate(X)
# y = np.concatenate(y)
print(X.shape, y.shape)
np.save("X.npy", X)
np.save("y.npy", y)
# Here is where the training happens
# First, create a set of indexes of the 5 folds
splits = list(
StratifiedKFold(n_splits=N_SPLITS, shuffle=True,
def main():
parser = argparse.ArgumentParser(
description=
'Train a new network on a dataset and save the model as a checkpoint')
parser.add_argument('data_dir',
metavar='path/to/dataset',
type=str,
nargs=1,
help='path to a data directory')
parser.add_argument(
'--save_dir',
metavar='path/to/save_dir',
type=str,
nargs='?',
help='path to a directory in which to save a checkpoint')
parser.add_argument('--learning_rate',
metavar='learning rate',
type=float,
nargs='?',
default=0.002,
help='learning rate value for model training')
parser.add_argument('--hidden_units',
metavar='hidden units',
type=int,
nargs='?',
default=512,
help='number of hidden units for model classifier')
parser.add_argument('--epochs',
metavar='epochs',
type=int,
nargs='?',
default=5,
help='number of epochs for model training')
parser.add_argument(
'--arch',
metavar='model name',
type=str,
nargs='?',
default='densenet161',
help='name of transfer model (e.g., resnet18 or densenet161)')
parser.add_argument('--gpu',
action='store_true',
help='use GPU for model training (recommended)')
args = parser.parse_args()
data_dir = args.data_dir[0]
save_dir = args.save_dir
learning_rate = args.learning_rate
hidden_units = args.hidden_units
epochs = args.epochs
model_name = args.arch
use_gpu = args.gpu
print('Using the following hyperparameters for training')
print(f' Learning rate: {learning_rate}')
print(f' Hidden units: {hidden_units}')
print(f' Epochs: {epochs}')
print(f'Transfer model name: {model_name}')
if use_gpu and not cuda.is_available():
print('Error: GPU not available. Try again without the --gpu flag')
exit(1)
if use_gpu:
print('Training on GPU...')
else:
print('Training on CPU...')
print(
'Warning: training on CPU could take a LONG time. Consider using --gpu flag.'
)
print('')
if model_name not in ALLOWED_ARCHS:
print(
f'Error: Model architecture {model_name} is not currently supported.'
)
print('Please try one of the following:')
for a in ALLOWED_ARCHS:
print(f' {a}')
exit(1)
dataloaders, image_datasets = prep_data(data_dir)
model = build_model_from_pretrained(model_name, hidden_units,
image_datasets['test'].class_to_idx)
# print(model.classifier)
with active_session():
trained, optimizer = train(model, dataloaders, learning_rate, epochs,
use_gpu)
if save_dir:
save_checkpoint(trained, epochs, optimizer, model_name, learning_rate,
save_dir)
f.close()
ind = d.field('type') == 6
d = d[ind]
cmd = np.zeros((len(d.field(0)), 2))
g = d.field('psfmag_g') - d.field('extinction_g')
r = d.field('psfmag_r') - d.field('extinction_r')
cmd[:, 0] = g - r
cmd[:, 1] = r
f = open(os.environ['sgdata'] + 'xdmodel_dr10_30000_32_128.pkl')
model = cPickle.load(f)
f.close()
from utils import prep_data
X, Xcov = prep_data(d, ['psf_minus_model', 'colors'])
post_alpha, post_mu, post_V = model.posterior(X, Xcov)
posts = np.zeros_like(X)
for i in range(X.shape[0]):
posts[i] = np.median(model.sample(post_alpha[i], post_mu[i], post_V[i],
size=1000), axis=0)
if post_alpha[i].sum() != post_alpha[i].sum():
posts[i] = np.zeros_like(X[0])
r = posts[:, 0]
gmr = posts[:, 7]
pcmd = np.zeros((X.shape[0], 2))
pcmd[:, 0] = posts[:, 7]
pcmd[:, 1] = posts[:, 0]
cmd_plot(cmd, pcmd, ['DR10', 'XD Post.'], os.environ['xdplots'] + 'foo.png')
default=False,
action='store_true')
parser.add_argument("--score_as_onnx",
dest="score_as_onnx",
help="Score as ONNX",
default=False,
action='store_true')
args = parser.parse_args()
print("Arguments:")
for arg in vars(args):
print(f" {arg}: {getattr(args, arg)}")
print("\n**** Data")
loader = utils.get_data(False, 10000)
print("loader.type:", type(loader))
data = utils.prep_data(loader)
print("data.type:", type(data))
print("data.shape:", data.shape)
print("\n**** pytorch.load_model")
model_uri = f"runs:/{args.run_id}/pytorch-model"
model = mlflow.pytorch.load_model(model_uri)
print("model.type:", type(model))
outputs = model(data)
print("outputs.type:", type(outputs))
outputs = outputs.detach().numpy()
print("outputs.shape:", outputs.shape)
utils.display_predictions(outputs)
def main(k_shots, num_tasks, adam_betas, gen_lr, dis_lr, total_epochs, model_folder_path):
torch.manual_seed(1)
# Initialize generator and discriminator
batch_size = 1
generator = Generator(batch_size=batch_size)
discriminator = Discriminator()
generator.cuda()
discriminator.cuda()
# Training the Model
# optimizer
optimizer_G = optim.Adam (generator.parameters(), lr= gen_lr, betas=adam_betas)
optimizer_D = optim.Adam (discriminator.parameters(), lr= dis_lr, betas=adam_betas)
# define dataloader
tf = transforms.Compose([transforms.Resize((256,256)),transforms.ToTensor()])
create_folder(model_folder_path)
generator_path = os.path.join(model_folder_path, str.format("Generator_previous.pt"))
discriminator_path = os.path.join(model_folder_path, str.format("Discriminator_previous.pt"))
torch.save(generator.state_dict(), generator_path)
torch.save(discriminator.state_dict(), discriminator_path)
previous_generator = generator_path
previous_discriminator = discriminator_path
frame_path = '/mnt/creeper/grad/luy2/Meta-Learning/data/shanghaitech-5tasks/training/frames/'
# Set Up Training Loop
for epoch in range(total_epochs):
train_path_list = createEpochData(frame_path, num_tasks, k_shots)
train_dataloader = Load_Dataloader(train_path_list, tf, batch_size)
for _, epoch_of_tasks in enumerate(train_dataloader):
# Create folder for saving results
epoch_results = 'results'.format(epoch+1)
create_folder(epoch_results)
gen_epoch_grads = []
dis_epoch_grads = []
print("Epoch: ", epoch+1)
# Meta-Training
for tidx, task in enumerate(epoch_of_tasks):
# Copy rGAN
print ('\n Meta Training \n')
print("Memory Allocated: ",torch.cuda.memory_allocated()/1e9)
generator.load_state_dict(torch.load(previous_generator))
discriminator.load_state_dict(torch.load(previous_discriminator))
inner_optimizer_G = optim.Adam(generator.parameters(), lr=1e-4)
inner_optimizer_D = optim.Adam(discriminator.parameters(), lr=1e-4)
print("Task: ", tidx)
for kidx, frame_sequence in enumerate(task[:k_shots]):
print('k-Shot Training: ', kidx)
# Configure input
img = frame_sequence[0]
gt = frame_sequence[1]
img, gt, valid, fake = prep_data(img, gt)
# Train Generator
inner_optimizer_G.zero_grad()
imgs, g_loss, recon_batch, loss, msssim = overall_generator_pass(generator, discriminator, img, gt, valid)
img_path = os.path.join(epoch_results,'{}-fig-train{}.png'.format(tidx+1, kidx+1))
imsave(img_path , imgs)
g_loss.backward()
inner_optimizer_G.step()
# Train Discriminator
inner_optimizer_D.zero_grad()
# Measure discriminator's ability to classify real from generated samples
d_loss = overall_discriminator_pass(discriminator, recon_batch, gt, valid, fake)
d_loss.backward()
inner_optimizer_D.step()
print ('Epoch [{}/{}], Step [{}/{}], Reconstruction_Loss: {:.4f}, G_Loss: {:.4f}, D_loss: {:.4f}, msssim:{:.4f} '.format(epoch+1, total_epochs, tidx+1, 5, loss.item(), g_loss, d_loss, msssim))
# Meta-Validation
print ('\n Meta Validation \n')
# Store Loss Values
gen_validation_loss_store = 0.0
dis_validation_loss_store = 0.0
gen_validation_loss = 0.0
dis_validation_loss = 0.0
dummy_frame_sequence = []
# forward pass
for vidx, val_frame_sequence in enumerate(task[-k_shots:]):
print(vidx)
if vidx == 0:
dummy_frame_sequence = val_frame_sequence
img = val_frame_sequence[0]
gt = val_frame_sequence[1]
img, gt, valid, fake = prep_data(img, gt)
# k-Validation Generator
imgs, g_loss, recon_batch, loss, msssim = overall_generator_pass(generator, discriminator, img, gt, valid)
img_path = os.path.join(epoch_results,'{}-fig-val{}.png'.format(tidx+1, vidx+1))
imsave(img_path , imgs)
# k-Validation Discriminator
d_loss = overall_discriminator_pass(discriminator, recon_batch, gt, valid, fake)
# Store Loss Items to reduce memory usage
gen_validation_loss_store += g_loss.item()
dis_validation_loss_store += d_loss.item()
if (vidx == k_shots-1):
# Store the loss
gen_validation_loss = g_loss
dis_validation_loss = d_loss
gen_validation_loss.data = torch.FloatTensor([gen_validation_loss_store/k_shots]).cuda()
dis_validation_loss.data = torch.FloatTensor([dis_validation_loss_store/k_shots]).cuda()
print("Generator Validation Loss: ", gen_validation_loss_store)
print("Discriminator Validation Loss: ", dis_validation_loss_store)
print ('Epoch [{}/{}], Step [{}/{}], G_Loss: {:.4f}, D_loss: {:.4f}'.format(epoch+1, total_epochs, tidx+1, 5, loss.item(), g_loss, d_loss))
print("Memory Allocated: ",torch.cuda.memory_allocated()/1e9)
# Compute Validation Grad
print("Memory Allocated: ",torch.cuda.memory_allocated()/1e9)
generator.load_state_dict(torch.load(previous_generator))
discriminator.load_state_dict(torch.load(previous_discriminator))
gen_grads = torch.autograd.grad(gen_validation_loss, generator.parameters())
dis_grads = torch.autograd.grad(dis_validation_loss, discriminator.parameters())
gen_meta_grads = {name:g for ((name, _), g) in zip(generator.named_parameters(), gen_grads)}
dis_meta_grads = {name:g for ((name, _), g) in zip(discriminator.named_parameters(), dis_grads)}
gen_epoch_grads.append(gen_meta_grads)
dis_epoch_grads.append(dis_meta_grads)
# Meta Update
print('\n Meta update \n')
generator.load_state_dict(torch.load(previous_generator))
discriminator.load_state_dict(torch.load(previous_discriminator))
# Configure input
img = dummy_frame_sequence[0]
gt = dummy_frame_sequence[1]
img, gt, valid, fake = prep_data(img, gt)
# Dummy Forward Pass
imgs, g_loss, recon_batch, loss, msssim = overall_generator_pass(generator, discriminator, img, gt, valid)
d_loss = overall_discriminator_pass(discriminator, recon_batch, gt, valid, fake)
# Unpack the list of grad dicts
gen_gradients = {k: sum(d[k] for d in gen_epoch_grads) for k in gen_epoch_grads[0].keys()}
dis_gradients = {k: sum(d[k] for d in dis_epoch_grads) for k in dis_epoch_grads[0].keys()}
meta_update_model(generator, optimizer_G, g_loss, gen_gradients)
meta_update_model(discriminator, optimizer_D, d_loss, dis_gradients)
# Save the Model
torch.save(generator.state_dict(), previous_generator)
torch.save(discriminator.state_dict(), previous_discriminator)
if (epoch % 10 == 0):
gen_path = os.path.join(model_folder_path, str.format("Generator_{}.pt", epoch+1))
dis_path = os.path.join(model_folder_path, str.format("Discriminator_{}.pt", epoch+1))
torch.save(generator.state_dict(), gen_path)
torch.save(discriminator.state_dict(), dis_path)
print("Training Complete")
gen_path = os.path.join(model_folder_path, str.format("Generator_Final.pt"))
dis_path = os.path.join(model_folder_path, str.format("Discriminator_Final.pt"))
torch.save(generator.state_dict(), gen_path)
torch.save(discriminator.state_dict(), dis_path)
f.close()
ind = d.field('type') == 6
d = d[ind]
cmd = np.zeros((len(d.field(0)), 2))
g = d.field('psfmag_g') - d.field('extinction_g')
r = d.field('psfmag_r') - d.field('extinction_r')
cmd[:, 0] = g - r
cmd[:, 1] = r
f = open(os.environ['sgdata'] + 'xdmodel_dr10_30000_32_128.pkl')
model = cPickle.load(f)
f.close()
from utils import prep_data
X, Xcov = prep_data(d, ['psf_minus_model', 'colors'])
post_alpha, post_mu, post_V = model.posterior(X, Xcov)
posts = np.zeros_like(X)
for i in range(X.shape[0]):
posts[i] = np.median(model.sample(post_alpha[i],
post_mu[i],
post_V[i],
size=1000),
axis=0)
if post_alpha[i].sum() != post_alpha[i].sum():
posts[i] = np.zeros_like(X[0])
r = posts[:, 0]
gmr = posts[:, 7]
pcmd = np.zeros((X.shape[0], 2))
pcmd[:, 0] = posts[:, 7]
import warnings
warnings.filterwarnings("ignore")
if __name__ == '__main__':
torch.backends.cudnn.enabled = False
torch.backends.cudnn.benchmark = False
for seed in range(1, 2):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
logging.basicConfig(level=logging.DEBUG,
format="[%(asctime)s %(filename)s] %(message)s")
cuda = True
train, test = prep_data(pca=True, pca_scale=True, inputation=True,
strategy='median', remove_low_variance=False)
columns_to_drop = ['Response']
x = train.drop(columns_to_drop, axis=1)
y = train.Response-1
input_dim = len(x.columns)
# with open('../5Others/config.txt', 'w') as fp:
# fp.write(json.dumps(param, indent=4))
# with open('../5Others/2019-01-07_00_57_25.027646.txt', 'rb') as fp:
# with open('../5Others/config.txt', 'rb') as fp:
with open('../5Others/tuning.txt', 'rb') as fp:
# with open( '../4TrainingWeights/2019-01-06_00_40_22.960024/2019-01-06_02_39_53.326797.txt', 'rb') as fp:
param = json.load(fp)
w_dir = param['working_dir']
num_nodes_list = [4, 8, 16, 32, 64, 128, 256]
layer_numbers = [1, 2, 3, 4]
for num_nodes in num_nodes_list:
from xgboost import XGBClassifier
import pandas as pd
import numpy as np
import xgboost as xgb
from scipy.optimize import fmin_powell
from utils import prep_data, quadratic_weighted_kappa, train_offset, digit,\
feature_importance_plot
from evaluate import y_transform, cross_validation, Within_n_rank
import random
# =============================================================================
# data
# =============================================================================
train, test = prep_data()
columns_to_drop = ['Response']
x = train.drop(columns_to_drop, axis=1)
y = train.Response - 1
test_x = test.drop(columns_to_drop, axis=1)
batch_size=opts.batchSize,
shuffle=False)
print 'Data loader ready.'
#Load model
gen = GEN(imSize=IM_SIZE, nz=opts.nz, fSize=opts.fSize)
if gen.useCUDA:
torch.cuda.set_device(opts.gpuNo)
gen.cuda()
gen.load_params(opts.exDir, gpuNo=opts.gpuNo)
gen.eval()
print 'params loaded'
# Get the data:
data = iter(testLoader).next()
x, y = prep_data(data, useCUDA=gen.useCUDA)
# Get men with glasses
idx_men_w_glasses = np.argwhere(
torch.prod(y.data == torch.Tensor([1, 1]).cuda(), dim=1))[0]
img_men_w_glasses = x[idx_men_w_glasses[:opts.numSamples]]
save_image(img_men_w_glasses.data,
join(
exDir, 'img_' + opts.labels[0] + '_w_' + opts.labels[1] +
'_original.png'),
nrow=10,
normalize=True)
# Get men without glasses
idx_men_wout_glasses = np.argwhere(
torch.prod(y.data == torch.Tensor([1, 0]).cuda(), dim=1))[0]
def main():
# Create model directory
if not os.path.exists(cfg.PYTORCH_MODELS):
os.makedirs(cfg.PYTORCH_MODELS)
# open loss info
today = datetime.datetime.today()
rand = random.randint(0,1000000000)
loss_info = open(cfg.PYTORCH_MODELS + 'loss_' + str(rand) + '.txt', 'w')
# Image preprocessing
transform = transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomCrop(cfg.PATCH_SIZE),
# n_random_crops.NRandomCrop(cfg.PATCH_SIZE, 10),
# transforms.Resize((cfg.RESIZE, cfg.RESIZE)), # avgpool u 3x3 yapinca 96 olmali.
transforms.Pad((math.ceil((224-cfg.PATCH_SIZE)/2.0), math.floor((224-cfg.PATCH_SIZE)/2.0)), padding_mode="reflect"),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Build the models
art_model = ArtModel()
# art_model = model_custom.resnet50(pretrained=True, num_classes=cfg.CLASS_COUNT)
if torch.cuda.is_available():
art_model.cuda()
if cfg.LOAD_TRAINED_MODEL:
art_model.load_state_dict(torch.load(cfg.PYTORCH_MODELS + cfg.MODEL_NAME))
# load dataset and labels
# image_path_list, labels_list = load_data_set()
image_path_list, labels_list = utils.prep_data(cfg.train_path)
# append main path
# image_path_list = [image_main_path + imm for imm in image_path_list]
# Build data loader
data_loader = get_loader(image_path_list, labels_list, cfg.BATCH_SIZE, shuffle=True, transform=transform, num_workers=cfg.NUM_WORKERS)
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam([x for x in list(art_model.parameters()) if x.requires_grad], lr=cfg.LEARNING_RATE)
# Train the Models
total_step = len(data_loader)
for epoch in range(1, cfg.EPOCH_COUNT):
for i, (images, label) in enumerate(data_loader):
# Set mini-batch dataset
images = to_var(images, volatile=True)
# Set mini-batch ground truth
label = to_var(label, volatile=False)
# Forward, Backward and Optimize
art_model.zero_grad()
# feed images to CNN model
predicted_label = art_model(images)
loss = criterion(predicted_label, label)
loss.backward()
optimizer.step()
# Print log info
if i % cfg.LOG_STEP == 0:
print(('Epoch [%d/%d], Step [%d/%d], Loss: %.7f'
% (epoch, cfg.EPOCH_COUNT, i, total_step, loss)))
loss_info.write('Epoch [%d/%d], Step [%d/%d], Loss: %.7f\n'
% (epoch, cfg.EPOCH_COUNT, i, total_step, loss))
# Save the models
if (epoch + 1) % cfg.SAVE_PERIOD_IN_EPOCHS == 0:
torch.save({
'epoch': epoch,
'model_state_dict': art_model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
}, os.path.join(cfg.PYTORCH_MODELS, 'art_model-%d.pkl' % epoch))
from evaluate import y_transform, cross_validation, Within_n_rank
import mord
import random
import warnings
import logging
import json
from catboost import CatBoostRegressor, CatBoostClassifier
warnings.filterwarnings("ignore")
logging.basicConfig(level=logging.DEBUG,
format="[%(asctime)s %(filename)s] %(message)s")
# =============================================================================
# data
# =============================================================================
train, test, target, validation, valid_target,\
categorical_variables = prep_data(160)
#train, test, target = prep_data(pca=True, pca_scale=True,
# remove_low_variance=False)
# =============================================================================
# Pure regression based
# =============================================================================
# lasso
lasso = Lasso(alpha=0.001, max_iter=5000)
lasso.fit(train, target)
train_y_pred = lasso.predict(train)
test_pred = lasso.predict(test)
# drop non important features (optional)
lasso_feature_importance = pd.Series(data=lasso.coef_,
index=train.columns,
