def init_comet(params, trainer):
if params['comet_key']:
from comet_ml import Experiment
experiment = Experiment(api_key=params['comet_key'], project_name=params['comet_project_name'], log_code=False)
hyperparams = {
name: str(params[name]) for name in params
}
experiment.log_multiple_params(hyperparams)
trainer.register_plugin(CometPlugin(
experiment, [
'G_loss.epoch_mean',
'D_loss.epoch_mean',
'D_real.epoch_mean',
'D_fake.epoch_mean',
'sec.kimg',
'sec.tick',
'kimg_stat'
] + (['depth', 'alpha'] if params['progressive_growing'] else [])
))
else:
print('Comet_ml logging disabled.')
def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(self.config.checkpoint_dir, '%s-{epoch:03d}-{val_nme:.5f}.hdf5' % self.config.exp_name),
monitor=self.config.checkpoint_monitor,
mode=self.config.checkpoint_mode,
save_best_only=self.config.checkpoint_save_best_only,
save_weights_only=self.config.checkpoint_save_weights_only,
verbose=self.config.checkpoint_verbose,
)
)
self.callbacks.append(
TensorBoard(
log_dir=self.config.tensorboard_log_dir,
write_graph=self.config.tensorboard_write_graph,
)
)
# self.callbacks.append(
# LearningRateScheduler(self.lr_scheduler)
# )
if hasattr(self.config,"comet_api_key"):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name)
experiment.disable_mp()
experiment.log_multiple_params(self.config)
self.callbacks.append(experiment.get_keras_callback())
from comet_ml import Experiment
from torchvision.io.video import read_video, write_video
import torch
from args_util import meow_parse
from data_flow import get_predict_video_dataloader
from models import create_model
import os
from visualize_util import save_density_map_normalize, save_density_map
if __name__ == "__main__":
COMET_ML_API = "S3mM1eMq6NumMxk2QJAXASkUM"
PROJECT_NAME = "crowd-counting-debug"
experiment = Experiment(project_name=PROJECT_NAME, api_key=COMET_ML_API)
args = meow_parse()
video_path = args.input
OUTPUT_FOLDER = args.output
MODEL_PATH = args.load_model
model_name = args.model
NAME = args.task_id
experiment.set_name(args.task_id)
experiment.set_cmd_args()
experiment.log_text(args.note)
print(args)
n_thread = int(os.environ['OMP_NUM_THREADS'])
torch.set_num_threads(n_thread) # 4 thread
print("n_thread ", n_thread)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 18 15:07:55 2019
@author: gabriel
"""
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="iter_plotting", workspace="gdreiman1", disabled = False
)
exp.log_code = True
exp.log_other('Hypothesis','''These are my plots from my random gcnn modification ''')
import pickle
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
data_dir = '/home/gabriel/Dropbox/UCL/Thesis/Data'
first8 = 'gcnn_mod0.pkl'
second8 = 'gcnn_mod1.pkl'
third8 = 'gcnn_mod2.pkl'
save_path = os.path.join(data_dir,first8)
pickle_off = open(save_path,'rb')
first8=pickle.load(pickle_off)
pickle_off.close()
save_path = os.path.join(data_dir,second8)
pickle_off = open(save_path,'rb')
second8=pickle.load(pickle_off)
def log_validation_proposals(experiment: Experiment,
proposals: Sequence[ImageAnnotation]):
experiment.log_asset_data(
[annotation.to_json() for annotation in proposals],
name="datatap/validation/proposals.json")
def main():
"""Main training program."""
print('Pretrain BERT model')
# Disable CuDNN.
torch.backends.cudnn.enabled = False
# Timer.
timers = Timers()
# Arguments.
args = get_args()
experiment = Experiment(api_key='1jl4lQOnJsVdZR6oekS6WO5FI',
project_name=args.model_type,
auto_param_logging=False, auto_metric_logging=False,
disabled=(not args.track_results))
experiment.log_parameters(vars(args))
metrics = {}
# Pytorch distributed.
initialize_distributed(args)
# Random seeds for reproducability.
set_random_seed(args.seed)
# Data stuff.
data_config = configure_data()
data_config.set_defaults(data_set_type='BERT', transpose=False)
(train_data, val_data, test_data), tokenizer = data_config.apply(args)
args.data_size = tokenizer.num_tokens
# Model, optimizer, and learning rate.
model, optimizer, lr_scheduler, criterion = setup_model_and_optimizer(
args, tokenizer)
#model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
timers("total time").start()
epoch = 0
# At any point you can hit Ctrl + C to break out of training early.
try:
start_epoch = 1
best_val_loss = float('inf')
# Resume data loader if necessary.
if args.resume_dataloader:
start_epoch = args.epoch
next_stage = None
current_stage = None
if args.continual_learning:
next_stage = set_up_stages(args)
current_stage = next_stage()
if args.resume_dataloader:
num_tokens = args.epoch * args.train_tokens
# Get to the right stage
while num_tokens > sum(current_stage.values()):
num_tokens -= sum(current_stage.values())
ns = next_stage()
for m in ns:
current_stage[m] = ns[m]
# Get to right part of stage
stage_tokens = sum(current_stage.values())
stage_ratios = {k: v / float(stage_tokens) for k, v in current_stage.items()}
for k in current_stage:
current_stage[k] -= num_tokens * stage_ratios[k]
# Train for required epochs
for epoch in range(start_epoch, args.epochs+1):
if args.shuffle:
train_data.batch_sampler.sampler.set_epoch(epoch+args.seed)
timers('epoch time').start()
# Train
train_epoch(epoch, model, optimizer, train_data, lr_scheduler, criterion, timers, experiment, metrics, args,
current_stage=current_stage, next_stage=next_stage)
elapsed_time = timers('epoch time').elapsed()
if args.save:
ck_path = 'ck/model_{}.pt'.format(epoch)
print('saving ck model to:',
os.path.join(args.save, ck_path))
save_checkpoint(ck_path, epoch+1, model, optimizer, lr_scheduler, args)
# Validate
val_loss = evaluate(epoch, val_data, model, criterion, elapsed_time, args)
if val_loss < best_val_loss:
best_val_loss = val_loss
if args.save:
best_path = 'best/model.pt'
print('saving best model to:',
os.path.join(args.save, best_path))
save_checkpoint(best_path, epoch+1, model, optimizer, lr_scheduler, args)
except KeyboardInterrupt:
print('-' * 100)
print('Exiting from training early')
exit()
if test_data is not None:
# Run on test data.
print('entering test')
elapsed_time = timers("total time").elapsed()
evaluate(epoch, test_data, model, criterion, elapsed_time, args, test=True)
torch.manual_seed(42)
epochs = 3000
model = MolecularVAE(i=max_len, c=len(vocab)).cuda()
model = nn.DataParallel(model)
optimizer = optim.SGD(model.parameters(), lr=3.0e-4 * 4, momentum=0.85)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
factor=0.8,
patience=10,
verbose=True,
threshold=1e-3)
log_interval = 100
experirment = Experiment(project_name='pytorch', auto_metric_logging=False)
def train(epoch):
with experirment.train():
model.train()
train_loss = 0
for batch_idx, (data, ohe) in enumerate(train_loader):
data = data.cuda()
ohe = ohe.cuda()
optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
loss = loss_function(recon_batch, ohe, mu, logvar)
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), 5.0)
def main():
args = parse_args()
if args is None:
exit()
setup_logging(args)
gan = BigGAN_128(args)
if args.use_tpu:
cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
tpu=args.tpu_name, zone=args.tpu_zone)
master = cluster_resolver.get_master()
else:
master = ''
tpu_run_config = tf.contrib.tpu.RunConfig(
master=master,
evaluation_master=master,
model_dir=model_dir(args),
session_config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False),
tpu_config=tf.contrib.tpu.TPUConfig(args.steps_per_loop,
args.num_shards),
)
tpu_estimator = tf.contrib.tpu.TPUEstimator(
model_fn=lambda features, labels, mode, params: gan.tpu_model_fn(
features, labels, mode, params),
config=tpu_run_config,
use_tpu=args.use_tpu,
train_batch_size=args._batch_size,
eval_batch_size=args._batch_size,
predict_batch_size=args._batch_size,
params=vars(args),
)
total_steps = 0
if args.use_comet:
experiment = Experiment(api_key="bRptcjkrwOuba29GcyiNaGDbj",
project_name="BigGAN",
workspace="davidhughhenrymack")
experiment.log_parameters(vars(args))
experiment.add_tags(args.tag)
experiment.set_name(model_name(args))
else:
experiment = None
prefetch_inception_model()
with tf.gfile.Open(
os.path.join(suffixed_folder(args, args.result_dir), "eval.txt"),
"a") as eval_file:
for epoch in range(args.epochs):
logger.info(f"Training epoch {epoch}")
tpu_estimator.train(input_fn=train_input_fn,
steps=args.train_steps)
total_steps += args.train_steps
logger.info(f"Evaluate {epoch}")
evaluation = tpu_estimator.evaluate(input_fn=eval_input_fn,
steps=args.eval_steps)
if args.use_comet:
experiment.set_step(total_steps)
experiment.log_metrics(evaluation)
logger.info(evaluation)
save_evaluation(args, eval_file, evaluation, epoch, total_steps)
logger.info(f"Generate predictions {epoch}")
predictions = tpu_estimator.predict(input_fn=predict_input_fn)
logger.info(f"Save predictions")
save_predictions(args, suffixed_folder(args,
args.result_dir), eval_file,
predictions, epoch, total_steps, experiment)
from comet_ml import Experiment
from train import train_cifar10
from models import get_model, get_model_same, get_model_3block, get_model_4block, get_model_5block
model_fncs = [
get_model, get_model_same, get_model_3block, get_model_4block,
get_model_5block
]
experiment_names = ['base', 'same', '3block', '4block', '5block']
for name, model_fnc in zip(experiment_names, model_fncs):
experiment = Experiment(api_key="cgss7piePhyFPXRw1J2uUEjkQ",
project_name="cifar10-09-base-depth",
workspace="matech96")
experiment.set_name(name)
model = model_fnc()
train_cifar10(batch_size=64,
learning_rate=0.001,
epochs=1000,
experiment=experiment,
model=model)
from comet_ml import Experiment
experiment = Experiment(api_key="oda8KKpxlDgWmJG5KsYrrhmIV",
project_name="consensusnet")
import numpy as np
from keras.models import Model
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Input
from keras.layers import Conv1D, MaxPooling1D, SeparableConv1D
from keras.callbacks import LearningRateScheduler, EarlyStopping
from keras.regularizers import l1, l2, l1_l2
import sys
module_path = '/home/diplomski-rad/consensus-net/src/python/utils/'
if module_path not in sys.path:
print('Adding utils module.')
sys.path.append(module_path)
from args_parsers import parse_train_args
def main(args):
args = parse_train_args(args)
X_train = np.load(args.X_train)
X_valid = np.load(args.X_validate)
y_train = np.load(args.y_train)
y_valid = np.load(args.y_validate)
model_save_path = args.model_save_path
import icecream as ic
from IPython.display import display
from torch.utils.data import DataLoader, Dataset, random_split
import torch
import random
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import init
from pydicom import dcmread
torch.cuda._initialized = True
# %%
experiment = Experiment(api_key="xleFjfKO3kcwc56tglgC1d3zU",
project_name="Chest Xray",log_code=True)
# %%
import pandas as pd
df = pd.read_csv('/media/gyasis/Drive 2/Data/vinbigdata/train.csv')
df.head(10)
# %%
df = df[['image_id', 'class_name','class_id']]
torch.cuda.empty_cache()
# %%
def build_path(x):
path_ = '/media/gyasis/Drive 2/Data/vinbigdata/train/'
filetype = '.dicom'
x = (path_+x+filetype)
return x
def main():
# Variable definition
precision = torch.float
module = "MountainCarContinuous-v0"
# Parameter and Object declarations
env_params = {
"score type": "score", # Function evaluation
"render": True,
"Discrete": False,
"module name": module
}
env = env_factory.make_env("openai", "control", env_params)
#print(env.obs_space)
#print(env.obs_space.high)
#print(env.obs_space.low)
#print(env.action_space)
#print(env.action_space.high)
#print(env.action_space.low)
#exit()
model_params = {
"precision": precision,
"weight initialization scheme": "Normal",
"grad": False,
"in features": 2,
"number of outputs": 1
}
model = model_factory.make_model("Roboschool Simple FC", model_params)
alg_params = {
"target": env.target,
"minimization mode": env.minimize,
"minimum entropy": 0.1,
"tolerance": 0.01,
"max steps": 64,
"memory size": 10
}
alg = algorithm_factory.make_alg("local search", model, alg_params)
experiment = Experiment(api_key="5xNPTUDWzZVquzn8R9oEFkUaa",
project_name="jeff-trinkle", workspace="aromorin")
experiment.set_name("MountainCarCont Learner Wider still")
hyper_params = {"Algorithm": "Learner",
"Parameterization": 1000000,
"Decay Factor": 0.3,
"Directions": 250000,
"Search Radius": 0.5
}
experiment.log_parameters(hyper_params)
slv_params = {
"environment": env,
"algorithm": alg,
"logger": experiment
}
slv = solver_factory.make_slv("RL", slv_params)
# Use solver to solve the problem
slv.solve(iterations=500, ep_len=350)
#slv.solve_online(iterations=1000)
#slv.solve_online_render(iterations=1000, ep_len=15000)
#slv.solve_aggregator(iterations=500, reps=10, ep_len=150)
#slv.solve_averager(iterations=500, reps=1, ep_len=500)
slv.demonstrate_env(episodes=3, ep_len=350)
def main():
# ---------------------------------------------------
# Parsing Arguments
# ---------------------------------------------------
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, default='gcn')
parser.add_argument('--type', type=str, default='base')
parser.add_argument('--runs', type=int, default=10)
parser.add_argument('--epochs', type=int, default=1000)
parser.add_argument('--lr', type=float, default=0.01)
parser.add_argument('--hidden_dim', type=int, default=50)
parser.add_argument('--num_layers', type=int, default=3)
args = parser.parse_args()
assert args.model in ['gcn', 'sage', 'gin']
assert args.type in ['base', 'conf1', 'conf2', 'conf3']
assert args.runs >= 1
assert args.epochs >= 1
assert args.lr > 0
assert args.hidden_dim > 0
# ---------------------------------------------------
# Comet Experiment Tags
# ---------------------------------------------------
experiment_tags = [args.model, args.type]
# ---------------------------------------------------
# Model
# ---------------------------------------------------
model = networks[args.model](in_channels=10,
hidden_channels=args.hidden_dim,
out_channels=10,
num_conv_layers=args.num_layers).to(device)
# ---------------------------------------------------
# Experiment details
# ---------------------------------------------------
print('Graph Classification Experiment - Synthetic Data')
print('Target task: Configuration 4')
print(exp_description[args.type])
print(
'---------------------------------------------------------------------'
)
print('Model: {}'.format(args.model))
print('Number of runs: {}'.format(args.runs))
print('Number of epochs: {}'.format(args.epochs))
print('Learning rate: {}'.format(args.lr))
print()
print(model)
# ---------------------------------------------------
# Experiment loop
# ---------------------------------------------------
for run in range(args.runs):
print()
print(
'---------------------------------------------------------------------'
)
print('Run #{}'.format(run + 1))
print()
# Model initialisation
if args.type == 'base':
# Random init
model.reset_parameters()
elif args.type in ['conf1', 'conf2', 'conf3']:
# Pretrain
print('Generating source dataset')
source_dataset = generate_dataset(
n_classes=10,
n_per_class=100,
percent_swap=conf_params[args.type]['percent_swap'],
percent_damage=conf_params[args.type]['percent_damage'])
source_loader = DataLoader(source_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
print()
model.reset_parameters()
source_optimiser = torch.optim.Adam(model.parameters(), lr=args.lr)
print('Pre-training model on generated dataset')
best_acc = pretrain(model, device, source_loader, source_optimiser)
print('Best accuracy: {:.3}'.format(best_acc))
model.load_state_dict(torch.load('source_model.pth'))
# Comet Experiment
experiment = Experiment(project_name='graph-classification-synthetic',
display_summary_level=0,
auto_metric_logging=False)
experiment.add_tags(experiment_tags)
experiment.log_parameters({
'hidden_dim': args.hidden_dim,
'num_features': 10,
'num_classes': 10,
'learning_rate': args.lr,
'num_epochs': args.epochs,
'generator_params': conf_params[args.type]
})
# Target training
print('Training on Target task')
target_optimiser = torch.optim.Adam(model.parameters(), lr=args.lr)
for epoch in tqdm(range(args.epochs)):
train_loss, acc = train(model, device, target_loader,
target_optimiser)
experiment.log_metric('train_loss', train_loss, step=epoch)
experiment.log_metric('accuracy', acc, step=epoch)
experiment.end()
help='trigger early stopping (boolean)')
parser.add_argument('--method',
type=str,
default='average',
help='aggregation prediction method (max, average)')
parser.add_argument('--decay_lr',
action='store_true',
help='activate decay learning rate function')
opt = parser.parse_args()
print(opt)
ngpu = int(opt.ngpu)
nc = int(opt.nc)
imgSize = int(opt.imgSize)
experiment = Experiment(api_key="qcf4MjyyOhZj7Xw7UuPvZluts", log_code=True)
hyper_params = vars(opt)
experiment.log_multiple_params(hyper_params)
"""
Save experiment
"""
if opt.experiment is None:
opt.experiment = 'samples'
os.system('mkdir experiments')
os.system('mkdir experiments/{0}'.format(opt.experiment))
os.system('mkdir experiments/{0}/images'.format(opt.experiment))
opt.manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", opt.manualSeed)
from __future__ import print_function
from comet_ml import Experiment
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.datasets import fetch_20newsgroups
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import accuracy_score
experiment = Experiment(api_key="YOUR-API-KEY", project_name=' 20 newsgroups project')
# Get dataset and put into train,test lists
categories = ['alt.atheism', 'soc.religion.christian',
'comp.graphics', 'sci.med']
twenty_train = fetch_20newsgroups(subset='train',categories=categories, shuffle=True, random_state=42)
twenty_test = fetch_20newsgroups(subset='test',categories=categories, shuffle=True, random_state=42)
#log hash of your dataset to Comet.ml
experiment.log_dataset_hash(twenty_train)
# Build training pipeline
text_clf = Pipeline([('vect', CountVectorizer()), # Counts occurrences of each word
('tfidf', TfidfTransformer()), # Normalize the counts based on document length
('clf', SGDClassifier(loss='hinge', penalty='l2', # Call classifier with vector
alpha=1e-3, random_state=42,
max_iter=5, tol=None)),
client_opt = "SGD"
client_opt_strategy = "reinit"
# image_norm = "tflike"
# TODO a paraméterek helytelen nevére nem adott hibát
s_opt_args = common.get_args(server_opt)
config = TorchFederatedLearnerCIFAR100Config(
BREAK_ROUND=1500,
CLIENT_LEARNING_RATE=client_lr,
CLIENT_OPT=client_opt,
# CLIENT_OPT_ARGS=common.get_args(client_opt),
CLIENT_OPT_L2=1e-4,
CLIENT_OPT_STRATEGY=client_opt_strategy,
SERVER_OPT=server_opt,
SERVER_OPT_ARGS=s_opt_args,
SERVER_LEARNING_RATE=server_lr,
IS_IID_DATA=is_iid,
BATCH_SIZE=B,
CLIENT_FRACTION=C,
N_CLIENTS=NC,
N_EPOCH_PER_CLIENT=E,
MAX_ROUNDS=max_rounds,
IMAGE_NORM="recordwisefull",
NORM="group",
INIT="tffed",
AUG="basicf"
)
config_technical = TorchFederatedLearnerTechnicalConfig(HIST_SAMPLE=0)
name = f"{config.SERVER_OPT}: {config.SERVER_LEARNING_RATE} - {config.CLIENT_OPT_STRATEGY} - {config.CLIENT_OPT}: {config.CLIENT_LEARNING_RATE}"
experiment = Experiment(workspace="federated-learning", project_name=project_name)
common.do_training(experiment, name, config, config_technical)
@author: gabriel
"""
'''Start of iterative process thing'''
'''Select data set, do smart sampling either rdkit: https://www.rdkit.org/docs/source/rdkit.ML.Cluster.Butina.html
https://squonk.it/docs/cells/RDKit%20MaxMin%20Picker/
or from deep chem: MaxMinSplitter(Splitter), ButinaSplitter(Splitter), FingerprintSplitter(Splitter)
can use maxmin splitter to fill out remainining space in next iter by specifying number remaining as sample size and the pool of
compounds as target
'''
#%%
'''import'''
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="iter_baseline",
workspace="gdreiman1",
disabled=False)
exp.log_code = True
exp.log_other(
'Hypothesis',
'''Taking 5% batches, 80% of batch is the top ranked compounds, remaining 20% is random selection for first 5
iterations, then continues to random sampling''')
import pickle, sys, os
from rdkit.SimDivFilters.rdSimDivPickers import MaxMinPicker
import numpy as np
from Iterative_help_funcs import get_Scaled_Data, train_SVM, train_DNN, train_RF, train_LGBM, calc_and_save_metrics, train_PyTorchGCNN, train_random_classifier
from imblearn.over_sampling import RandomOverSampler
#choosing a 3:1 Inactive to Active ratio
ros = RandomOverSampler(sampling_strategy=0.33)
import pandas as pd
from joblib import Parallel, delayed
param.requires_grad = True
# saving outputs of all Basic Blocks
mdl = learn.model
sf = [
SaveFeatures(m)
for m in [mdl[0][2], mdl[0][4], mdl[0][5], mdl[0][6], mdl[0][7]]
]
sf2 = [
SaveFeatures(m)
for m in [net.relu2, net.layer1, net.layer2, net.layer3, net.layer4]
]
project_name = 'kd-' + hyper_params['model'] + '-' + hyper_params['dataset']
experiment = Experiment(api_key="1jNZ1sunRoAoI2TyremCNnYLO",
project_name=project_name,
workspace="akshaykvnit")
experiment.log_parameters(hyper_params)
if hyper_params['stage'] == 0:
filename = '../saved_models/' + str(
hyper_params['dataset']) + '/' + str(
hyper_params['model']) + '_stage' + str(
hyper_params['stage']) + '/model' + str(
hyper_params['repeated']) + '.pt'
else:
filename = '../saved_models/' + str(
hyper_params['dataset']) + '/' + str(
hyper_params['model']) + '_stage' + str(
hyper_params['stage'] + 1) + '/model' + str(
hyper_params['repeated']) + '.pt'
optimizer = torch.optim.Adam(net.parameters(),
import copy
import argparse
import numpy as np
from numpy import genfromtxt
device = torch.device('cpu')
# device = torch.device('cuda') # Uncomment this to run on GPU
parser = argparse.ArgumentParser()
parser.add_argument("--epochs",type=int, required=True)
parser.add_argument("--inputSize", nargs='*', type=int, required=True)
parser.add_argument("--hiddenSize", nargs='*', type=float, required=True)
args = parser.parse_args()
# Define comet experiment
experiment = Experiment("S1gz8UBL6fpAmuPjMIP4zGXMt", project_name="boulder")
learning_rate = 20
# Define nonlinear activation function
def f(x): # nonlinear conversion function to binary
return x.sigmoid()
# Track performance for comet
with experiment.train():
for index_n1 in range(len(args.inputSize)):
N = args.inputSize[index_n1] # input dimension and number of training behaviors
M = N # output dimension
comb_on = M/2 # number of motors turned on for each label
x = torch.tensor(np.genfromtxt("x_{}.csv".format(N), delimiter=','), device=device).float()
y = torch.tensor(np.genfromtxt("y_{}.csv".format(N), delimiter=','), device=device).float()
class CometMLProtocol(Protocol):
# cometml://[username:password@]host1[:port1][,...hostN[:portN]]][/[database][?options]]
def __init__(self, uri):
uri = parse_uri(uri)
# cometml:workspace/project
path = uri.get('path')
if not path:
# cometml://workspace/project
path = uri.get('address')
workspace, project = path.split('/', maxsplit=2)
self.cml = Experiment(project_name=project, workspace=workspace)
self.chrono = {}
self._api = None
@property
def api(self):
# only use API if requested
if self._api is None:
self._api = API()
return self._api
def log_trial_chrono_start(
self,
trial,
name: str,
aggregator: Callable[[], Aggregator] = StatAggregator.lazy(1),
start_callback=None,
end_callback=None):
if start_callback is not None:
start_callback()
self.chrono[name] = {'start': time.time(), 'cb': end_callback}
def log_trial_chrono_finish(self, trial, name, exc_type, exc_val, exc_tb):
if exc_type is not None:
raise exc_type
data = self.chrono.get(name)
if data is None:
return
cb = data.get('cb')
if cb is not None:
cb()
data['end'] = time.time()
self.cml.log_metric(f'chrono_{name}', data['end'] - data['start'])
def log_trial_start(self, trial):
self.cml.log_other('trial_start', time.time())
def log_trial_finish(self, trial, exc_type, exc_val, exc_tb):
self.cml.log_other('trial_end', time.time())
self.cml.end()
if exc_type is not None:
raise exc_type
def log_trial_arguments(self, trial: Trial, **kwargs):
self.cml.log_parameters(kwargs)
def log_trial_metadata(self,
trial: Trial,
aggregator: Callable[[], Aggregator] = None,
**kwargs):
for k, v in kwargs.items():
self.cml.log_other(k, v)
def log_trial_metrics(self,
trial: Trial,
step: any = None,
aggregator: Callable[[], Aggregator] = None,
**kwargs):
self.cml.log_metrics(kwargs, step=step)
def set_trial_status(self, trial: Trial, status, error=None):
self.cml.log_other('trial_status_name', status.name)
self.cml.log_other('trial_status_value', status.value)
def add_trial_tags(self, trial, **kwargs):
self.cml.add_tags([f'{k}-{v}' for k, v in kwargs])
# Object Creation
def get_project(self, project: Project):
return self.api.get(workspace=project.uid)
def new_project(self, project: Project):
warning('CometML does not expose this functionality')
def get_trial_group(self, group: TrialGroup):
return self.api.get(project=group.uid)
def new_trial_group(self, group: TrialGroup):
warning('CometML does not expose this functionality')
def add_project_trial(self, project: Project, trial: Trial):
warning('CometML does not expose this functionality')
def add_group_trial(self, group: TrialGroup, trial: Trial):
warning('CometML does not expose this functionality')
def commit(self, **kwargs):
pass
def get_trial(self, trial: Trial):
return self.api.get(workspace=trial.project_id,
project=trial.group_id,
experiment=trial.uid)
def new_trial(self, trial: Trial):
warning('CometML does not expose this functionality')
parser.add_argument('-gpu', default='0', type=str)
parser.add_argument('-span', default=1.5, type=float)
parser.add_argument('-res', default=12, type=int)
parser.add_argument('-ckpt', default=None, type=str)
parser.add_argument('-url', default=None, type=str)
parser.add_argument('-name', default='svhn_poison', type=str)
parser.add_argument('-projname', default='surface_mean', type=str)
parser.add_argument('-notsvhn', action='store_true')
parser.add_argument('-ntrial', default=None, type=int)
parser.add_argument('-batchsize', default=2**13, type=int)
parser.add_argument('-nworker', default=8, type=int)
args = parser.parse_args()
# comet stuff
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name=args.projname,
workspace="wronnyhuang")
# exptname = 'span_%s-seed_%s-%s' % (args.span, args.seed, args.name)
experiment.set_name(args.name)
experiment.log_parameters(vars(args))
# apply settings
np.random.seed(np.random.randint(1, 99999))
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# load data and model
# cleanloader, _, _ = get_loader(join(home, 'datasets'), batchsize=2 * 2**13, fracdirty=.5, nogan=True, svhn=not args.notsvhn, surface=True, nworker=8)
cleanloader, _, _ = get_loader(join(home, 'datasets'),
batchsize=args.batchsize,
fracdirty=.5,
nogan=True,
from models import MolecularVAE
import argparse
from config import get_parser
from vocab import OneHotVocab
from mosesfile import VAE
epochs=100
df = pd.read_csv("/vol/ml/aclyde/ZINC/zinc_cleaned_cannon.smi", header=None)
max_len = 128
smiles = []
for i in df.itertuples(index=False):
if len(i < max_len):
smiles.append(i)
vocab = OneHotVocab.from_data(smiles)
experiment = Experiment(project_name='pytorch')
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=8 * 6,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=True, num_workers=8 * 6,
pin_memory=True)
model = VAE(vocab, get_parser())
optimizer = optim.Adam((p for p in model.vae.parameters() if p.requires_grad), lr=0.0003)
def train():
return
def train_model(
xpath,
ypath,
xvalidpath,
yvalidpath,
xtestpath,
ytestpath,
modelpath,
models,
scaler,
):
if not os.path.exists(os.path.abspath(modelpath)):
os.mkdir(os.path.abspath(modelpath))
experiment = Experiment(project_name="mof-oxidation-states")
experiment.log_asset(xpath)
experiment.log_asset(ypath)
experiment.log_asset(xvalidpath)
experiment.log_asset(yvalidpath)
experiment.log_asset(xtestpath)
experiment.log_asset(ytestpath)
trainlogger.info("Train X: {}".format(xpath))
trainlogger.info("Train y: {}".format(ypath))
trainlogger.info("Validation X: {}".format(xvalidpath))
trainlogger.info("Validation y: {}".format(yvalidpath))
trainlogger.info("Test X: {}".format(xtestpath))
trainlogger.info("Test y: {}".format(ytestpath))
train_stem = Path(xpath).stem
ml_object = MLOxidationStates.from_x_y_paths(
xpath=os.path.abspath(xpath),
ypath=os.path.abspath(ypath),
xvalidpath=os.path.abspath(xvalidpath),
yvalidpath=os.path.abspath(yvalidpath),
modelpath=os.path.abspath(modelpath),
scaler=scaler,
n=int(10),
voting="soft",
calibrate="istonic",
experiment=experiment,
)
X_test = np.load(xtestpath)
y_test = np.load(ytestpath)
X_test = ml_object.scaler.transform(X_test)
models_loaded = []
for model in models:
name = Path(model).stem
model = joblib.load(model)
models_loaded.append((name, model))
votingclassifier, _ = ml_object.calibrate_ensemble(
models_loaded,
ml_object.x_valid,
ml_object.y_valid,
ml_object.experiment,
ml_object.voting,
ml_object.calibrate,
)
votingclassifier_tuple = [("votingclassifier_" + train_stem, votingclassifier)]
cores_test = ml_object.model_eval(
votingclassifier_tuple, X_test, y_test, experiment, "test", modelpath
)
scores_train = ml_object.model_eval(
votingclassifier_tuple, ml_object.x, ml_object.y, experiment, "train", modelpath
)
scores_valid = ml_object.model_eval(
votingclassifier_tuple,
ml_object.x_valid,
ml_object.y_valid,
experiment,
"valid",
modelpath,
)
def log_dataset(experiment: Experiment, dataset: Dataset):
experiment.log_other("datatap-dataset", dataset.get_stable_identifier())
import math, random
from collections import defaultdict
import tensorflow as tf
import numpy as np
import pickle
from comet_ml import Experiment
experiment = Experiment(api_key="")
TF_CPP_MIN_LOG_LEVEL = 2
def lrelu(x, alpha):
return tf.nn.relu(x) - alpha * tf.nn.relu(-x)
def generator(z, alpha=0.2):
"""Generate b' conditional on input a = z
"""
with tf.variable_scope("generator"):
#fc1 = tf.layers.dense(z, 30*10*256)
# Reshape it to start the convolutional stack
vocab_size = 300
conv1 = tf.layers.Conv1D(filters=32,
kernel_size=1,
strides=1,
padding='SAME')(z)
conv1 = tf.layers.batch_normalization(conv1) #, training=training)
from visualize_util import get_readable_time
import torch
from torch import nn
from models import AttnCanAdcrowdNetSimpleV5
import os
from ignite.contrib.handlers import PiecewiseLinear
from model_util import get_lr
from torchsummary import summary
COMET_ML_API = "S3mM1eMq6NumMxk2QJAXASkUM"
PROJECT_NAME = "crowd-counting-framework"
if __name__ == "__main__":
experiment = Experiment(project_name=PROJECT_NAME, api_key=COMET_ML_API)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
args = my_args_parse()
print(args)
experiment.set_name(args.task_id)
experiment.set_cmd_args()
DATA_PATH = args.input
TRAIN_PATH = os.path.join(DATA_PATH, "train_data")
TEST_PATH = os.path.join(DATA_PATH, "test_data")
dataset_name = args.datasetname
if dataset_name == "shanghaitech":
print("will use shanghaitech dataset with crop ")
elif dataset_name == "shanghaitech_keepfull":
if __name__ == '__main__':
import argparse
#Set training or training
mode_parser = argparse.ArgumentParser(description='Retinanet training or finetuning?')
mode_parser.add_argument('--mode', help='train, retrain or final')
mode_parser.add_argument('--dir', help='destination dir on HPC' )
mode=mode_parser.parse_args()
#load config
DeepForest_config = load_config()
#set experiment and log configs
experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2", project_name='deeplidar', log_code=True)
#save time for logging
if mode.dir:
dirname = os.path.split(mode.dir)[-1]
else:
dirname = datetime.now().strftime("%Y%m%d_%H%M%S")
#Load DeepForest_config and data file based on training or retraining mode. Final mode firsts run training then retrains on top.
DeepForest_config["mode"] = mode.mode
if mode.mode in ["train","final"]:
data = load_training_data(DeepForest_config)
if mode.mode == "retrain":
data = load_retraining_data(DeepForest_config)
def train(args, use_comet : bool = True):
data_cls = funcs[args['dataset']]
model_cls = funcs[args['model']]
network = funcs[args['network']]
print ('[INFO] Getting dataset...')
data = data_cls()
data.load_data()
(x_train, y_train), (x_test, y_test) = (data.x_train, data.y_train), (data.x_test, data.y_test)
classes = data.mapping
# #Used for testing only
# x_train = x_train[:100, :, :]
# y_train = y_train[:100, :]
# x_test = x_test[:100, :, :]
# y_test = y_test[:100, :]
# print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
# print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
# #delete these lines
# distribute 90% test 10% val dataset with equal class distribution
(x_test, x_valid, y_test, y_valid) = train_test_split(x_test, y_test, test_size=0.2, random_state=42)
print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
print ('[INFO] Validation shape: ', x_valid.shape, y_valid.shape)
print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
print ('[INFO] Setting up the model..')
if args['network'] == 'lstmctc':
network_args = {'backbone' : args['backbone'],
'seq_model' : args['seq'],
'bi' : args['bi']
}
model = model_cls(network, data_cls, network_args)
else:
model = model_cls(network, data_cls)
print (model)
dataset = dict({
'x_train' : x_train,
'y_train' : y_train,
'x_valid' : x_valid,
'y_valid' : y_valid,
'x_test' : x_test,
'y_test' : y_test
})
if use_comet and args['find_lr'] == False:
#create an experiment with your api key
experiment = Experiment(api_key='API Key here',
project_name='iam_lines',
auto_param_logging=False)
print ('[INFO] Starting Training...')
#will log metrics with the prefix 'train_'
with experiment.train():
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
#will log metrics with the prefix 'test_'
with experiment.test():
score = model.evaluate(dataset, int(args['batch_size']))
print(f'[INFO] Test evaluation: {score*100}...')
metrics = {
'accuracy':score
}
experiment.log_metrics(metrics)
experiment.log_parameters(args)
experiment.log_dataset_hash(x_train) #creates and logs a hash of your data
experiment.end()
elif use_comet and args['find_lr'] == True:
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
FIND_LR=args['find_lr'],
name=args['network']
)
else :
print ('[INFO] Starting Training...')
train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
score = model.evaluate(dataset, args['batch_size'])
print(f'[INFO] Test evaluation: {score*100}...')
if args['weights']:
model.save_weights()
if args['save_model']:
model.save_model()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 18 15:07:55 2019
@author: gabriel
"""
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="iter_plotting", workspace="gdreiman1", disabled = False
)
exp.log_code = True
exp.log_other('Hypothesis','''These are my plots for tuned with 5%, 5%, random''')
import pickle
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
data_dir = '/home/gabriel/Dropbox/UCL/Thesis/Data'
first8 = 'tuned_7_svmmod0.pkl'
second8 = 'tuned_7_svmmod1.pkl'
third8 = 'tuned_7_svmmod2.pkl'
save_path = os.path.join(data_dir,first8)
pickle_off = open(save_path,'rb')
first8=pickle.load(pickle_off)
pickle_off.close()
save_path = os.path.join(data_dir,second8)
pickle_off = open(save_path,'rb')
second8=pickle.load(pickle_off)
else:
for shapefile in shapefiles:
_predict_(shapefile,
model_path,
savedir=savedir,
create_records=generate)
if __name__ == "__main__":
sleep(randint(0, 20))
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
save_dir = "{}/{}".format(
"/orange/idtrees-collab/DeepTreeAttention/snapshots/", timestamp)
os.mkdir(save_dir)
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.add_tag("Train")
#Create output folder
experiment.log_parameter("timestamp", timestamp)
experiment.log_parameter("log_dir", save_dir)
#Create a class and run
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml",
log_dir=save_dir)
model.create()
if model.config["train"]["pretraining_dir"]:
model.HSI_model.load_weights("{}/HSI_model.h5".format(
model.config["train"]["pretraining_dir"]))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Kaggle: House Prices Regression
# https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python
# Import comet_ml in the top of your file
from comet_ml import Experiment
experiment = Experiment(api_key="iEod5gYY85CcLACpKpvrgYAbQ")
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from scipy.stats import norm
from sklearn.preprocessing import StandardScaler
from scipy import stats
import warnings
warnings.filterwarnings('ignore')
df_train = pd.read_csv(
'/Users/Viggy/Desktop/Data Science/Kaggle/House Prices/train.csv')
df_train.columns
# descriptive statistics
df_train['SalePrice'].describe()
# histogram
sns.distplot(df_train['SalePrice'])
def train(path):
name = os.path.splitext(os.path.basename(path))[0]
print('Processing: ', name)
features = pd.read_csv(path, index_col=None)
selected_features_names = [name for name, desc in selected_features]
features = features[selected_features_names]
split_idx = 1200
features = features.drop(['sound.files'], axis=1)
noise_only_df, df = features.iloc[:split_idx], features.iloc[split_idx:]
y = df.pop('petrel')
X = df.values
y_noise = noise_only_df.pop('petrel')
X_noise = noise_only_df.values
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
hyperparams = {
'n_estimators': [100, 300, 500, 1000],
'learning_rate': [0.1],
'gamma': [0.0, 0.5],
'max_depth': [2, 3, 4],
'min_child_weight': [1, 2],
'subsample': [1.0, 0.8],
'reg_alpha': [0.0, 0.1],
'reg_lambda': [1, 2, 3]
}
#
# hyperparams = {
# 'n_estimators': [100],
# 'learning_rate': [0.1],
# 'gamma': [0.0],
# 'max_depth': [2],
# 'min_child_weight': [1],
# 'subsample': [1.0],
# 'reg_alpha': [0.0],
# 'reg_lambda': [1]
# }
clf = model_selection.GridSearchCV(estimator=xg.XGBClassifier(objective='binary:logistic', n_jobs=-1),
param_grid=hyperparams,
cv=4)
fit_params = clf.fit(X_train, y_train)
estimator = fit_params.best_estimator_
joblib.dump(estimator, name + '_model.pkl')
test_pred = estimator.predict(X_test)
metrics = calculate_metrics(test_pred, y_test)
noise_pred = estimator.predict(X_noise)
noise_detection_accuracy = accuracy_score(y_noise, noise_pred)
experiment = Experiment(api_key="4PdGdUZmGf6P8QsMa5F2zB4Ui",
project_name="storm petrels",
workspace="tracewsl")
experiment.set_name(name)
experiment.log_parameter('name', name)
experiment.log_multiple_params(fit_params.best_params_)
experiment.log_multiple_metrics(metrics)
experiment.log_metric('Noise detection accuracy', noise_detection_accuracy)
experiment.log_figure('Confusion matrix', get_confusion_matrix_figure(test_pred, y_test))
experiment.log_figure('Feature importnace', get_feature_importance_figure(estimator, list(df.columns.values)))
import os
import numpy
import torch
import dataset
import time
import model
import train1 as train
import test1 as test
import warnings
warnings.simplefilter('ignore')
start_time = time.time()
if o.opt.comet:
experiment = Experiment(api_key="aSb5hPnLFt1wjOyjJfTJy4fkJ",
project_name="general",
workspace="arjunsbalaji")
else:
experiment = None
sys.stdout.write('No comet logging' + '\n')
if o.opt.loadcheckpoint is not None:
checkpoint = torch.load(o.opt.loadcheckpoint)
else:
checkpoint = None
data = dataset.OCTDataset(o.opt.dataroot,
start_size=o.opt.start_size,
cropped_size=o.opt.c_size,
transform=o.opt.transforms,
input_images=[0, 1, 2])
