def pretraining(deepforest, BASE_PATH):
# import comet_ml logger
comet_experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",
project_name="deepforest", workspace="bw4sz")
comet_experiment.log_parameters(deepforest_model.config)
comet_experiment.log_parameter("Type","Pretraining")
comet_experiment.log_parameter("timestamp",timestamp)
# create a dir for the run
save_path = BASE_PATH + "snapshots/{}/".format(timestamp)
os.mkdir(save_path)
deepforest_model.config["save_path"] = save_path
deepforest_model.config["snapshot_path"] = save_path
list_of_tfrecords = glob.glob(BASE_PATH + "pretraining/tfrecords/*.tfrecord")
deepforest_model.train(annotations=BASE_PATH + "pretraining/crops/pretraining.csv",
input_type="tfrecord",
list_of_tfrecords=list_of_tfrecords,
comet_experiment=comet_experiment)
if not deepforest_model.config["validation_annotations"] == "None":
mAP = deepforest_model.evaluate_generator(annotations = deepforest_model.config["validation_annotations"], comet_experiment=comet_experiment)
comet_experiment.log_metric("mAP", mAP)
#retrain model based on hand annotation crops, assign the weights from pretraining model, multi-gpu model weights are split.
deepforest_model.model.save_weights(BASE_PATH + "snapshots/pretraining_weights_{}.h5".format(timestamp))
deepforest_model.config["weights"] = BASE_PATH + "snapshots/pretraining_weights_{}.h5".format(timestamp)
return deepforest_model
def train(pretrain_model_path, proportion_data, DeepForest_config):
###Log experiments
experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",
project_name='deeplidar',
log_code=False)
#make snapshot dir
dirname = datetime.now().strftime("%Y%m%d_%H%M%S")
experiment.log_parameter("Start Time", dirname)
save_snapshot_path = DeepForest_config["save_snapshot_path"] + dirname
os.mkdir(save_snapshot_path)
##Replace config file and experiment
DeepForest_config["batch_size"] = 40
DeepForest_config["epochs"] = 40
experiment.log_parameter("mode", "ablation")
DeepForest_config["evaluation_images"] = 0
#set training images, as a function of the number of training windows
DeepForest_config["training_proportion"] = proportion_data
experiment.log_parameters(DeepForest_config)
#Create model
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=1,
weights=pretrain_model_path,
multi_gpu=2,
freeze_backbone=False,
nms_threshold=DeepForest_config["nms_threshold"],
input_channels=DeepForest_config["input_channels"])
if not proportion_data == 0:
#Run training, and pass comet experiment class
#start training
data = load_retraining_data(DeepForest_config)
train_generator, validation_generator = create_h5_generators(
data, DeepForest_config=DeepForest_config)
#ensure directory created first; otherwise h5py will error after epoch.
history = training_model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.size() /
DeepForest_config["batch_size"],
epochs=DeepForest_config["epochs"],
verbose=2,
shuffle=False,
workers=DeepForest_config["workers"],
use_multiprocessing=DeepForest_config["use_multiprocessing"],
max_queue_size=DeepForest_config["max_queue_size"])
num_trees = train_generator.total_trees
else:
num_trees = 0
#Log trees
experiment.log_parameter("Number of Training Trees", num_trees)
return prediction_model, num_trees
class CometTracker:
def __init__(self, comet_params, run_params):
self.experiment = Experiment(**comet_params)
self.experiment.log_parameters(run_params)
def track_metric(self, metric, value):
self.experiment.log_metric(metric, value)
def main_eval(args):
assert args.load_from is not None, '--load_from required in eval mode'
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
dataset_train, dataset_test, scaler = get_data(args)
logging.info(f'evaluation mode. Level: {args.level}')
device = torch.device(
'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
n_features = dataset_train.items.shape[1]
generator, discriminator = get_models(args, n_features, device)
experiment = Experiment(args.comet_api_key,
project_name=args.comet_project_name,
workspace=args.comet_workspace)
experiment.log_parameters(vars(args))
load_model(Path(args.load_from), generator, discriminator, None, None,
device)
n_events = len(dataset_test)
steps = (args.gan_test_ratio * n_events) // args.eval_batch_size
evaluate_model(generator, experiment, dataset_test, args.eval_batch_size,
steps, args, device, scaler, 0)
def train(x_train, y_train, x_test, y_test):
exp = Experiment(project_name="perception",
auto_histogram_gradient_logging=True)
# log custom hyperparameters
exp.log_parameters(params)
# log any custom metric
exp.log_metric('custom_metric', 0.95)
# log a dataset hash
exp.log_dataset_hash(x_train)
# Define model
model = build_model_graph(exp)
model.fit(
x_train,
y_train,
batch_size=exp.get_parameter('batch-size'),
epochs=exp.get_parameter('epochs'),
validation_data=(x_test, y_test),
)
score = model.evaluate(x_test, y_test, verbose=0)
logging.info("Score %s", score)
# Finalize model includes the following calls
# exp.log_confusion_matrix()
# exp.log_image()
# exp.log_histogram_3d()
# exp.add_tag()
# exp.log_model()
utils.finalize_model(model, x_train, y_train, x_test, y_test, exp)
def main(config, comet=False):
config = Config(config)
# comet-ml setting
if comet:
experiment = Experiment(api_key=config.api_key,
project_name=config.project_name,
workspace=config.workspace)
experiment.log_parameters(config)
# device and dataset setting
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
dataset = MCDataset(config.root, config.dataset_name)
data = dataset[0].to(device)
config.num_nodes = dataset.num_nodes
config.num_users = int(data.num_users)
config.num_relations = dataset.num_relations # defines number of edge types
# set and init model
model = GAE(config, random_init).to(device)
model.apply(init_xavier)
# train
if comet:
trainer = Trainer(model, dataset, data, calc_rmse, config.epochs,
config.lr, config.weight_decay, experiment)
else:
trainer = Trainer(model, dataset, data, calc_rmse, config.epochs,
config.lr, config.weight_decay)
trainer.iterate()
def main():
STARTTIME0 = time.strftime('run_%Y_%m_%d_%H_%M_%s')
METRICS = []
for ts_size in [3000, 5000, 5600]:
for iteration in range(10):
_, _, X_train, X_test, y_train, y_test, _ = process_data(
size=ts_size)
experiment = Experiment(api_key=os.environ['COMET_API_KEY'],
project_name='color-ml')
experiment.log_parameters(PARAMETERS_MEDIAN)
with experiment.train():
regressor_median = fit(X_train, y_train)
metrics_dict = get_metrics_dict(regressor_median, X_test, y_test,
experiment)
metrics_dict['iteration'] = iteration
metrics_dict['ts_size'] = ts_size
METRICS.append(metrics_dict)
df = pd.DataFrame(METRICS)
df.to_csv('learningurve_' + STARTTIME0 + '.csv')
experiment.log_asset('learningurve_' + STARTTIME0 + '.csv')
class Experiment():
def __init__(self, api_key=None, **kwargs):
self._exp = None
self._id = uuid4().hex
if api_key:
self._exp = CometExperiment(api_key,
log_code=False,
auto_param_logging=False,
auto_metric_logging=False,
**kwargs)
self._id = self._exp.get_key()
def log_metric(self, name, value, step=None, epoch=None):
if self._exp:
self._exp.log_metric(name, value, step, epoch)
def log_epoch_end(self, epoch_cnt, step=None):
if self._exp:
self._exp.log_epoch_end(epoch_cnt, step=step)
def log_parameters(self, hp):
if self._exp:
self._exp.log_parameters(flatten(hp, reducer='underscore'))
@property
def id(self):
return self._id[:12]
class CometMLLogger:
def __init__(self):
self.experiment = Experiment(api_key="iU4f44llKnowZwmrEo9wfR2ch",
project_name="general",
workspace="yahyaalaamassoud",
log_code=False,
log_graph=False)
def log_params(self, params: Dict[str, int]):
self.experiment.log_parameters(params)
def log_metric(self, metric_name, metric_val, step=None):
self.experiment.log_metric(metric_name, metric_val, step=step)
def log_metrics(self, metrics: Dict[str, float], step=None):
self.experiment.log_metrics(metrics, step=step)
def log_figure(self, figure_name: str, step: str):
self.experiment.log_image(image_data=self.__savefig(),
name=figure_name,
step=step,
overwrite=False)
def __savefig(self):
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return Image.open(io.BytesIO(buf.getvalue()))
def train_lgbm(x_train, x_val, y_train, y_val):
# Make sure labels not one-hot
experiment = Experiment(
api_key="VNQSdbR1pw33EkuHbUsGUSZWr",
project_name="piratesofthecaribbean",
workspace="florpi",
auto_param_logging=False,
)
experiment.log_parameters(hyperparameters)
clf = lgb.LGBMClassifier(**hyperparameters)
clf.fit(X=x_train, y=y_train, eval_set=(x_val, y_val), eval_metric='AUC')
y_pred = clf.predict(x_val)
visualize.plot_confusion_matrix(
y_val,
y_pred,
classes=LABELS,
normalize=True,
experiment=experiment,
)
visualize.plot_confusion_matrix(
y_val,
y_pred,
classes=LABELS,
normalize=False,
experiment=experiment,
)
def get_experiment_objects():
args = CLIParser().parse_args()
comet = Experiment(api_key="<your_key>",
project_name="<your_project>",
workspace="<your_workspace>",
log_code=False,
auto_param_logging=False,
auto_metric_logging=False,
disabled=args.no_comet,
display_summary=False)
if not args.continue_train:
# else args.save is the directory from which assets are loaded to
# continue training
args.save = "{}-{}".format(args.save, time.strftime("%Y%m%d-%H%M%S"))
create_exp_dir(args)
comet.set_name(args.save.split('/')[-1])
comet.log_parameters(vars(args))
copy_assets(args, comet)
last_state = None
if args.continue_train:
last_state = torch.load(os.path.join(args.save, 'state.pt'))
continue_random(last_state, args.cuda)
else:
init_random(args.seed, args.cuda)
return args, comet, last_state
def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.callbacks.checkpoint_dir,
'%s-{epoch:02d}-{val_loss:.2f}.hdf5' %
self.config.exp.name),
monitor=self.config.callbacks.checkpoint_monitor,
mode=self.config.callbacks.checkpoint_mode,
save_best_only=self.config.callbacks.checkpoint_save_best_only,
save_weights_only=self.config.callbacks.
checkpoint_save_weights_only,
verbose=self.config.callbacks.checkpoint_verbose,
))
self.callbacks.append(
TensorBoard(
log_dir=self.config.callbacks.tensorboard_log_dir,
write_graph=self.config.callbacks.tensorboard_write_graph,
))
if hasattr(self.config, "comet_api_key"):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config.comet_api_key,
project_name=self.config.exp_name)
experiment.add_tags(self.config.tags)
experiment.disable_mp()
experiment.log_parameters(self.config)
self.callbacks.append(experiment.get_keras_callback())
def train(x_train, y_train, x_test, y_test):
exp = Experiment(api_key="API_KEY",
project_name="PROJECT",
workspace="WORKSPACE")
# log custom hyperparameters
exp.log_parameters(params)
# Define model
model = build_model_graph(exp)
model.fit(
x_train,
y_train,
batch_size=exp.get_parameter('batch-size'),
epochs=exp.get_parameter('epochs'),
validation_data=(x_test, y_test),
)
score = model.evaluate(x_test, y_test, verbose=0)
logging.info("Score %s", score)
# Finalize model includes the following calls
# exp.log_confusion_matrix()
# exp.log_image()
# exp.log_histogram_3d()
# exp.add_tag()
# exp.log_model()
utils.finalize_model(model, x_train, y_train, x_test, y_test, exp)
def prerun(args, run_dir=True, exp=True):
if run_dir:
p = Path(args["run_dir"])
if p.exists():
raise ValueError("Rundir exists, please remove.")
p.mkdir()
with open(Path(p, "args.json"), "w") as fp:
json.dump(args, fp)
if exp:
experiment = Experiment(
api_key="pJ6UYxQwjYoYbCmmutkqP66ni",
project_name=args["comet_project"],
workspace="mlippie",
auto_metric_logging=True,
auto_param_logging=False,
log_graph=True,
disabled="maximl" in socket.gethostname() # disable on dev machine
)
experiment.log_parameters(args)
return experiment
else:
return None
def __init__(self, experiment: Experiment, step: STEP, n_classes=2, topk: [int] = [1], class_map: List[str] = [],
metrics=[]):
"""
:param experiment:
:param step:
:param n_classes:
:param topk:
:param class_map:
:param metrics:
"""
assert len(class_map) == n_classes, \
f"Class map: {class_map} length is {len(class_map)} and is not equal to n_classes: {n_classes}"
assert len(topk) != 0, \
f"topk param should be not empty"
topk = np.sort(topk)
assert topk[-1] <= n_classes, \
f"topk max class is {topk[-1]} which is greater than maximum class number: {n_classes}"
self.n_classes = n_classes
self.class_map = class_map
experiment.log_parameters(dict(zip(class_map, [i for i in range(len(class_map))])))
self.step = step
self.topk = topk
self.experiment = experiment
self.curr_state = {}
self.metrics = ['ACC', 'LOSS'] + metrics
self.accuracy_meter = torchnet.meter.ClassErrorMeter(topk=topk, accuracy=True) # accepts probs + labels
self.ap_accuracy_meter = APMeter(n_classes)
self.loss_meter = torchnet.meter.AverageValueMeter()
self.other_meters: Dict[str, torchnet.meter.AverageValueMeter] = \
{metric_name: torchnet.meter.AverageValueMeter() for metric_name in metrics}
def main() -> None:
args = get_args()
config = get_bunch_config_from_json(args.config)
comet_experiment = Experiment(
api_key=config.comet_api_key,
project_name=config.comet_project_name,
workspace=config.comet_workspace,
disabled=not config.use_comet_experiments,
)
comet_experiment.set_name(config.experiment_name)
comet_experiment.log_parameters(config)
test_tweets = load_test_tweets(config.test_data_path)
client = LanguageServiceClient()
result = []
predictions = np.zeros(len(test_tweets), dtype=np.int32)
for i, tweet in enumerate(test_tweets):
start_iter_timestamp = time.time()
document = types.Document(
type=enums.Document.Type.PLAIN_TEXT, content=tweet, language="en"
)
response = client.analyze_sentiment(document=document)
response_dict = MessageToDict(response)
result.append(response_dict)
prediction_present = bool(response_dict["documentSentiment"])
if prediction_present:
# -1, 1 predictions
predictions[i] = 2 * (response.document_sentiment.score > 0) - 1
print("iteration", i, "took:", time.time() - start_iter_timestamp, "seconds")
comet_experiment.log_asset_data(result, name="google_nlp_api_response.json")
ids = np.arange(1, len(test_tweets) + 1).astype(np.int32)
predictions_table = np.column_stack((ids, predictions))
if comet_experiment.disabled:
save_path = build_save_path(config)
os.makedirs(save_path)
formatted_predictions_table = pd.DataFrame(
predictions_table, columns=["Id", "Prediction"], dtype=np.int32,
)
formatted_predictions_table.to_csv(
os.path.join(save_path, "google_nlp_api_predictions.csv"), index=False
)
else:
comet_experiment.log_table(
filename="google_nlp_api_predictions.csv",
tabular_data=predictions_table,
headers=["Id", "Prediction"],
)
percentage_predicted = np.sum(predictions != 0) / predictions.shape[0]
comet_experiment.log_metric(name="percentage predicted", value=percentage_predicted)
class CometTracker:
def __init__(self, comet_params, run_params=None, prev_exp_id=None):
if prev_exp_id: # previous experiment
api_key = comet_params['api_key']
del comet_params[
'api_key'] # removing this because the rest of the items need to be passed
self.experiment = ExistingExperiment(
api_key=api_key,
previous_experiment=prev_exp_id,
**comet_params)
print(
f'In CometTracker: ExistingExperiment initialized with id: {prev_exp_id}'
)
else: # new experiment
self.experiment = Experiment(**comet_params)
self.experiment.log_parameters(run_params)
def track_metric(self, metric, value, step):
self.experiment.log_metric(metric, value, step)
def add_tags(self, tags):
self.experiment.add_tags(tags)
print(f'In [add_tags]: Added these tags to the new experiment: {tags}')
def set_name(self, name):
self.experiment.set_name(name)
def fit_validate(exp_params, k, data_path, write_path, others=None, custom_tag=''):
"""Fit model and compute metrics on train and validation set. Intended for hyperparameter search.
Only logs final metrics and scatter plot of final embedding.
Args:
exp_params(dict): Parameter dict. Should at least have keys model_name, dataset_name & random_state. Other
keys are assumed to be model parameters.
k(int): Fold identifier.
data_path(str): Data directory.
write_path(str): Where to write temp files.
others(dict): Other things to log to Comet experiment.
custom_tag(str): Custom tag for comet experiment.
"""
# Comet experiment
exp = Experiment(parse_args=False)
exp.disable_mp()
custom_tag += '_validate'
exp.add_tag(custom_tag)
exp.log_parameters(exp_params)
if others is not None:
exp.log_others(others)
# Parse experiment parameters
model_name, dataset_name, random_state, model_params = parse_params(exp_params)
# Fetch and split dataset.
data_train = getattr(grae.data, dataset_name)(split='train', random_state=random_state, data_path=data_path)
data_train, data_val = data_train.validation_split(random_state=FOLD_SEEDS[k])
# Model
m = getattr(grae.models, model_name)(random_state=FOLD_SEEDS[k], **model_params)
m.write_path = write_path
m.data_val = data_val
with exp.train():
m.fit(data_train)
# Log plot
m.comet_exp = exp
m.plot(data_train, data_val, title=f'{model_name} : {dataset_name}')
# Probe embedding
prober = EmbeddingProber()
prober.fit(model=m, dataset=data_train, mse_only=True)
train_z, train_metrics = prober.score(data_train, is_train=True)
# Log train metrics
exp.log_metrics(train_metrics)
with exp.validate():
val_z, val_metrics = prober.score(data_val)
# Log train metrics
exp.log_metrics(val_metrics)
# Log marker to mark successful experiment
exp.log_other('success', 1)
def main(logger, optimized_function, optimizer, diff_scheme,
optimizer_config_file, project_name, work_space, tags,
num_repetitions, n, h, use_true_grad, init_psi, p):
device = torch.device('cpu')
print("Using device = {}".format(device))
optimizer_config = getattr(__import__(optimizer_config_file),
'optimizer_config')
init_psi = torch.tensor([float(x.strip())
for x in init_psi.split(',')]).float().to(device)
psi_dim = len(init_psi)
optimized_function_cls = str_to_class(optimized_function)
optimizer_cls = str_to_class(optimizer)
diff_scheme_func = str_to_class(diff_scheme)
experiment = Experiment(project_name=project_name, workspace=work_space)
experiment.add_tags([x.strip() for x in tags.split(',')])
experiment.log_parameter('optimizer_type', optimizer)
experiment.log_parameters({
"optimizer_{}".format(key): value
for key, value in optimizer_config.items()
})
experiment.log_parameters({
"optimizer_{}".format(key): value
for key, value in optimizer_config.get('line_search_options',
{}).items()
})
logger = str_to_class(logger)(experiment)
y_model = optimized_function_cls(device=device, psi_init=init_psi)
if use_true_grad:
ndiff = y_model
else:
grad_func = lambda f, x, n, h: compute_gradient_of_vector_function(
f=f, x=x, n=n, h=h, scheme=diff_scheme_func)
ndiff = NumericalDifferencesModel(y_model=y_model,
psi_dim=psi_dim,
y_dim=1,
x_dim=1,
n=n,
h=h,
num_repetitions=num_repetitions,
grad_func=grad_func)
max_iters = optimizer_config['max_iters']
optimizer_config['max_iters'] = 1
optimizer_config['p'] = p
optimizer = optimizer_cls(oracle=ndiff, x=init_psi, **optimizer_config)
for iter in range(max_iters):
current_psi, status, history = optimizer.optimize()
print(current_psi)
# if iter % 10 == 0:
logger.log_performance(y_sampler=y_model,
current_psi=current_psi,
n_samples=5000)
torch.cuda.empty_cache()
logger.log_optimizer(optimizer)
def init_callbacks(self):
if (self.config.model.name == "encoder"):
import keras
from keras.callbacks import ModelCheckpoint, TensorBoard, ReduceLROnPlateau, EarlyStopping
else:
import tensorflow.keras as keras
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, ReduceLROnPlateau, EarlyStopping
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.callbacks.checkpoint_dir,
'%s-{epoch:02d}-{val_loss:.2f}.hdf5' %
self.config.exp.name),
monitor=self.config.callbacks.checkpoint_monitor,
mode=self.config.callbacks.checkpoint_mode,
save_best_only=self.config.callbacks.checkpoint_save_best_only,
save_weights_only=self.config.callbacks.
checkpoint_save_weights_only,
verbose=self.config.callbacks.checkpoint_verbose,
))
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.callbacks.checkpoint_dir,
'best_model-%s.hdf5' %
self.config.callbacks.checkpoint_monitor),
monitor=self.config.callbacks.checkpoint_monitor,
mode=self.config.callbacks.checkpoint_mode,
save_best_only=self.config.callbacks.checkpoint_save_best_only,
))
self.callbacks.append(
ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=10,
min_lr=0.0001))
self.callbacks.append(
EarlyStopping(monitor='val_loss', patience=10, verbose=1), )
# 在TCN中使用了tensorflow_addson中的WeightNormalization层，与tensorboard不兼容
# if (self.config.model.name != "tcn"):
# self.callbacks.append(
# TensorBoard(
# log_dir=self.config.callbacks.tensorboard_log_dir,
# write_graph=self.config.callbacks.tensorboard_write_graph,
# histogram_freq=1,
# )
# )
# if self.config.dataset.name == "ptbdb":
# self.callbacks.append(
# AdvancedLearnignRateScheduler(monitor='val_main_output_loss', patience=6, verbose=1, mode='auto',
# decayRatio=0.1),
# )
if ("comet_api_key" in self.config):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config.comet_api_key,
project_name=self.config.exp_name)
experiment.disable_mp()
experiment.log_parameters(self.config["trainer"])
self.callbacks.append(experiment.get_callback('keras'))
def train_hierarchy_extractor(
data_path,
bsz,
num_workers,
lr,
weight_decay,
warmup_updates,
max_updates,
accumulation_steps,
validation_interval,
seed,
model_path,
tag,
device,
save_metric,
save_min,
):
experiment = Experiment(project_name="information-retrieval",
auto_output_logging=False)
experiment.add_tags([tag])
parameters = {
"bsz": bsz,
"num_workers": num_workers,
"lr": lr,
"weight_decay": weight_decay,
"warmup_updates": warmup_updates,
"max_updates": max_updates,
"validation_interval": validation_interval,
"seed": seed,
"model_path": model_path,
"device": device,
"accumulation_steps": accumulation_steps,
"save_metric": save_metric,
"save_min": save_min
}
experiment.log_parameters(parameters)
model = BertHierarchyExtractor(model_path, 12, device)
trainer = BertExtractorTrainer(
experiment,
model,
data_path,
model_path,
bsz,
num_workers,
lr,
weight_decay,
warmup_updates,
max_updates,
accumulation_steps,
validation_interval,
save_metric,
save_min,
device,
seed,
)
try:
trainer.train()
except Exception as e:
raise e
def main():
precision = torch.float
# Parameter and Object declarations
env_params = {
"data path": "C:/Users/aaa2cn/Documents/nao_data/",
"ip": "localhost",
"port": 52232,
"score type": "score" # Aggregate error in pose
}
env = env_factory.make_env("nao", "pose assumption", env_params)
# Make a pool object
model_params = {
"precision": precision,
"weight initialization scheme": "Sparse",
"grad": False
}
model = model_factory.make_model("NAO FC model", model_params)
# Make an algorithm object
alg_params = {
"target": env.target,
"minimization mode": env.minimize,
"minimum entropy": 0.1,
"tolerance": 0.1,
"max steps": 64,
"memory size": 10
}
alg = algorithm_factory.make_alg("local search", model, alg_params)
experiment = Experiment(api_key="5xNPTUDWzZVquzn8R9oEFkUaa",
project_name="nao",
workspace="aromorin")
experiment.set_name("Pose Assumption virtual")
hyper_params = {
"Algorithm": "LS",
"Parameterization": 35000,
"Decay Factor": 0.01,
"Directions": 10,
"Search Radius": 0.1
}
experiment.log_parameters(hyper_params)
slv_params = {"environment": env, "algorithm": alg, "logger": experiment}
slv = solver_factory.make_slv("robot", slv_params)
slv.solve(iterations=5000)
slv.save_elite_weights(path='', name='pose_assump_virtual')
# Recreate the target pose
alg.eval()
pred = alg.model(env.observation)
angles = [p.item() for p in pred]
print("These are the angles: ")
print(angles)
env.set_joints(angles)
env.say("Is this the pose you set for me?")
env.rest()
def main() -> None:
args = get_args()
config = get_bunch_config_from_json(args.config)
comet_experiment = Experiment(
api_key=config.comet_api_key,
project_name=config.comet_project_name,
workspace=config.comet_workspace,
disabled=not config.use_comet_experiments,
)
comet_experiment.set_name(config.experiment_name)
comet_experiment.log_parameters(config)
if config.model == "randomforest":
classifier = GloveEmbeddingsClassifier(
RandomForestClassifier(random_state=config.random_seed))
elif config.model == "logregression":
classifier = GloveEmbeddingsClassifier(
LogisticRegression(solver="saga", random_state=config.random_seed))
elif config.model == "decisiontree":
classifier = GloveEmbeddingsClassifier(
DecisionTreeClassifier(random_state=config.random_seed))
else:
raise ValueError("chosen model not available")
training_features, training_labels = classifier.generate_training_data(
config)
best_model, best_model_score, best_model_params = classifier.run_grid_search(
config.random_seed, config.model_parameters, training_features,
training_labels)
comet_experiment.log_metric("mean accuracy", best_model_score)
comet_experiment.log_parameters(best_model_params)
test_data_features = classifier.generate_test_data_features(config)
ids = np.arange(1, test_data_features.shape[0] + 1)
predictions = best_model.predict(test_data_features)
predictions_table = np.stack([ids, predictions], axis=-1).astype(int)
if comet_experiment.disabled:
save_path = build_save_path(config)
os.makedirs(save_path)
formatted_predictions_table = pd.DataFrame(
predictions_table,
columns=["Id", "Prediction"],
dtype=np.int32,
)
formatted_predictions_table.to_csv(os.path.join(
save_path, "test_predictions.csv"),
index=False)
else:
comet_experiment.log_table(
filename="test_predictions.csv",
tabular_data=predictions_table,
headers=["Id", "Prediction"],
)
def setup_comet_ml_experiment(api_key, project_name, experiment_name,
parameters, tags):
""" Function for setting up comet ml experiment """
experiment = Experiment(api_key=api_key,
project_name=project_name,
auto_metric_logging=False)
experiment.set_name(experiment_name)
experiment.log_parameters(parameters)
experiment.add_tags(tags)
return experiment
def main(logger, optimizer, optimized_function, optimizer_config_file,
model_config_file, project_name, work_space, tags, reuse_optimizer,
init_psi, epochs):
model_config = getattr(__import__(model_config_file), 'model_config')
optimizer_config = getattr(__import__(optimizer_config_file),
'optimizer_config')
current_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',')
]).float().to(device)
psi_dim = len(init_psi)
print(psi_dim, current_psi)
optimized_function_cls = str_to_class(optimized_function)
optimizer_cls = str_to_class(optimizer)
experiment = Experiment(project_name=project_name, workspace=work_space)
experiment.add_tags([x.strip() for x in tags.split(',')])
experiment.log_parameters({
"optimizer_{}".format(key): value
for key, value in optimizer_config.items()
})
experiment.log_parameters({
"optimizer_{}".format(key): value
for key, value in optimizer_config.get('line_search_options',
{}).items()
})
logger = str_to_class(logger)(experiment)
y_model = optimized_function_cls(device=device, psi_init=current_psi)
model = VoidModel(y_model=y_model, **model_config)
optimizer = optimizer_cls(oracle=model, x=current_psi, **optimizer_config)
for epoch in range(epochs):
if reuse_optimizer:
optimizer.update(oracle=model, x=current_psi)
else:
# find new psi
optimizer = optimizer_cls(oracle=model,
x=current_psi,
**optimizer_config)
current_psi, status, history = optimizer.optimize()
try:
logger.log_optimizer(optimizer)
logger.log_grads(model,
y_sampler=y_model,
current_psi=current_psi,
num_repetitions=5000)
logger.log_performance(y_sampler=y_model,
current_psi=current_psi,
n_samples=5000)
except Exception as e:
print(e)
raise
def main(cmd=None, stdout=True):
"""Run finetuning experiment for fixed seed."""
# Initialize system
args = get_args(cmd)
assert torch.cuda.is_available()
torch.cuda.set_device(args.device)
# Initialize logging
model_id = ("Finetune {}, seed size {}, epochs {}, labels {}, "
"batch size {}, lr {}").format(args.model, args.seed_size,
args.epochs, args.label_budget,
args.batch_size, args.lr)
logging.basicConfig(filename="{}/{}.txt".format(args.dout, model_id),
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
if stdout:
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logger = Experiment(comet_ml_key, project_name="ActiveDialogue")
logger.set_name(model_id)
logger.log_parameters(vars(args))
# Select model and environment
if args.model == "glad":
model_arch = GLAD
elif args.model == "gce":
model_arch = GCE
env = DSTEnv(load_dataset, model_arch, args)
# Load seed if need-be
if not env.load('seed'):
raise ValueError("No loaded seed.")
# Initialize evaluation
best_metrics = env.metrics(True)
for k, v in best_metrics.items():
logger.log_metric(k, v, step=0)
logging.info("Initial metrics: %s", best_metrics)
# Finetune
env.label_all()
for epoch in range(1, args.epochs + 1):
logging.info('Starting fit epoch %d.', epoch)
env.fit()
metrics = env.metrics(True)
logging.info("Epoch metrics: %s", metrics)
for k, v in metrics.items():
logger.log_metric(k, v, step=epoch)
if best_metrics is None or metrics[args.stop] > best_metrics[
args.stop]:
logging.info("Saving best!")
best_metrics = metrics
def main():
hyper_params = {
"learning_rate": 1,
"style_weight": 10000,
"content_weight": 1,
"n_steps": 300
}
experiment = Experiment(api_key="a604AfX0S9Bmt6HdpMHxg9MCI",
project_name="style-transfer",
workspace="polmonroig")
experiment.log_parameters(hyper_params)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
print("Currently using cuda device.")
# define file paths and directories
content_images_dir = "content_images/"
style_images_dir = "style_images/"
output_images_dir = "output_images/"
content_image_name = "content_01.png"
style_image_name = "style_01.jpg"
output_image_path = join(
output_images_dir,
content_image_name.split('.')[0] + "_" +
style_image_name.split('.')[0] + ".jpg")
content_image_path = join(content_images_dir, content_image_name)
style_image_path = join(style_images_dir, style_image_name)
# define image file manager
max_shape = (800, 800)
fileManager = FileManager(content_image_path, style_image_path, device,
max_shape)
# read images
content_image, style_image = fileManager.read_images()
input_image = content_image.clone()
model = ArtNet(device=device)
output_image = model.train(hyper_params['content_weight'],
hyper_params['style_weight'],
hyper_params['n_steps'], content_image,
style_image, input_image,
hyper_params['learning_rate'], experiment)
fileManager.save_image(output_image, output_image_path)
experiment.log_image(
output_image_path,
content_image_name.split('.')[0] + "_" +
style_image_name.split('.')[0])
def train_simultaneous(hyper_params, teacher, student, sf_teacher, sf_student,
trainloader, valloader, args):
if args.api_key:
project_name = 'simultaneous-' + hyper_params[
'dataset'] + '-' + hyper_params['model']
experiment = Experiment(api_key=args.api_key,
project_name=project_name,
workspace=args.workspace)
experiment.log_parameters(hyper_params)
optimizer = torch.optim.Adam(student.parameters(),
lr=hyper_params['learning_rate'])
scheduler = torch.optim.lr_scheduler.OneCycleLR(
optimizer,
max_lr=1e-2,
steps_per_epoch=len(trainloader),
epochs=hyper_params['num_epochs'])
if hyper_params['dataset'] == 'camvid':
criterion = nn.CrossEntropyLoss(ignore_index=11)
else:
criterion = nn.CrossEntropyLoss(ignore_index=250)
hyper_params['num_classes'] = 19
criterion2 = nn.MSELoss()
savename = get_savename(hyper_params,
dataset=args.dataset,
mode='simultaneous',
p=args.percentage)
highest_iou = 0
for epoch in range(hyper_params['num_epochs']):
_, _, train_loss, val_loss, avg_iou, avg_px_acc, avg_dice_coeff = train_simult(
model=student,
teacher=teacher,
sf_teacher=sf_teacher,
sf_student=sf_student,
train_loader=trainloader,
val_loader=valloader,
num_classes=hyper_params['num_classes'],
loss_function=criterion,
loss_function2=criterion2,
optimiser=optimizer,
scheduler=scheduler,
epoch=epoch,
num_epochs=hyper_params['num_epochs'],
savename=savename,
highest_iou=highest_iou,
args=args)
if args.api_key:
experiment.log_metric('train_loss', train_loss)
experiment.log_metric('val_loss', val_loss)
experiment.log_metric('avg_iou', avg_iou)
experiment.log_metric('avg_pixel_acc', avg_px_acc)
experiment.log_metric('avg_dice_coeff', avg_dice_coeff)
def multi_train(configs_path="./configs.json"):
# configs_path = "/home/shared/Layer-Wise-Learning-Trends-PyTorch/configs.json"
print(f"Configs Path: {configs_path}")
configs = SimpleNamespace(**json.load(open(configs_path)))
print(configs)
experiment = Experiment(
api_key="ZgD8zJEiZErhwIzPMfZpitMjq",
project_name="imagenet-rwc",
workspace="ayushm-agrawal",
)
experiment.log_parameters(configs)
average_rmae_dict, average_train_acc, average_test_acc = {
}, np.zeros(configs.epochs), np.zeros(configs.epochs)
og_config_exp_name = configs.exp_name
for _, seed in enumerate(configs.seed_list):
configs.exp_name = f"{seed}_{og_config_exp_name}"
print(f"Training: {configs.exp_name}")
configs.seed = seed
configs.experiment = experiment
rmae_dict, train_acc_arr, test_acc_arr = run_experiment(
configs.epochs, configs.model_name, "untrained", configs)
for layer in rmae_dict:
if layer not in average_rmae_dict:
average_rmae_dict[layer] = np.array(rmae_dict[layer])
else:
average_rmae_dict[layer] += np.array(rmae_dict[layer])
average_train_acc += train_acc_arr
average_test_acc += test_acc_arr
for layer in average_rmae_dict:
average_rmae_dict[layer] /= len(configs.seed_list)
average_train_acc /= len(configs.seed_list)
average_test_acc /= len(configs.seed_list)
save_path = join(configs.arr_save_path, og_config_exp_name)
print(f"Saving Arrays at {save_path}")
np.save(save_path + "avg_train_acc.npy", average_train_acc)
np.save(save_path + "avg_test_acc.npy", average_test_acc)
np.save(save_path + "avg_rmae_dict.npy", average_rmae_dict)
print("Done!")
def main(cfg, comet=False):
cfg = Config(cfg)
print('start')
# comet-ml setting
if comet:
experiment = Experiment(api_key=cfg.api_key,
project_name=cfg.project_name,
workspace=cfg.workspace)
experiment.log_parameters(cfg)
else:
experiment = None
# device and dataset setting
#device = (torch.device(f'cuda:{cfg.gpu_id}')
# if torch.cuda.is_available() and cfg.gpu_id >= 0
# else torch.device('cpu'))
device = torch.device('cuda:0')
print(device)
dataset = MCDataset(cfg.root, cfg.dataset_name)
print('load done')
data = dataset[0].to(device)
# add some params to config
cfg.num_nodes = dataset.num_nodes
cfg.num_relations = dataset.num_relations
cfg.num_users = int(data.num_users)
# set and init model
model = GAE(cfg, random_init).to(device)
model.cuda()
model.apply(init_xavier)
# optimizer
optimizer = torch.optim.Adam(
model.parameters(),
lr=cfg.lr,
weight_decay=cfg.weight_decay,
)
# train
trainer = Trainer(
model,
dataset,
data,
calc_rmse,
optimizer,
experiment,
)
trainer.training(cfg.epochs)
