class CometLogger():
def __init__(self, enabled, is_existing=False, prev_exp_key=None):
"""
Handles logging of experiment to comet and also persistence to local file system.
Supports resumption of stopped experiments.
"""
disabled = not enabled
if not is_existing:
self.experiment = Experiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled)
else:
if prev_exp_key is None:
raise ValueError(
"Requested existing experiment, but no key provided")
print("Continuing existing experiment with key: ", prev_exp_key)
self.experiment = ExistingExperiment(
api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled,
previous_experiment=prev_exp_key)
self.disabled = disabled
def get_experiment_key(self):
return self.experiment.get_key()[:9]
def add_tag(self, tag):
self.experiment.add_tag(tag)
def log_metric(self, name, value, step=None):
self.experiment.log_metric(name, value, step=step)
def log_metrics(self, metrics_dict, prefix, step=None):
self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step)
def log_params(self, params_dict):
self.experiment.log_parameters(params_dict)
def set_name(self, name_str):
self.experiment.set_name(name_str)
def log_dataset(self, dataset: SpeakerVerificationDataset):
if self.disabled:
return
dataset_string = ""
dataset_string += "<b>Speakers</b>: %s\n" % len(dataset.speakers)
dataset_string += "\n" + dataset.get_logs()
dataset_string = dataset_string.replace("\n", "<br>")
self.vis.text(dataset_string, opts={"title": "Dataset"})
def log_implementation(self, params):
if self.disabled:
return
implementation_string = ""
for param, value in params.items():
implementation_string += "<b>%s</b>: %s\n" % (param, value)
implementation_string = implementation_string.replace("\n", "<br>")
self.implementation_string = implementation_string
self.implementation_win = self.vis.text(
implementation_string, opts={"title": "Training implementation"})
def draw_projections(self,
embeds,
utterances_per_speaker,
step,
out_fpath=None,
max_speakers=16):
if self.disabled:
return
max_speakers = min(max_speakers, len(colormap))
embeds = embeds[:max_speakers * utterances_per_speaker]
n_speakers = len(embeds) // utterances_per_speaker
ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker)
colors = [colormap[i] for i in ground_truth]
reducer = umap.UMAP()
projected = reducer.fit_transform(embeds)
plt.scatter(projected[:, 0], projected[:, 1], c=colors)
plt.gca().set_aspect("equal", "datalim")
plt.title("UMAP projection (step %d)" % step)
if out_fpath is not None:
plt.savefig(out_fpath)
plt.clf()
self.experiment.log_image(out_fpath, step=step)
loss = running_loss / total
acc = 100 * float(correct) / total
if experiment:
with experiment.validate():
experiment.log_metric('loss', loss, step=epoch)
experiment.log_metric('acc', acc, step=epoch)
print('VALIDATION - Epoch: %d, Loss: %.3f, Accuracy: %.3f' % (
epoch, loss, acc))
print('Epoch Time: %.1f' % (time.time()-running_time))
print()
scheduler.step(loss)
early_stopping.step(loss)
is_best = acc > best_acc
best_acc = max(acc, best_acc)
checkpoint = {
'epoch': epoch + 1,
'best_acc': best_acc,
'train_indices': train_indices,
'validation_indices': validation_indices,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict()}
if experiment:
checkpoint['experiment_key'] = experiment.get_key()
save_checkpoint(checkpoint, is_best)
print('Finished Training')
class CometConnection:
def __init__(self, comet_name=None, dataset_config=None, exp_key=None):
self.experiment = None
if comet_name is not None and dataset_config is not None:
self._init_new_experiment(comet_name, dataset_config)
elif exp_key is not None:
self._init_continue_experiment(exp_key)
def _init_new_experiment(self, comet_name, dataset_config):
self.experiment = Experiment(api_key=COMET_KEY,
project_name=PROJECT_NAME)
self.experiment.set_name(comet_name)
self.log_data_attributes(dataset_config)
self.experiment.log_asset('datagen/spectra_generator.m')
def _init_continue_experiment(self, exp_key):
self.experiment = ExistingExperiment(api_key=COMET_KEY,
previous_experiment=exp_key)
def serialize(self):
params = dict()
params["comet_exp_key"] = self.experiment.get_key()
return params
def save(self, save_dir):
info_dict = self.serialize()
json.dump(info_dict,
open(os.path.join(save_dir, COMET_SAVE_FILENAME), "w"))
def persist(self, config_path):
info = json.load(open(config_path, 'r'))
self.__init__(exp_key=info["comet_exp_key"])
def log_data_attributes(self, dataset_config):
for key, value in dataset_config.items():
self.experiment.log_parameter("SPECTRUM_" + key, value)
def log_imgs(self, dataset_name):
try:
imgs_dir = os.path.join(DATA_DIR, dataset_name, 'imgs')
self.experiment.log_asset_folder(imgs_dir)
except:
print(f"No images found for dataset: {dataset_name}")
def log_script(self, dataset_config):
script_name = dataset_config['matlab_script']
try:
matlab_dir = os.path.join(GEN_DIR, script_name)
self.experiment.log_asset(matlab_dir)
except:
print(f"Could not find {script_name} under {GEN_DIR}.")
def format_classification_report(self, classification_report):
return {
f'{k}_test_{metric}': metric_val
for k, v in classification_report.items()
for metric, metric_val in v.items()
}
def get_classification_report(self, y_test, preds):
preds_formatted = np.argmax(preds, axis=1)
test_formatted = np.argmax(y_test, axis=1)
peak_labels = [
f"n_peaks_{1 + num_peak}" for num_peak in range(y_test.shape[1])
]
classif_report = classification_report(test_formatted,
preds_formatted,
target_names=peak_labels,
output_dict=True)
classif_report_str = classification_report(test_formatted,
preds_formatted,
target_names=peak_labels)
if self.experiment is not None:
formatted = self.format_classification_report(classif_report)
self.experiment.log_metrics(formatted)
self.experiment.log_text(classif_report_str)
return classif_report
class CometLogger():
def __init__(self, disabled, is_existing=False, prev_exp_key=None):
"""
Handles logging of experiment to comet and also persistence to local file system.
Supports resumption of stopped experiments.
"""
if not is_existing:
self.experiment = Experiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled)
else:
if prev_exp_key is None:
raise ValueError("Requested existing experiment, but no key provided")
print("Continuing existing experiment with key: ", prev_exp_key)
self.experiment = ExistingExperiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled,
previous_experiment=prev_exp_key)
self.disabled = disabled
self.name = None
def get_experiment_key(self):
return self.experiment.get_key()[:9]
def add_tag(self, tag):
self.experiment.add_tag(tag)
def log_metric(self, name, value, step=None):
self.experiment.log_metric(name, value, step=step)
def log_metrics(self, metrics_dict, prefix, step=None):
self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step)
def log_params(self, params_dict):
self.experiment.log_parameters(params_dict)
def set_name(self, name_str):
self.experiment.set_name(name_str)
self.name = name_str
def save_act_grads(self, log_dict):
"""Save a dictionary of activation/gradients records to disk"""
assert isinstance(log_dict, dict)
if self.name is None:
warnings.warn("Experiment name not set, not saving")
return
# Save the log dictionary
file_name = f"./.{self.name}.record"
with open(file_name, 'wb') as f:
pickle.dump(log_dict, f)
# TODO: need to rewrite before can be used for MNIST.
def draw_projections(self, embeds, utterances_per_speaker, step, out_fpath=None,
max_speakers=16):
import umap
if self.disabled:
return
max_speakers = min(max_speakers, len(colormap))
embeds = embeds[:max_speakers * utterances_per_speaker]
n_speakers = len(embeds) // utterances_per_speaker
ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker)
colors = [colormap[i] for i in ground_truth]
reducer = umap.UMAP()
projected = reducer.fit_transform(embeds)
plt.scatter(projected[:, 0], projected[:, 1], c=colors)
plt.gca().set_aspect("equal", "datalim")
plt.title("UMAP projection (step %d)" % step)
if out_fpath is not None:
plt.savefig(out_fpath)
plt.clf()
self.experiment.log_image(out_fpath, step=step)
def __init__(
self,
api_key=None,
parameters=None,
experiment_key=None,
artifacts_directory=None,
credentials_path=None,
):
artifacts_directory = (artifacts_directory if artifacts_directory else
os.getenv('ARTIFACTS_DIRECTORY'))
api_key = api_key if api_key else os.getenv('COMET_API_KEY')
self.credentials_path = {
credentials_path
if credentials_path else os.getenv('PATH_TO_GOOGLE_CREDENTIALS')
}
parameters = parameters if parameters else {}
if not parameters and not experiment_key:
raise ValueError(
'Either parameters or experiment_key must be provided!')
if experiment_key is not None:
logging.info(f"Updating existing experiment: {experiment_key}")
experiment = ExistingExperiment(api_key=api_key,
previous_experiment=experiment_key)
current_parameters = self.load_experiment_parameters(
experiment_key, artifacts_directory)
parameters = deep_update(current_parameters, parameters)
else:
experiment = Experiment(
api_key=api_key,
project_name=parameters['info']['project_name'])
self.experiment_key = experiment.get_key()
self.experiment_url = experiment._get_experiment_url()
self.experiment = experiment
if 'filename' in parameters:
if os.path.exists(parameters['filename']):
with open(parameters['filename'], 'r') as f:
data = f.read()
self.experiment.set_code(data)
self.runner = DockerRunner()
self.parameters = parameters
flattened_parameters = flatten(parameters)
self.experiment.log_parameters(flattened_parameters)
self.output_folder = os.path.join(artifacts_directory,
self.experiment_key)
os.makedirs(os.path.join(self.output_folder), exist_ok=True)
fpath = os.path.join(self.output_folder, 'config.yml')
with open(fpath, 'w') as f:
yaml.safe_dump(parameters, f)
self.init_gsheet(self.credentials_path)
logging.info(
f"Set up {self.experiment_key} - {self.parameters['info']['notes']}"
)
self.experiment.end()