def main() -> None:
args = get_args()
config = get_bunch_config_from_json(args.config)
comet_experiment = Experiment(
api_key=config.comet_api_key,
project_name=config.comet_project_name,
workspace=config.comet_workspace,
disabled=not config.use_comet_experiments,
)
comet_experiment.set_name(config.experiment_name)
comet_experiment.log_parameters(config)
test_tweets = load_test_tweets(config.test_data_path)
client = LanguageServiceClient()
result = []
predictions = np.zeros(len(test_tweets), dtype=np.int32)
for i, tweet in enumerate(test_tweets):
start_iter_timestamp = time.time()
document = types.Document(
type=enums.Document.Type.PLAIN_TEXT, content=tweet, language="en"
)
response = client.analyze_sentiment(document=document)
response_dict = MessageToDict(response)
result.append(response_dict)
prediction_present = bool(response_dict["documentSentiment"])
if prediction_present:
# -1, 1 predictions
predictions[i] = 2 * (response.document_sentiment.score > 0) - 1
print("iteration", i, "took:", time.time() - start_iter_timestamp, "seconds")
comet_experiment.log_asset_data(result, name="google_nlp_api_response.json")
ids = np.arange(1, len(test_tweets) + 1).astype(np.int32)
predictions_table = np.column_stack((ids, predictions))
if comet_experiment.disabled:
save_path = build_save_path(config)
os.makedirs(save_path)
formatted_predictions_table = pd.DataFrame(
predictions_table, columns=["Id", "Prediction"], dtype=np.int32,
)
formatted_predictions_table.to_csv(
os.path.join(save_path, "google_nlp_api_predictions.csv"), index=False
)
else:
comet_experiment.log_table(
filename="google_nlp_api_predictions.csv",
tabular_data=predictions_table,
headers=["Id", "Prediction"],
)
percentage_predicted = np.sum(predictions != 0) / predictions.shape[0]
comet_experiment.log_metric(name="percentage predicted", value=percentage_predicted)
class CometTracker:
def __init__(self, comet_params, run_params=None, prev_exp_id=None):
if prev_exp_id: # previous experiment
api_key = comet_params['api_key']
del comet_params[
'api_key'] # removing this because the rest of the items need to be passed
self.experiment = ExistingExperiment(
api_key=api_key,
previous_experiment=prev_exp_id,
**comet_params)
print(
f'In CometTracker: ExistingExperiment initialized with id: {prev_exp_id}'
)
else: # new experiment
self.experiment = Experiment(**comet_params)
self.experiment.log_parameters(run_params)
def track_metric(self, metric, value, step):
self.experiment.log_metric(metric, value, step)
def add_tags(self, tags):
self.experiment.add_tags(tags)
print(f'In [add_tags]: Added these tags to the new experiment: {tags}')
def set_name(self, name):
self.experiment.set_name(name)
def train_autofeature_model(data_path,
embedding_dimension,
batch_size):
training_data = load_data(data_path)
X = training_data.drop(labels=['SK_ID_CURR'], axis=1)
experiment = Experiment(
api_key=API_KEY, project_name="home-credit")
experiment.set_name(
'home-credit-autofeature-selection')
model = build_model(X.shape[1], int(embedding_dimension))
model.summary()
model.compile(optimizer='adam', loss='mean_squared_logarithmic_error')
model.fit(X, X,
epochs=5,
batch_size=int(batch_size))
experiment.log_multiple_params(
{"embedding_dimension": embedding_dimension,
"batch_size": batch_size})
model.save(
'home-credit-encoder-{}-{}.hdf5'.format(
embedding_dimension, batch_size))
def get_experiment_objects():
args = CLIParser().parse_args()
comet = Experiment(api_key="<your_key>",
project_name="<your_project>",
workspace="<your_workspace>",
log_code=False,
auto_param_logging=False,
auto_metric_logging=False,
disabled=args.no_comet,
display_summary=False)
if not args.continue_train:
# else args.save is the directory from which assets are loaded to
# continue training
args.save = "{}-{}".format(args.save, time.strftime("%Y%m%d-%H%M%S"))
create_exp_dir(args)
comet.set_name(args.save.split('/')[-1])
comet.log_parameters(vars(args))
copy_assets(args, comet)
last_state = None
if args.continue_train:
last_state = torch.load(os.path.join(args.save, 'state.pt'))
continue_random(last_state, args.cuda)
else:
init_random(args.seed, args.cuda)
return args, comet, last_state
def train(path):
name = os.path.splitext(os.path.basename(path))[0]
print('Processing: ', name)
features = pd.read_csv(path, index_col=None)
selected_features_names = [name for name, desc in selected_features]
features = features[selected_features_names]
split_idx = 1200
features = features.drop(['sound.files'], axis=1)
noise_only_df, df = features.iloc[:split_idx], features.iloc[split_idx:]
y = df.pop('petrel')
X = df.values
y_noise = noise_only_df.pop('petrel')
X_noise = noise_only_df.values
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
hyperparams = {
'n_estimators': [100, 300, 500, 1000],
'learning_rate': [0.1],
'gamma': [0.0, 0.5],
'max_depth': [2, 3, 4],
'min_child_weight': [1, 2],
'subsample': [1.0, 0.8],
'reg_alpha': [0.0, 0.1],
'reg_lambda': [1, 2, 3]
}
#
# hyperparams = {
# 'n_estimators': [100],
# 'learning_rate': [0.1],
# 'gamma': [0.0],
# 'max_depth': [2],
# 'min_child_weight': [1],
# 'subsample': [1.0],
# 'reg_alpha': [0.0],
# 'reg_lambda': [1]
# }
clf = model_selection.GridSearchCV(estimator=xg.XGBClassifier(objective='binary:logistic', n_jobs=-1),
param_grid=hyperparams,
cv=4)
fit_params = clf.fit(X_train, y_train)
estimator = fit_params.best_estimator_
joblib.dump(estimator, name + '_model.pkl')
test_pred = estimator.predict(X_test)
metrics = calculate_metrics(test_pred, y_test)
noise_pred = estimator.predict(X_noise)
noise_detection_accuracy = accuracy_score(y_noise, noise_pred)
experiment = Experiment(api_key="4PdGdUZmGf6P8QsMa5F2zB4Ui",
project_name="storm petrels",
workspace="tracewsl")
experiment.set_name(name)
experiment.log_parameter('name', name)
experiment.log_multiple_params(fit_params.best_params_)
experiment.log_multiple_metrics(metrics)
experiment.log_metric('Noise detection accuracy', noise_detection_accuracy)
experiment.log_figure('Confusion matrix', get_confusion_matrix_figure(test_pred, y_test))
experiment.log_figure('Feature importnace', get_feature_importance_figure(estimator, list(df.columns.values)))
def main():
precision = torch.float
# Parameter and Object declarations
env_params = {
"data path": "C:/Users/aaa2cn/Documents/nao_data/",
"ip": "localhost",
"port": 52232,
"score type": "score" # Aggregate error in pose
}
env = env_factory.make_env("nao", "pose assumption", env_params)
# Make a pool object
model_params = {
"precision": precision,
"weight initialization scheme": "Sparse",
"grad": False
}
model = model_factory.make_model("NAO FC model", model_params)
# Make an algorithm object
alg_params = {
"target": env.target,
"minimization mode": env.minimize,
"minimum entropy": 0.1,
"tolerance": 0.1,
"max steps": 64,
"memory size": 10
}
alg = algorithm_factory.make_alg("local search", model, alg_params)
experiment = Experiment(api_key="5xNPTUDWzZVquzn8R9oEFkUaa",
project_name="nao",
workspace="aromorin")
experiment.set_name("Pose Assumption virtual")
hyper_params = {
"Algorithm": "LS",
"Parameterization": 35000,
"Decay Factor": 0.01,
"Directions": 10,
"Search Radius": 0.1
}
experiment.log_parameters(hyper_params)
slv_params = {"environment": env, "algorithm": alg, "logger": experiment}
slv = solver_factory.make_slv("robot", slv_params)
slv.solve(iterations=5000)
slv.save_elite_weights(path='', name='pose_assump_virtual')
# Recreate the target pose
alg.eval()
pred = alg.model(env.observation)
angles = [p.item() for p in pred]
print("These are the angles: ")
print(angles)
env.set_joints(angles)
env.say("Is this the pose you set for me?")
env.rest()
def experiment_flag(parser_com_dis, parser_name) :
if not bool(parser_com_dis):
experiment = Experiment(api_key="PUT YOUR COMET API KEY",
project_name="spoof19", workspace="WORKSPACE",
disabled=bool(parser_com_dis))
experiment.set_name(parser_name)
return experiment
return []
def generate_categories():
# capture the config path from the run arguments then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except ValueError:
print("Missing or invalid arguments")
exit(0)
print("Logging experiment name: {name}".format(
name=config.experiment.experiment_name))
experiment = Experiment(api_key=config.experiment.api_key,
project_name=config.experiment.project_name,
workspace=config.experiment.workspace)
experiment.set_name(config.experiment.experiment_name)
print('Creating the data loader...')
data_loader = DataLoader(config.defects_summarizer.paths)
train_data, test_data = data_loader.get_data()
print('Creating the Preprocessor...')
preprocessor = CorexPreprocessor(train_data, config)
preprocessor.prepare_data()
print('Loading and evaluating the Model...')
model = CorexModel(config.defects_summarizer, preprocessor, seed=False)
trainer = CorexTrainer(model, preprocessor.get_data())
trainer.train()
trainer.generate_topics()
top_docs_df = trainer.get_top_documents(
config.defects_summarizer.evaluate.extract_topics,
preprocessor.get_raw_corpus(),
config.defects_summarizer.evaluate.extraction_quantile,
labels=True)
top_docs_df.to_csv(config.defects_summarizer.paths.save_data_path)
print('Saving the trained topic model...')
model.save()
print('Preprocessing the summarizer...')
summary_preprocessor = TextRankPreprocessor(
top_docs_df, n_docs=config.defects_summarizer.evaluate.n_docs)
summary_preprocessor.prepare_data()
print('Loading and evaluating the summarizer...')
summary_model = TextRankModel(config)
summary_trainer = TextRankTrainer(summary_model, summary_preprocessor)
avg_prec, avg_recall, avg_f1 = summary_trainer.train_and_evaluate(
test_data)
# Log the rest of the experiment
metrics = {"precision": avg_prec, "recall": avg_recall, "f1": avg_f1}
experiment.log_metrics(metrics)
experiment.log_model(
name=config.experiment.model_name,
file_or_folder=config.labels_generator.paths.save_model_path)
def main() -> None:
args = get_args()
config = get_bunch_config_from_json(args.config)
comet_experiment = Experiment(
api_key=config.comet_api_key,
project_name=config.comet_project_name,
workspace=config.comet_workspace,
disabled=not config.use_comet_experiments,
)
comet_experiment.set_name(config.experiment_name)
comet_experiment.log_parameters(config)
if config.model == "randomforest":
classifier = GloveEmbeddingsClassifier(
RandomForestClassifier(random_state=config.random_seed))
elif config.model == "logregression":
classifier = GloveEmbeddingsClassifier(
LogisticRegression(solver="saga", random_state=config.random_seed))
elif config.model == "decisiontree":
classifier = GloveEmbeddingsClassifier(
DecisionTreeClassifier(random_state=config.random_seed))
else:
raise ValueError("chosen model not available")
training_features, training_labels = classifier.generate_training_data(
config)
best_model, best_model_score, best_model_params = classifier.run_grid_search(
config.random_seed, config.model_parameters, training_features,
training_labels)
comet_experiment.log_metric("mean accuracy", best_model_score)
comet_experiment.log_parameters(best_model_params)
test_data_features = classifier.generate_test_data_features(config)
ids = np.arange(1, test_data_features.shape[0] + 1)
predictions = best_model.predict(test_data_features)
predictions_table = np.stack([ids, predictions], axis=-1).astype(int)
if comet_experiment.disabled:
save_path = build_save_path(config)
os.makedirs(save_path)
formatted_predictions_table = pd.DataFrame(
predictions_table,
columns=["Id", "Prediction"],
dtype=np.int32,
)
formatted_predictions_table.to_csv(os.path.join(
save_path, "test_predictions.csv"),
index=False)
else:
comet_experiment.log_table(
filename="test_predictions.csv",
tabular_data=predictions_table,
headers=["Id", "Prediction"],
)
def main(args):
tensorboard_writer = None
comet_experiment = None
if (not args.no_log):
# Create necessary directories
if (not os.path.isdir(args.log_dir)):
os.mkdir(args.log_dir)
# Create log_dir for run
run_log_dir = os.path.join(args.log_dir, args.exp_name)
if (os.path.isdir(run_log_dir)):
cur_count = len(glob.glob(run_log_dir + "_*"))
run_log_dir = run_log_dir + "_" + str(cur_count)
os.mkdir(run_log_dir)
# Create tensorboard writer if requested
if (args.tensorboard):
tensorboard_dir = os.path.join(run_log_dir, "tensorboard")
writer = SummaryWriter(log_dir=tensorboard_dir)
# Create comet experiment if requested
if (args.comet_config is not None):
with open(args.comet_config, 'r') as f:
comet_dict = json.load(f)
comet_experiment = Experiment(
api_key=comet_dict["api_key"],
project_name=comet_dict["project_name"],
workspace=comet_dict["workspace"],
)
comet_experiment.set_name(args.exp_name)
# Get hash for latest git commit for logging
last_commit_hash = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).decode("utf-8").rstrip()
comet_experiment.log_parameter("git_commit_id", last_commit_hash)
# Instantiate dataset
dynamics_data = Maze2DDataset()
dataloader = DataLoader(dynamics_data, batch_size=128, shuffle=True)
agent = Morel(4,
2,
tensorboard_writer=tensorboard_writer,
comet_experiment=comet_experiment)
agent.train(dataloader, dynamics_data)
if (not args.no_log):
agent.save(os.path.join(run_log_dir, "models"))
if comet_experiment is not None:
upload_assets(comet_experiment, run_log_dir)
agent.eval(dynamics_data.env)
def log_file_to_comet_output(project_name, exp_name, log_file_path,
refresh_rate):
experiment = Experiment(api_key=os.environ["COMET_API_KEY"],
project_name=project_name)
experiment.set_name(exp_name)
with open(log_file_path, "r") as f:
while True:
print(f.read(), end='')
time.sleep(refresh_rate)
def setup_comet_ml_experiment(api_key, project_name, experiment_name,
parameters, tags):
""" Function for setting up comet ml experiment """
experiment = Experiment(api_key=api_key,
project_name=project_name,
auto_metric_logging=False)
experiment.set_name(experiment_name)
experiment.log_parameters(parameters)
experiment.add_tags(tags)
return experiment
def setup_comet_ml(dpmodel):
if dpmodel.config["train"]["comet_ml"]["track"]:
experiment = Experiment(
api_key=dpmodel.config["train"]["comet_ml"]["api_key"],
project_name=dpmodel.config["train"]["comet_ml"]["project_name"])
if "experiment_name" in dpmodel.config["train"]["comet_ml"].keys():
experiment.set_name(
dpmodel.config["train"]["comet_ml"]["experiment_name"])
else:
experiment = None
return experiment
def create_experiment(config: dict) -> Experiment:
experiment = None
key = config.get('api_key', None)
name = config.get('project_name', None)
workspace = config.get('workspace', None)
if None not in (key, name, workspace):
experiment = Experiment(
api_key=key,
project_name=name,
workspace=workspace)
experiment.set_name(f'{datetime.datetime.now().time()}')
return experiment
def main(cmd=None, stdout=True):
"""Run finetuning experiment for fixed seed."""
# Initialize system
args = get_args(cmd)
assert torch.cuda.is_available()
torch.cuda.set_device(args.device)
# Initialize logging
model_id = ("Finetune {}, seed size {}, epochs {}, labels {}, "
"batch size {}, lr {}").format(args.model, args.seed_size,
args.epochs, args.label_budget,
args.batch_size, args.lr)
logging.basicConfig(filename="{}/{}.txt".format(args.dout, model_id),
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
if stdout:
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logger = Experiment(comet_ml_key, project_name="ActiveDialogue")
logger.set_name(model_id)
logger.log_parameters(vars(args))
# Select model and environment
if args.model == "glad":
model_arch = GLAD
elif args.model == "gce":
model_arch = GCE
env = DSTEnv(load_dataset, model_arch, args)
# Load seed if need-be
if not env.load('seed'):
raise ValueError("No loaded seed.")
# Initialize evaluation
best_metrics = env.metrics(True)
for k, v in best_metrics.items():
logger.log_metric(k, v, step=0)
logging.info("Initial metrics: %s", best_metrics)
# Finetune
env.label_all()
for epoch in range(1, args.epochs + 1):
logging.info('Starting fit epoch %d.', epoch)
env.fit()
metrics = env.metrics(True)
logging.info("Epoch metrics: %s", metrics)
for k, v in metrics.items():
logger.log_metric(k, v, step=epoch)
if best_metrics is None or metrics[args.stop] > best_metrics[
args.stop]:
logging.info("Saving best!")
best_metrics = metrics
def start_comet(args):
exp = None
if args.comet is not None and len(args.comet) > 0:
workspace, project, apikey = args.comet.split("/")
exp = Experiment(api_key=apikey,
project_name=project,
workspace=workspace)
exp.set_name("td3")
if len(args.comet_tags) > 0:
comet_tags = args.comet_tags.split(",")
for tag in comet_tags:
exp.add_tag(tag)
return exp
def opus_wrapper(**kwargs):
from comet_ml import Experiment
import torch
import torch.nn.functional as F
import os
import os.path as osp
import argparse
from data import load_dataset
from torch_geometric.datasets import Planetoid,PPI,TUDataset
import torch_geometric.transforms as T
from torch_geometric.nn import GATConv, GCNConv, GAE, VGAE
from torch_geometric.data import DataLoader
from maml import meta_gradient_step
from models import Encoder, MetaEncoder, GraphSignature, MetaMLPEncoder, MetaSignatureEncoder, MetaGatedSignatureEncoder
from utils import global_test, test, seed_everything
from collections import OrderedDict
from torchviz import make_dot
import wandb
import ipdb
os.environ['WANDB_API_KEY'] = "7110d81f721ee9a7da84c67bcb319fc902f7a180"
parser = argparse.ArgumentParser()
my_args = parser.parse_args([])
my_args.__dict__.update(kwargs)
my_args.dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("Checking CUDA")
print(my_args.dev)
if my_args.dataset=='PPI':
project_name = 'meta-graph-ppi'
elif my_args.dataset=='REDDIT-MULTI-12K':
project_name = "meta-graph-reddit"
elif my_args.dataset=='FIRSTMM_DB':
project_name = "meta-graph-firstmmdb"
elif my_args.dataset=='DD':
project_name = "meta-graph-dd"
elif my_args.dataset=='AMINER':
project_name = "meta-graph-aminer"
else:
project_name='meta-graph'
if my_args.comet:
experiment = Experiment(api_key=my_args.comet_apikey,\
project_name=project_name,\
workspace=my_args.comet_username)
experiment.set_name(my_args.namestr)
my_args.experiment = experiment
if my_args.wandb:
wandb.init(project=project_name,name=my_args.namestr)
print(my_args)
return main(my_args)
def __train_config(self):
name = f"{self.config.SERVER_OPT}: {self.config.SERVER_LEARNING_RATE} - {self.config.CLIENT_OPT_STRATEGY} - {self.config.CLIENT_OPT}: {self.config.CLIENT_LEARNING_RATE}"
logging.info(name)
experiment = Experiment(
workspace=self.workspace, project_name=self.project_name
)
experiment.set_name(name)
learner = self.Learner(experiment, self.config, self.config_technical)
try:
learner.train()
except ToLargeLearningRateExcpetion:
pass # TODO
self.__refresh_df()
def do_training(config: TorchFederatedLearnerCIFAR100Config):
config_technical = TorchFederatedLearnerTechnicalConfig(
STORE_OPT_ON_DISK=False,
STORE_MODEL_IN_RAM=False,
)
name = f"{config.SERVER_OPT}: {config.SERVER_LEARNING_RATE} - {config.CLIENT_OPT_STRATEGY} - {config.CLIENT_OPT}: {config.CLIENT_LEARNING_RATE}"
logging.info(name)
experiment = Experiment(workspace="federated-learning",
project_name=project_name)
experiment.set_name(name)
learner = TorchFederatedLearnerCIFAR100(experiment, config,
config_technical)
learner.train()
def generate_topics():
# capture the config path from the run arguments then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except ValueError:
print("Missing or invalid arguments")
exit(0)
print("Logging experiment name: {name}".format(
name=config.experiment.experiment_name))
experiment = Experiment(api_key=config.experiment.api_key,
project_name=config.experiment.project_name,
workspace=config.experiment.workspace)
experiment.set_name(config.experiment.experiment_name)
params = config.labels_generator.model
experiment.log_parameters(params)
print('Creating the data loader...')
data_loader = DataLoader(config.labels_generator.paths)
data = data_loader.get_data()
print('Creating the Preprocessor...')
preprocessor = CorexPreprocessor(data, config)
preprocessor.prepare_data()
print('Creating and training the Model...')
model = CorexModel(config, preprocessor)
trainer = CorexTrainer(model, preprocessor.get_data())
trainer.train()
print('Evaluating the model...')
coherence_lst, avg_coherence = trainer.evaluate(preprocessor.get_data(),
preprocessor.get_corpus())
trainer.generate_topics()
print("Coherence score: {score_lst} \nAvg coherence score: {avg_score}".
format(score_lst=coherence_lst, avg_score=avg_coherence))
print('Saving the trained model...')
model.save()
# Log the rest of the experiment
metrics = {"coherence": avg_coherence}
experiment.log_metrics(metrics)
experiment.log_model(
name=config.experiment.model_name,
file_or_folder=config.labels_generator.paths.save_model_path)
def make_experiment(env_file, name=None, tags=None):
# Get environment values
load_dotenv(env_file)
COMETML_KEY = os.environ.get("COMETML_KEY")
COMETML_PROJECT = os.environ.get("COMETML_PROJECT")
# Start and configure experiment
experiment = Experiment(COMETML_KEY, COMETML_PROJECT)
if name is not None:
experiment.set_name(name)
if tags is not None:
experiment.add_tags(tags)
return experiment
class CometMlAdapter(BaseAdapter):
def __init__(self, api_key, project_name, experiment_name):
self.experiment = Experiment(api_key=api_key,
project_name=project_name)
self.experiment.set_name(experiment_name)
def log_parameters(self, hyper_params):
self.experiment.log_parameters(hyper_params)
def set_model_graph(self, graph):
self.experiment.set_model_graph(graph)
def log_metric(self, name, metric, step):
self.experiment.log_metric(name, metric, step=step)
def register(self, name):
pass
def training_loop(x,
y,
cfg,
exp_name="exp_1",
device=torch.device('cuda'),
starting_epoch=1):
model = init_model(cfg, device)
datasets = init_datasets(x, y, cfg)
dataloaders = init_loaders(datasets, cfg)
optim = init_optimizers(cfg, model)
n_epochs = cfg.TRAIN.MAX_EPOCHS
experiment = Experiment(project_name='mbm-pos-metrics',
api_key='w4JbvdIWlas52xdwict9MwmyH')
experiment.set_name(exp_name)
hyper = {
'train_batch_size': cfg.TRAIN.BATCH_SIZE,
'age_lambda': cfg.MODEL.AGE_LAMBDA,
'n_epochs': cfg.TRAIN.MAX_EPOCHS,
'learning_rate': cfg.TRAIN.LEARNING_RATE,
'weight_decay': cfg.MODEL.WEIGHT_DECAY
}
experiment.log_parameters(hyper)
print("Starting training loop!")
for epoch in range(starting_epoch, n_epochs + 1):
for phase in ['train', 'val', 'test']:
print("Epoch {}/{} - {}".format(epoch, n_epochs, phase.upper()))
forward_model(model,
dataloaders[phase],
optim,
cfg,
device,
experiment,
phase,
current_epoch=epoch)
# checkpoint model
check = {
'model': model.state_dict(),
'optim': optim['optimizer'].state_dict()
}
torch.save(
check,
"../../storage/mbm/checkpoints/{}_{}.pth".format(exp_name, epoch))
experiment.end()
return experiment
def setup_exp(args):
exp_name = ("nb_" if args.test_notebook else "") + "_".join(
[args.task, args.mode, args.embedder_name,
get_hyp_str(args)])
exp_kwargs = dict(project_name="self-supervised-survey",
workspace="eracah")
if args.comet_mode == "online":
from comet_ml import Experiment
exp_kwargs.update(api_key="kH9YI2iv3Ks9Hva5tyPW9FAbx")
elif args.comet_mode == "offline":
from comet_ml.offline import OfflineExperiment as Experiment
offline_directory = Path(".logs")
exp_kwargs.update(offline_directory=str(offline_directory))
experiment = Experiment(**exp_kwargs)
experiment.set_name(exp_name)
experiment.log_parameters(args.__dict__)
return experiment, experiment.id
class CometTracker:
def __init__(self, comet_params, experiment_name=None, run_params=None):
self.experiment = Experiment(**comet_params)
if run_params is not None:
self.experiment.log_parameters(run_params)
if experiment_name is not None:
self.experiment.set_name(experiment_name)
def track_metric(self, metric, value, step=None):
self.experiment.log_metric(metric, value, step)
def add_tags(self, tags):
self.experiment.add_tags(tags)
print(f'In [add_tags]: Added these tags to the new experiment: {tags}')
def set_name(self, name):
self.experiment.set_name(name)
def get_comet_logger(self):
if not self.paras.load :
comet_exp = Experiment(project_name=COMET_PROJECT_NAME,
workspace=COMET_WORKSPACE,
auto_output_logging=None,
auto_metric_logging=None,
display_summary=False,
)
if self.paras.transfer:
comet_exp.set_name(self.exp_name)
comet_exp.add_tag(Path(self.ckpdir).parent.name)
comet_exp.add_tag('transfer')
comet_exp.add_tag(self.config['data']['corpus']['metas'][0])
if self.paras.test:
comet_exp.set_name(Path(self.paras.outdir).name)
comet_exp.add_tag(Path(self.paras.config).parents[2].name)
comet_exp.add_tag('test')
comet_exp.add_tag(Path(self.paras.config).parent.stem)
#comet_exp.add_tag(Path(self.paras.outdir).name)
else:
comet_exp.add_tag('train')
for name, param in self.config.items():
if isinstance(param, dict):
comet_exp.log_parameters(param, prefix=name)
else:
comet_exp.log_parameter(name, param)
comet_exp.log_other('seed', self.paras.seed)
with open(Path(self.logdir,'exp_key'), 'w') as f:
print(comet_exp.get_key(),file=f)
else:
with open(Path(self.logdir,'exp_key'),'r') as f:
exp_key = f.read().strip()
comet_exp = ExistingExperiment(previous_experiment=exp_key,
project_name=COMET_PROJECT_NAME,
workspace=COMET_WORKSPACE,
auto_output_logging=None,
auto_metric_logging=None,
display_summary=False,
)
return comet_exp
def setup_comet(args, init_distributed):
if init_distributed and args.distributed_rank>0: # Only the rank 0 process handled comet
args.comet = False
if args.comet: # This will only be true if the user set the --comet flag, and the rank is 0
print('Activating comet')
experiment = Experiment(api_key=api_key,
project_name=args.comet_project, workspace=workspace,
auto_param_logging=False, auto_metric_logging=False,
parse_args=True, auto_output_logging=True, log_env_gpu=False
)
# experiment.disable_mp() # Turn off monkey patching
experiment.log_parameters(vars(args))
# experiment.add_tag(args.comet_tag)
experiment.set_name(args.comet_tag)
experiment.add_tag(args.comet_real_tag)
print("* Finished comet setup... ")
return experiment
else:
return None
def main():
"""Entry point."""
random_seed = 69420
print(random_seed, "random_seed")
seed_libraries(random_seed)
flags = _parse_args(sys.argv[1:])
experiment = Comet_Experiment(
api_key=flags.comet_api_key,
project_name="ideal-pancake",
workspace="s0lvang",
)
experiment.set_name(flags.experiment_name)
# Set up config and select datasets
globals.init(
experiment=experiment,
_flags=flags,
)
# Trigger the experiment
experiment = ExperimentManager(flags.datasets)
experiment.run_experiments()
def setup_comet_ml():
"""Initialise Experiment object."""
experiment = Experiment(
api_key=config.COMET_API_KEY,
disabled=True if not config.COMET_MONITOR else False,
log_code=False,
project_name=config.COMET_PROJECT_NAME,
workspace=config.COMET_WORKSPACE,
)
experiment.set_name(config.EXPERIMENT_NAME)
experiment.log_others({
"conditioning": config.EXPERIMENT_Z,
"dataset": config.EXPERIMENT_DATASET,
})
experiment.log_parameters({
"batch_size": config.EXPERIMENT_BATCH_SIZE,
"epochs": config.EXPERIMENT_EPOCHS,
})
return experiment
def trainLoop(num_epochs, batch_size, gen_lr, disc_lr, gen_train_freq, disc_train_freq, logger):
# Create run directory for current training run
os.chdir(os.path.expanduser('~') + '/psig-gan/runs/')
run_dir = str(datetime.datetime.now()).replace(' ','',) + "--" + "disc_lr"+ str(disc_lr) + " " + "gen_lr"+ str(gen_lr) + " " + "gen_train_freq" + str(gen_train_freq) + " " + "disc_train_freq" + str(disc_train_freq) + " " + "num_epochs"+ str(num_epochs)
os.mkdir(run_dir)
# Instantiate GAN
gan = DCGAN(latent_shape=100, output_image_shape=256, num_gen_images=batch_size, gen_filter_size=5, discrim_filter_size=5, gen_num_channels=128, discrim_num_channels=64)
# Load real grassweeds image data
data_path = '/home/data/dcgan-data/'
data_batch = util.createDataBatch(data_path,batch_size)
# Define Comet-ML API key here for error logging
comet_api_key = 'Gdy4QDrOmu0P01XuBI33rPuIS'
#Define Comet Project Name
logger = Experiment(comet_api_key, project_name="psig-gan")
#Define experiment name
logger.set_name("disc_lr"+ str(disc_lr) + " " + "gen_lr"+ str(gen_lr) + " " + "gen_train_freq"+ str(gen_train_freq) + " " + "disc_train_freq" + str(disc_train_freq) + " " + "num_epochs"+ str(num_epochs))
# Execute training loop
train(gan, data_batch, num_epochs, run_dir, gen_lr, disc_lr, gen_train_freq, disc_train_freq, logger)
def run_main_loop(args, train_estimator, predict_estimator):
total_steps = 0
train_steps = math.ceil(args.train_examples / args._batch_size)
eval_steps = math.ceil(args.eval_examples / args._batch_size)
if args.use_comet:
experiment = Experiment(api_key=comet_ml_api_key, project_name=comet_ml_project, workspace=comet_ml_workspace)
experiment.log_parameters(vars(args))
experiment.add_tags(args.tag)
experiment.set_name(model_name(args))
else:
experiment = None
prefetch_inception_model()
with tf.gfile.Open(os.path.join(suffixed_folder(args, args.result_dir), "eval.txt"), "a") as eval_file:
for epoch in range(0, args.epochs, args.predict_every):
logger.info(f"Training epoch {epoch}")
train_estimator.train(input_fn=train_input_fn, steps=train_steps * args.predict_every)
total_steps += train_steps * args.predict_every
if args.use_comet:
experiment.set_step(epoch)
# logger.info(f"Evaluate {epoch}")
# evaluation = predict_estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)
# logger.info(evaluation)
# save_evaluation(args, eval_file, evaluation, epoch, total_steps)
# if args.use_comet:
# experiment.log_metrics(evaluation)
logger.info(f"Generate predictions {epoch}")
predictions = predict_estimator.predict(input_fn=predict_input_fn)
logger.info(f"Save predictions")
save_predictions(args, suffixed_folder(args, args.result_dir), eval_file, predictions, epoch, total_steps, experiment)
logger.info(f"Completed {args.epochs} epochs")
