def meta_hook(config, command_name, logger):
experiment_meta_dir = os.path.join(config['train']['metadata_path'],
ex.path)
if command_name == 'run':
if 'connect_mongo_db' not in config['train'] or config['train'][
'connect_mongo_db']:
ex.observers.append(
MongoObserver.create(url='mongodb://192.168.77.15:27017',
db_name='sacred'))
ex.observers.append(
OverwritingFileStorageObserver.create(experiment_meta_dir))
logger.info('Storing metadata at: %s' % experiment_meta_dir)
if 'connect_slack' not in config['train'] or config['train'][
'connect_slack']:
complete_template = ":white_check_mark: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[" \
"hostname]}* completed after _{elapsed_time}_ with result=`{result}` "
interrupted_template = ":warning: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[" \
"hostname]}* interrupted after _{elapsed_time}_ "
error_template = ":x: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[hostname]}* failed " \
"after _{elapsed_time}_ with `{error}` "
slack_observer = SlackObserver(
'https://hooks.slack.com/services/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',
icon=':imp:')
slack_observer.completed_text = complete_template
slack_observer.interrupted_text = interrupted_template
slack_observer.failed_text = error_template
ex.observers.append(slack_observer)
def create_slack_observer(webhook=None):
from sacred.observers import SlackObserver
slack_obs = None
if webhook is None:
if "OBSERVERS" in SETTINGS and "SLACK" in SETTINGS.OBSERVERS:
if "WEBHOOK" in SETTINGS.OBSERVERS.SLACK:
webhook = SETTINGS.OBSERVERS.SLACK.WEBHOOK
slack_obs = SlackObserver(webhook)
else:
slack_obs = SlackObserver(webhook)
if slack_obs is None:
print('Failed to create Slack observer.')
return slack_obs
def apply(cls, args, run):
a = SacredExperimentAccess()
if run.config["log_level"] == 'info':
logging.getLogger().setLevel(logging.INFO)
elif run.config["log_level"] == 'debug':
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
try:
if len(a.get_experiments(config=run.config, complete=True)) > 0:
logger.info("Experiment has already been run - don't bother!")
logger.info("Note that this will **not** show up in sacred")
sys.exit()
mongo = MongoObserver.create(url="localhost:9001", db_name=args)
run.observers.append(mongo)
except pymongo.errors.ServerSelectionTimeoutError:
logger.warning("Could not connect to MongoDB database. Experiment will still run, but you won't be able to plot results!")
try:
run.observers.append(
SlackObserver.from_config(run.config["slack_json_file"])
)
except FileNotFoundError:
logger.warning("Slack json file not found - not sending slack notifications")
run.info = {
**run.info,
"test": False
}
def setup_experiment(experiment_name):
mongo_uri = 'mongodb://*****:*****@localhost:27017/sacred?authSource=admin'
ex = Experiment(experiment_name, save_git_info=False)
ex.observers.append(MongoObserver(url=mongo_uri, db_name='sacred'))
slack_obs = SlackObserver.from_config(os.environ['SLACK_CONFIG'])
ex.observers.append(slack_obs)
return ex
def add_slack_observer(experiment):
slack_config_path = Path('~/slack.json').expanduser()
if not slack_config_path.is_file():
print('Slack config path not found:', slack_config_path)
slack_observer = None
else:
slack_observer = SlackObserver.from_config(str(slack_config_path))
experiment.observers.append(slack_observer)
return slack_observer
def create_base_experiment(
sacred_args,
name=None): #path=Config.LOG_DIR, db_name=Config.DB_NAME):
ex = Experiment(name)
print(name)
ex.capture(set_device)
ex.main(main)
ex.capture(run_train)
ex.command(evaluate_experiment)
ex.command(test_config)
ex.command(show_options)
ex.command(evaluate_checkpoint)
# set default values
ex = config_builder.build(ex)
# set observers but check if maybe sacred will create them
# on its own
""" TODO
Problem is that we create the experiment before the command line is parsed by sacred.
But then we cannot set default values without using a shell script. Or
modify the file path for the logger."""
if Config.LOG_DIR is not None and '-F' not in sacred_args:
path = Config.LOG_DIR
if name is not None:
path = os.path.join(path, name)
file_ob = FileStorageObserver.create(path)
ex.observers.append(file_ob)
if Config.SLACK_WEBHOOK_URL != "":
slack_ob = SlackObserver(Config.SLACK_WEBHOOK_URL)
ex.observers.append(slack_ob)
if (Config.MONGO_DB_NAME is not None and Config.MONGO_DB_NAME != "") \
and '-m' not in sacred_args:
if Config.MONGO_USER != "":
mongo_ob = MongoObserver.create(username=Config.MONGO_USER,
password=Config.MONGO_PW,
url=Config.MONGO_URL,
authMechanism="SCRAM-SHA-256",
db_name=Config.MONGO_DB_NAME)
else:
mongo_ob = MongoObserver.create(url=Config.MONGO_URL,
db_name=Config.MONGO_DB_NAME)
ex.observers.append(mongo_ob)
return ex
from sacred import Experiment
from sacred.observers import FileStorageObserver, SlackObserver
from graspy.datasets import load_drosophila_left
from graspy.utils import binarize, symmetrize, is_fully_connected
from graspy.plot import heatmap
from src.models import select_sbm, select_rdpg
from src.utils import compute_mse_from_assignments
import matplotlib.pyplot as plt
ex = Experiment("Drosophila model selection 3 - SBM, RDPG, tSBM")
current_file = basename(__file__)[:-3]
sacred_file_path = Path(f"./maggot_models/models/runs/{current_file}")
slack_obs = SlackObserver.from_config("slack.json")
file_obs = FileStorageObserver.create(sacred_file_path)
ex.observers.append(slack_obs)
ex.observers.append(file_obs)
@ex.config
def config():
# Variables defined in config get automatically passed to main
n_block_try_range = list(range(1, 11)) # noqa: F841
n_components_try_range = list(range(1, 13)) # noqa: F841
n_components_try_rdpg = list(range(1, 13)) # noqa: F841
directed = True # noqa: F841
# Create experiment, assign the name if provided in env variables
if os.environ.get('EXPERIMENT_NAME') is not None:
ex = Experiment(os.environ.get('EXPERIMENT_NAME'))
else:
ex = Experiment('POIS')
# Set a File Observer
if os.environ.get('SACRED_RUNS_DIRECTORY') is not None:
print("Sacred logging at:", os.environ.get('SACRED_RUNS_DIRECTORY'))
ex.observers.append(
FileStorageObserver.create(os.environ.get('SACRED_RUNS_DIRECTORY')))
if os.environ.get('SACRED_SLACK_CONFIG') is not None:
print("Sacred is using slack.")
ex.observers.append(
SlackObserver.from_config(os.environ.get('SACRED_SLACK_CONFIG')))
@ex.config
def custom_config():
seed = 0
env = 'rllab.cartpole'
num_episodes = 100
episode_cap = False
max_iters = 500
horizon = 500
file_name = 'progress'
logdir = 'logs'
step_size = 0.1
njobs = -1
policy = 'nn'
from sacred.observers import FileStorageObserver
from sacred.observers import MongoObserver
from sacred.observers import SlackObserver
from sacred.utils import apply_backspaces_and_linefeeds
EXPERIMENT_NAME = 'experiment'
DATABASE_NAME = 'experiments'
URL_NAME = 'mongodb://localhost:27017/'
ex = Experiment()
ex.observers.append(FileStorageObserver.create('results_features'))
#ex.observers.append(MongoObserver.create(url=URL_NAME, db_name=DATABASE_NAME))
ex.captured_out_filter = apply_backspaces_and_linefeeds
#Send a message to slack if the run is succesfull or if it failed
slack_obs = SlackObserver.from_config('slack.json')
ex.observers.append(slack_obs)
#Device
if torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
def log_scalars(results, name_dataset):
"""Log scalars of the results for MongoDB and Omniboard
Args:
results: Results with the loss, accuracy, recall, precision and f1-score
name_dataset: The name of the dataset so it can store the scalers by name
"""
loss = nn.functional.mse_loss(polished_model.matrix()[:, trainable.perm, 0], trainable.target_matrix)
loss.backward()
return loss
for i in range(N_LBFGS_STEPS):
optimizer.step(closure)
torch.save(polished_model.state_dict(), str((Path(trial.logdir) / trial._checkpoint.value).parent / 'polished_model.pth'))
loss = nn.functional.mse_loss(polished_model.matrix()[:, trainable.perm, 0], trainable.target_matrix)
return loss.item()
ex = Experiment('VandermondeEval_factorization')
ex.observers.append(FileStorageObserver.create('logs'))
slack_config_path = Path('config/slack.json') # Add webhook_url there for Slack notification
if slack_config_path.exists():
ex.observers.append(SlackObserver.from_config(str(slack_config_path)))
@ex.named_config
def softmax_config():
fixed_order = False # Whether the order of the factors are fixed
softmax_fn = 'softmax' # Whether to use softmax (+ semantic loss) or sparsemax
@ex.named_config
def sparsemax_config():
fixed_order = False # Whether the order of the factors are fixed
softmax_fn = 'sparsemax' # Whether to use softmax (+ semantic loss) or sparsemax
@ex.config
from baselines.policy.mlp_policy import MlpPolicy
from baselines.pois import pois
import baselines.common.tf_util as U
import time
import os
import tensorflow as tf
from baselines.pois.parallel_sampler import ParallelSampler
from sacred import Experiment
from sacred.observers import FileStorageObserver, SlackObserver
# Create experiment
ex = Experiment('POIS')
# Set a File Observer
ex.observers.append(FileStorageObserver.create('sacred_runs'))
ex.observers.append(SlackObserver.from_config('../../configs/slack.json'))
@ex.config
def custom_config():
seed = 0
env = 'cartpole'
num_episodes = 100
max_iters = 500
horizon = 500
iw_method = 'is'
iw_norm = 'none'
natural = False
file_name = 'progress'
logdir = '.'
bound = 'max-d2'
delta = 0.99
def maybe_add_slack(ex):
if os.path.exists('slack.json'):
slack_obs = SlackObserver.from_config('slack.json')
ex.observers.append(slack_obs)
print('Added slack observer.')
