def meta_hook(config, command_name, logger): experiment_meta_dir = os.path.join(config['train']['metadata_path'], ex.path) if command_name == 'run': if 'connect_mongo_db' not in config['train'] or config['train'][ 'connect_mongo_db']: ex.observers.append( MongoObserver.create(url='mongodb://192.168.77.15:27017', db_name='sacred')) ex.observers.append( OverwritingFileStorageObserver.create(experiment_meta_dir)) logger.info('Storing metadata at: %s' % experiment_meta_dir) if 'connect_slack' not in config['train'] or config['train'][ 'connect_slack']: complete_template = ":white_check_mark: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[" \ "hostname]}* completed after _{elapsed_time}_ with result=`{result}` " interrupted_template = ":warning: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[" \ "hostname]}* interrupted after _{elapsed_time}_ " error_template = ":x: *{experiment[name]}: {_id}* from *{config[user]}* on *{host_info[hostname]}* failed " \ "after _{elapsed_time}_ with `{error}` " slack_observer = SlackObserver( 'https://hooks.slack.com/services/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', icon=':imp:') slack_observer.completed_text = complete_template slack_observer.interrupted_text = interrupted_template slack_observer.failed_text = error_template ex.observers.append(slack_observer)
def create_slack_observer(webhook=None): from sacred.observers import SlackObserver slack_obs = None if webhook is None: if "OBSERVERS" in SETTINGS and "SLACK" in SETTINGS.OBSERVERS: if "WEBHOOK" in SETTINGS.OBSERVERS.SLACK: webhook = SETTINGS.OBSERVERS.SLACK.WEBHOOK slack_obs = SlackObserver(webhook) else: slack_obs = SlackObserver(webhook) if slack_obs is None: print('Failed to create Slack observer.') return slack_obs
def apply(cls, args, run): a = SacredExperimentAccess() if run.config["log_level"] == 'info': logging.getLogger().setLevel(logging.INFO) elif run.config["log_level"] == 'debug': logging.getLogger().setLevel(logging.DEBUG) else: logging.getLogger().setLevel(logging.INFO) try: if len(a.get_experiments(config=run.config, complete=True)) > 0: logger.info("Experiment has already been run - don't bother!") logger.info("Note that this will **not** show up in sacred") sys.exit() mongo = MongoObserver.create(url="localhost:9001", db_name=args) run.observers.append(mongo) except pymongo.errors.ServerSelectionTimeoutError: logger.warning("Could not connect to MongoDB database. Experiment will still run, but you won't be able to plot results!") try: run.observers.append( SlackObserver.from_config(run.config["slack_json_file"]) ) except FileNotFoundError: logger.warning("Slack json file not found - not sending slack notifications") run.info = { **run.info, "test": False }
def setup_experiment(experiment_name): mongo_uri = 'mongodb://*****:*****@localhost:27017/sacred?authSource=admin' ex = Experiment(experiment_name, save_git_info=False) ex.observers.append(MongoObserver(url=mongo_uri, db_name='sacred')) slack_obs = SlackObserver.from_config(os.environ['SLACK_CONFIG']) ex.observers.append(slack_obs) return ex
def add_slack_observer(experiment): slack_config_path = Path('~/slack.json').expanduser() if not slack_config_path.is_file(): print('Slack config path not found:', slack_config_path) slack_observer = None else: slack_observer = SlackObserver.from_config(str(slack_config_path)) experiment.observers.append(slack_observer) return slack_observer
def create_base_experiment( sacred_args, name=None): #path=Config.LOG_DIR, db_name=Config.DB_NAME): ex = Experiment(name) print(name) ex.capture(set_device) ex.main(main) ex.capture(run_train) ex.command(evaluate_experiment) ex.command(test_config) ex.command(show_options) ex.command(evaluate_checkpoint) # set default values ex = config_builder.build(ex) # set observers but check if maybe sacred will create them # on its own """ TODO Problem is that we create the experiment before the command line is parsed by sacred. But then we cannot set default values without using a shell script. Or modify the file path for the logger.""" if Config.LOG_DIR is not None and '-F' not in sacred_args: path = Config.LOG_DIR if name is not None: path = os.path.join(path, name) file_ob = FileStorageObserver.create(path) ex.observers.append(file_ob) if Config.SLACK_WEBHOOK_URL != "": slack_ob = SlackObserver(Config.SLACK_WEBHOOK_URL) ex.observers.append(slack_ob) if (Config.MONGO_DB_NAME is not None and Config.MONGO_DB_NAME != "") \ and '-m' not in sacred_args: if Config.MONGO_USER != "": mongo_ob = MongoObserver.create(username=Config.MONGO_USER, password=Config.MONGO_PW, url=Config.MONGO_URL, authMechanism="SCRAM-SHA-256", db_name=Config.MONGO_DB_NAME) else: mongo_ob = MongoObserver.create(url=Config.MONGO_URL, db_name=Config.MONGO_DB_NAME) ex.observers.append(mongo_ob) return ex
from sacred import Experiment from sacred.observers import FileStorageObserver, SlackObserver from graspy.datasets import load_drosophila_left from graspy.utils import binarize, symmetrize, is_fully_connected from graspy.plot import heatmap from src.models import select_sbm, select_rdpg from src.utils import compute_mse_from_assignments import matplotlib.pyplot as plt ex = Experiment("Drosophila model selection 3 - SBM, RDPG, tSBM") current_file = basename(__file__)[:-3] sacred_file_path = Path(f"./maggot_models/models/runs/{current_file}") slack_obs = SlackObserver.from_config("slack.json") file_obs = FileStorageObserver.create(sacred_file_path) ex.observers.append(slack_obs) ex.observers.append(file_obs) @ex.config def config(): # Variables defined in config get automatically passed to main n_block_try_range = list(range(1, 11)) # noqa: F841 n_components_try_range = list(range(1, 13)) # noqa: F841 n_components_try_rdpg = list(range(1, 13)) # noqa: F841 directed = True # noqa: F841
# Create experiment, assign the name if provided in env variables if os.environ.get('EXPERIMENT_NAME') is not None: ex = Experiment(os.environ.get('EXPERIMENT_NAME')) else: ex = Experiment('POIS') # Set a File Observer if os.environ.get('SACRED_RUNS_DIRECTORY') is not None: print("Sacred logging at:", os.environ.get('SACRED_RUNS_DIRECTORY')) ex.observers.append( FileStorageObserver.create(os.environ.get('SACRED_RUNS_DIRECTORY'))) if os.environ.get('SACRED_SLACK_CONFIG') is not None: print("Sacred is using slack.") ex.observers.append( SlackObserver.from_config(os.environ.get('SACRED_SLACK_CONFIG'))) @ex.config def custom_config(): seed = 0 env = 'rllab.cartpole' num_episodes = 100 episode_cap = False max_iters = 500 horizon = 500 file_name = 'progress' logdir = 'logs' step_size = 0.1 njobs = -1 policy = 'nn'
from sacred.observers import FileStorageObserver from sacred.observers import MongoObserver from sacred.observers import SlackObserver from sacred.utils import apply_backspaces_and_linefeeds EXPERIMENT_NAME = 'experiment' DATABASE_NAME = 'experiments' URL_NAME = 'mongodb://localhost:27017/' ex = Experiment() ex.observers.append(FileStorageObserver.create('results_features')) #ex.observers.append(MongoObserver.create(url=URL_NAME, db_name=DATABASE_NAME)) ex.captured_out_filter = apply_backspaces_and_linefeeds #Send a message to slack if the run is succesfull or if it failed slack_obs = SlackObserver.from_config('slack.json') ex.observers.append(slack_obs) #Device if torch.cuda.is_available(): device = torch.device("cuda") else: device = torch.device("cpu") def log_scalars(results, name_dataset): """Log scalars of the results for MongoDB and Omniboard Args: results: Results with the loss, accuracy, recall, precision and f1-score name_dataset: The name of the dataset so it can store the scalers by name """
loss = nn.functional.mse_loss(polished_model.matrix()[:, trainable.perm, 0], trainable.target_matrix) loss.backward() return loss for i in range(N_LBFGS_STEPS): optimizer.step(closure) torch.save(polished_model.state_dict(), str((Path(trial.logdir) / trial._checkpoint.value).parent / 'polished_model.pth')) loss = nn.functional.mse_loss(polished_model.matrix()[:, trainable.perm, 0], trainable.target_matrix) return loss.item() ex = Experiment('VandermondeEval_factorization') ex.observers.append(FileStorageObserver.create('logs')) slack_config_path = Path('config/slack.json') # Add webhook_url there for Slack notification if slack_config_path.exists(): ex.observers.append(SlackObserver.from_config(str(slack_config_path))) @ex.named_config def softmax_config(): fixed_order = False # Whether the order of the factors are fixed softmax_fn = 'softmax' # Whether to use softmax (+ semantic loss) or sparsemax @ex.named_config def sparsemax_config(): fixed_order = False # Whether the order of the factors are fixed softmax_fn = 'sparsemax' # Whether to use softmax (+ semantic loss) or sparsemax @ex.config
from baselines.policy.mlp_policy import MlpPolicy from baselines.pois import pois import baselines.common.tf_util as U import time import os import tensorflow as tf from baselines.pois.parallel_sampler import ParallelSampler from sacred import Experiment from sacred.observers import FileStorageObserver, SlackObserver # Create experiment ex = Experiment('POIS') # Set a File Observer ex.observers.append(FileStorageObserver.create('sacred_runs')) ex.observers.append(SlackObserver.from_config('../../configs/slack.json')) @ex.config def custom_config(): seed = 0 env = 'cartpole' num_episodes = 100 max_iters = 500 horizon = 500 iw_method = 'is' iw_norm = 'none' natural = False file_name = 'progress' logdir = '.' bound = 'max-d2' delta = 0.99
def maybe_add_slack(ex): if os.path.exists('slack.json'): slack_obs = SlackObserver.from_config('slack.json') ex.observers.append(slack_obs) print('Added slack observer.')