def fetch_parents(current_path, parents=[]): tmp_ex = Experiment('treeqn') tmp_ex.add_config(current_path) with suppress_stdout(): tmp_ex.run("print_config") if tmp_ex.current_run is not None and "parent_config" in tmp_ex.current_run.config: return fetch_parents(tmp_ex.current_run.config["parent_config"], [current_path] + parents) else: return [current_path] + parents
path = "./conf/jack.yaml" def fetch_parents(current_path, parents=[]): tmp_ex = Experiment('jack') tmp_ex.add_config(current_path) if "parent_config" in tmp_ex.configurations[0]._conf: return fetch_parents(tmp_ex.configurations[0]._conf["parent_config"], [current_path] + parents) else: return [current_path] + parents configs = fetch_parents(path) logger.info("Loading {}".format(configs)) ex = Experiment('jack') for path in configs: ex.add_config(path) class Duration(object): def __init__(self): self.t0 = time() self.t = time() def __call__(self): logger.info('Time since last checkpoint : {0:.2g}min'.format((time() - self.t) / 60.)) self.t = time() checkpoint = Duration() logging.basicConfig(level=logging.INFO)
from mot_neural_solver.utils.misc import make_deterministic, get_run_str_and_save_dir from mot_neural_solver.path_cfg import OUTPUT_PATH, DATA_PATH import os.path as osp from mot_neural_solver.pl_module.pl_module import MOTNeuralSolver from mot_neural_solver.utils.evaluation import compute_mot_metrics import pandas as pd from sacred import SETTINGS SETTINGS.CONFIG.READ_ONLY_CONFIG = False ex = Experiment() ex.add_config('configs/tracking_cfg.yaml') ex.add_config({ 'ckpt_path': 'output/experiments/07-08_09:07_train_weighedge/checkpoints/epoch_025.ckpt', 'run_id': 'evaluation', 'add_date': True, 'precomputed_embeddings': True }) @ex.automain def main(_config, _run): sacred.commands.print_config(_run) make_deterministic(12345)
('plotter.informationplane', [])#, # ('plotter.snr', []), # ('plotter.informationplane_movie', []), #('plotter.activations', []) ] else: plotters = [ ('plotter.informationplane', [])#, #('plotter.snr', []), #('plotter.informationplane_movie', []), #('plotter.activations', []), #('plotter.activations_single_neuron', []) ] ex.add_config('configs/basic.json') @ex.capture def load_dataset(dataset): module = importlib.import_module(dataset) return module.load() @ex.capture def load_model(model, architecture, activation_fn, optimizer, learning_rate, input_size, output_size, max_norm_weights, initial_bias): module = importlib.import_module(model) return module.load(architecture, activation_fn, optimizer, learning_rate, input_size, output_size, max_norm_weights, initial_bias)
from ....run.online.blocks.RunDQN import RunDQN from .... import constants from ....constants import Constants from ....models.h**o.AAF import AAF from ....modules.hand_obs.ResUCatEncoder import ResUCatEncoder from .... import paths from ....utils.logger import Logger from ....utils import sacred_utils from ....utils.policy.DynaPartitionedEpsGreedyPolicy import DynaPartitionedEpsGreedyPolicy ex = Experiment("blocks_DQN_AAF") if constants.MONGO_URI is not None and constants.DB_NAME is not None: ex.observers.append(MongoObserver(url=constants.MONGO_URI, db_name=constants.DB_NAME)) else: print("WARNING: results are not being saved. See 'Setup MongoDB' in README.") ex.add_config(paths.CFG_BLOCKS_DEFAULT_ENV) ex.add_config(paths.CFG_BLOCKS_DEFAULT_DECONSTRUCTION_PLANNER) ex.add_config(paths.CFG_BLOCKS_DEFAULT_DQN) ex.add_config(paths.CFG_BLOCKS_AAF_MODEL) @ex.config def config(): prob_threshold = 0.5 positive_prob = 0.8 aaf_load_path = None @ex.capture(prefix="AAF_model_config") def get_aaf_model_config(balance_loss):
import os.path as osp import os import numpy as np import yaml import cv2 import torch from torch.utils.data import DataLoader from tracktor.config import get_output_dir, get_tb_dir from tracktor.reid.solver import Solver from tracktor.datasets.factory import Datasets from tracktor.reid.resnet import resnet50 ex = Experiment() ex.add_config('experiments/cfgs/reid.yaml') Solver = ex.capture(Solver, prefix='reid.solver') @ex.automain def my_main(_config, reid): # set all seeds torch.manual_seed(reid['seed']) torch.cuda.manual_seed(reid['seed']) np.random.seed(reid['seed']) torch.backends.cudnn.deterministic = True print(_config) output_dir = osp.join(get_output_dir(reid['module_name']), reid['name'])
"""A standard machine learning task without much sacred magic.""" from sacred import Experiment from sacred.observers import FileStorageObserver from sklearn import svm, datasets, model_selection ex = Experiment("svm") ex.observers.append( FileStorageObserver.create("my_runs") ) ex.add_config({ # Configuration is explicitly defined as dictionary. "C": 1.0, "gamma": 0.7, "kernel": "rbf", "seed": 42 }) def get_model(C, gamma, kernel): return svm.SVC(C=C, kernel=kernel, gamma=gamma) @ex.main # Using main, command-line arguments will not be interpreted in any special way. def run(_config): X, y = datasets.load_breast_cancer(return_X_y=True) X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2) clf = get_model(_config["C"], _config["gamma"], _config["kernel"]) # Parameters are passed explicitly. clf.fit(X_train, y_train) return clf.score(X_test, y_test)
from mot_neural_solver.utils.misc import make_deterministic, get_run_str_and_save_dir from mot_neural_solver.path_cfg import OUTPUT_PATH, DATA_PATH import os.path as osp from mot_neural_solver.pl_module.pl_module import MOTNeuralSolver from mot_neural_solver.utils.evaluation import compute_mot_metrics import pandas as pd from sacred import SETTINGS SETTINGS.CONFIG.READ_ONLY_CONFIG=False ex = Experiment() ex.add_config('configs/tracking_cfg.yaml') ex.add_config({'run_id': 'evaluation', 'add_date': True, 'precomputed_embeddings': True}) @ex.automain def main(_config, _run): #sacred.commands.print_config(_run) # No need to print config, as it's overwritten by the one from the ckpt. make_deterministic(12345) run_str, save_dir = get_run_str_and_save_dir(_config['run_id'], None, _config['add_date']) out_files_dir = osp.join(save_dir, 'mot_files') # Load model from checkpoint and update config entries that may vary from the ones used in training model = MOTNeuralSolver.load_from_checkpoint(checkpoint_path=_config['ckpt_path'] if osp.exists(_config['ckpt_path']) else osp.join(OUTPUT_PATH, _config['ckpt_path']))
sys.path.append('../src') from training.handlers import Tracer # Set up experiment ex = Experiment(name='decoders', ingredients=[dataset, model, training]) # Observers ex.observers.append(FileStorageObserver.create('../data/sims/decoders')) # ex.observers.append(MongoObserver.create(url='127.0.0.1:27017', # db_name='disent')) # General configs ex.add_package_dependency('torch', torch.__version__) ex.add_config(no_cuda=False, save_folder='../data/sims/temp/dsprites') # Functions @ex.capture def set_seed_and_device(seed, no_cuda): np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available() and not no_cuda: torch.cuda.manual_seed(seed) device = torch.device('cuda') else: device = torch.device('cpu') return device
if not isinstance(current_path, list): current_path = [current_path] all_paths = list(current_path) for p in current_path: tmp_ex.add_config(p) if "parent_config" in tmp_ex.configurations[-1]._conf: all_paths = fetch_parents( tmp_ex.configurations[-1]._conf["parent_config"]) + all_paths return all_paths configs = fetch_parents(path) logger.info("Loading {}".format(configs)) ex = Experiment('jack') for c_path in configs: ex.add_config(c_path) class Duration(object): def __init__(self): self.t0 = time() self.t = time() def __call__(self): logger.info('Time since last checkpoint : {0:.2g}min'.format( (time() - self.t) / 60.)) self.t = time() checkpoint = Duration()
from mot_neural_solver.utils.misc import make_deterministic, get_run_str_and_save_dir, ModelCheckpoint from mot_neural_solver.path_cfg import OUTPUT_PATH import os.path as osp from mot_neural_solver.pl_module.pl_module import MOTNeuralSolver from pytorch_lightning import Trainer from pytorch_lightning.loggers import TensorBoardLogger #from pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint from sacred import SETTINGS SETTINGS.CONFIG.READ_ONLY_CONFIG = False ex = Experiment() ex.add_config('configs/tracking_cfg.yaml') ex.add_config({ 'run_id': 'train_w_default_config', 'add_date': True, 'cross_val_split': None }) @ex.config def cfg(cross_val_split, eval_params, dataset_params, graph_model_params, data_splits): # Training requires the use of precomputed embeddings assert dataset_params[ 'precomputed_embeddings'], "Training without precomp. embeddings is not supp"
import argparse import os from types import SimpleNamespace from sacred import Experiment from experiments.train_dqn_distributed.main_actor import main_actor from experiments.train_dqn_distributed.main_learner import main_learner ex = Experiment("train_dqn_distributed") base_config_path = os.path.dirname(__file__) + "/../dqn_base_config.json" ex.add_config(base_config_path) config_path = os.path.dirname(__file__) + "/config.json" ex.add_config(config_path) @ex.automain def main(_run, _config): parser = argparse.ArgumentParser() parser.add_argument("--actor", type=int) parser.add_argument("--learner_address", type=str) args, _ = parser.parse_known_args() config = SimpleNamespace(**_config) run_id = _run._id if args.learner_address: config.distributed["learner_address"] = args.learner_address if args.actor: main_actor(run_id, config, args.actor)
def __init__(self, f_main, f_config, f_capture, observer_type='file', mongo_url='mongodb://localhost:27017', verbose=False): """ :param f_main: function The main function for the experiment :param f_config: function or str The function where all the sacred parameters are init or a file with the config parameters :param f_capture: function The function that implements the metrics logging API with sacred (should be used with Lambda in keras but has problem right now. Thus it can be ignored) :param mongo_url: str The url for MongoDB :param verbose: bool If True logging is enabled """ self.sacred_db_name() ex = Experiment(self.sacred_ex_name()) ex.captured_out_filter = apply_backspaces_and_linefeeds if observer_type == 'mongodb': print('Connecting to MongoDB at {}:{}'.format( mongo_url, self.sacred_db_name())) ex.observers.append( MongoObserver.create(url=mongo_url, db_name=self.sacred_db_name())) elif observer_type == 'file': basedir = os.path.join(config['logs'], 'sacred') ex.observers.append(FileStorageObserver.create(basedir)) else: raise ValueError( '{} is not a valid type for a SACRED observer.'.format( observer_type)) if hasattr(f_config, '__call__'): # init the experiment configuration using a function ex.config(f_config) elif isinstance(f_config, str): # init the experiment configuration usinga file ex.add_config(f_config) elif isinstance(f_config, dict): # init the experiment configuration usinga file ex.add_config(f_config) else: raise ValueError( 'You should provide either a fucntion or a config file for setting up an experiemet.' 'The given paramter has type {} which is not valid'.format( type(f_config))) # init the experiment logging (capture) method f_ex_capture = ex.capture(f_capture) # init the experiment main @ex.main def ex_main(_run): if observer_type == 'mongodb': return main_wrapper(f_main, ex, f_ex_capture, self.sacred_db_name(), _run) else: f_main(ex, _run, f_ex_capture) self.ex = ex
from tqdm import tqdm import sacred from sacred import Experiment from tracktor.frcnn_fpn import FRCNN_FPN from tracktor.config import get_output_dir from tracktor.datasets.factory import Datasets from tracktor.oracle_tracker import OracleTracker from tracktor.motion import Seq2Seq, CorrelationSeq2Seq, RelativeCorrelationModel from tracktor.tracker import Tracker from tracktor.reid.resnet import resnet50 from tracktor.utils import interpolate, plot_sequence, get_mot_accum, evaluate_mot_accums mm.lap.default_solver = 'lap' ex = Experiment() ex.add_config('experiments/cfgs/tracktor.yaml') # hacky workaround to load the corresponding configs and not having to hardcode paths here ex.add_config(ex.configurations[0]._conf['tracktor']['reid_config']) ex.add_named_config('oracle', 'experiments/cfgs/oracle_tracktor.yaml') ex.add_named_config('correlation_model', 'experiments/cfgs/correlation_model.yaml') ex.add_named_config('pos_model', 'experiments/cfgs/pos_model.yaml') ex.add_named_config('cva_model', 'experiments/cfgs/cva_model.yaml') @ex.automain def main(tracktor, reid, _config, _log, _run): sacred.commands.print_config(_run) # set all seeds
import os from sys import platform as _platform import cPickle as pickle import numpy as np import cv2 from spikefuel import tools, gui, helpers exp = Experiment("DVS Recording - UCF-50") exp.add_config({ "ucf50_dir": "", "ucf50_stats_path": "", "recording_save_path": "", "viewer_id": 1, "screen_height": 0, "screen_width": 0, "work_win_scale": 0.9, "bg_color": [255, 0, 0], "fps": 0 }) @exp.automain def dvs_ucf50_exp(ucf50_dir, ucf50_stats_path, recording_save_path, viewer_id, screen_height, screen_width, work_win_scale,
import os import cPickle as pickle import numpy as np import cv2 from spikefuel import tools, gui, helpers exp = Experiment("DVS Recording - Lipreading") exp.add_config({ "lipreading_dir": "", "lipreading_stats_path": "", "recording_save_path": "", "viewer_id": 1, "screen_height": 0, "screen_width": 0, "work_win_scale": 0.9, "bg_color": [255, 0, 0], "fps": 0 }) @exp.automain def dvs_lipreading_exp(lipreading_dir, lipreading_stats_path, recording_save_path, viewer_id, screen_height, screen_width, work_win_scale,
dataset.add_config('configs/dummy-dataset.yaml') training.add_config('configs/dummy-training.yaml') # Configs for both models. Should execute using 'with model.<config>' model.add_named_config('lstm', 'configs/dummy-lstm.yaml') # Create experiment ex = Experiment( name='Dummy Experiment', ingredients=[dataset, model, training] ) # Runtime options save_folder = '../../data/sims/test/' ex.add_config({ 'save': save_folder, 'no_cuda': False, }) # Add dependencies ex.add_package_dependency('torch', torch.__version__) # Add observer ex.observers.append( FileStorageObserver.create(save_folder)) # ex.observers.append( # MongoObserver.create(url='127.0.0.1:27017', # db_name='MY_DB') # ) @ex.capture def log_training(tracer):
Hello world! INFO - 02_hello_config_dict - Completed after 0:00:00 The message can also easily be changed using the ``with`` command-line argument:: $ ./02_hello_config_dict.py with message='Ciao world!' WARNING - 02_hello_config_dict - No observers have been added to this run INFO - 02_hello_config_dict - Running command 'main' INFO - 02_hello_config_dict - Started Ciao world! INFO - 02_hello_config_dict - Completed after 0:00:00 """ from sacred import Experiment ex = Experiment() # We add message to the configuration of the experiment here ex.add_config({"message": "Hello world!"}) # Equivalent: # ex.add_config( # message="Hello world!" # ) # notice how we can access the message here by taking it as an argument @ex.automain def main(message): print(message)
torch._utils._rebuild_tensor_v2 except AttributeError: def _rebuild_tensor_v2(storage, storage_offset, size, stride, requires_grad, backward_hooks): tensor = torch._utils._rebuild_tensor(storage, storage_offset, size, stride) tensor.requires_grad = requires_grad tensor._backward_hooks = backward_hooks return tensor torch._utils._rebuild_tensor_v2 = _rebuild_tensor_v2 ex = Experiment() ex.add_config('experiments/cfgs/tracktor.yaml') # hacky workaround to load the corresponding configs and not having to hardcode paths here ex.add_config(ex.configurations[0]._conf['tracktor']['reid_network_config']) ex.add_config(ex.configurations[0]._conf['tracktor']['obj_detect_config']) ex.add_named_config('oracle', 'experiments/cfgs/oracle_tracktor.yaml') # Tracker = ex.capture(Tracker, prefix='tracker.tracker') @ex.automain def my_main(tracktor, siamese, _config): # set all seeds torch.manual_seed(tracktor['seed']) torch.cuda.manual_seed(tracktor['seed']) np.random.seed(tracktor['seed'])
import numpy as np from keras.callbacks import ModelCheckpoint from keras.callbacks import CSVLogger import spiker from spiker.models import resnet from spiker.data import ddd17 exp = Experiment("ResNet - Steering - Experiment") exp.add_config({ "model_name": "", # the model name "data_name": "", # the data name "channel_id": 0, # which channel to chose, 0: dvs, 1: aps, 2: both "stages": 0, # number of stages "blocks": 0, # number of blocks of each stage "filter_list": [], # number of filters per stage "nb_epoch": 0, # number of training epochs "batch_size": 0, # batch size "frame_cut": [], # cut frames }) @exp.automain def resnet_exp(model_name, data_name, channel_id, stages, blocks, filter_list, nb_epoch, batch_size, frame_cut): """Perform ResNet experiment.""" model_path = os.path.join(spiker.HOME, "data", "exps", "ral-exps", model_name) if not os.path.isdir(model_path): os.makedirs(model_path)
import time from sacred import Experiment from sacred.observers import FileStorageObserver from ingredients_func.dataset_ingredients import dataset_ingredient, get_input from ingredients_func.data_cleaning import cleaner_ingredient, ApplicationCleaning, apply_cleaners from src.validation import CrossValidationSplitter from src.models import CatBoostClassifierCV ex = Experiment(name="kaggle-home-credit-competition-pipeline", ingredients=[dataset_ingredient, cleaner_ingredient]) ex.observers.append(FileStorageObserver("runs")) ex.add_config("config.yaml") @ex.capture def get_cv(cv_strategy, n_folds, shuffle, cv_seed): return CrossValidationSplitter(cv_strategy, n_folds, shuffle, cv_seed) @ex.automain def main(_run): _run.add_artifact("scrd.py") _run.add_artifact("ingredients_func/dataset_ingredients.py") _run.add_artifact("ingredients_func/data_cleaning.py") _run.add_artifact("model.pkl") print(f"{time.ctime()}, pipeline start.") data, target = get_input() splitter = get_cv()
import torch.utils.data from semantic8_dataset import DatasetTrainVal as Dataset import lightconvpoint.utils.metrics as metrics from lightconvpoint.utils import get_network # SACRED from sacred import Experiment from sacred import SETTINGS from sacred.utils import apply_backspaces_and_linefeeds from sacred.config import save_config_file SETTINGS.CAPTURE_MODE = "sys" # for tqdm ex = Experiment("Semantic8") ex.captured_out_filter = apply_backspaces_and_linefeeds # for tqdm ex.add_config("semantic8.yaml") ###### class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' # wrap blue / green
learning_exp_decay: float = 2.0 learning_rollout_steps_clip: int = 3 task_size: int = 5 n_experts: int = 2 save_period: int = 1000 image_save_period: int = 10000 attention_beta: float = 5 attention_operation: str = 'euclidean_distance' description: str = 'randomized experts and task_size' model_name: str = '2' ex.add_config(dataclasses.asdict(Params(random.randint(0, 2 ** 32)))) def load_agent(experiment_id, epoch, agent): reader = SacredReader(experiment_id, SacredConfigFactory.local()) reader.load_model(agent, 'agent', epoch) def create_agent(p: Params) -> SearchAgent: return SearchAgent(hidden_state_size=p.hidden_state_size, key_size=p.key_size, id_size=p.id_size, value_size=p.value_size, input_size=1, n_inputs=p.task_size, onehot_expert_ids=p.onehot_expert_ids, attention_beta=p.attention_beta, attention_operation=AttentionOperation.from_string(p.attention_operation), learning_rate=p.learning_rate, model_name=p.model_name).to(default_device())
# torch imports import torch import torch.nn.functional as F import torch.utils.data # SACRED from sacred import Experiment from sacred import SETTINGS from sacred.utils import apply_backspaces_and_linefeeds from sacred.config import save_config_file SETTINGS.CAPTURE_MODE = "sys" # for tqdm ex = Experiment("Shapenet") ex.captured_out_filter = apply_backspaces_and_linefeeds # for tqdm ex.add_config("shapenet.yaml") ###### def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) @ex.automain def main(_run, _config): print(_config) savedir_root = _config["training"]["savedir"] device = torch.device(_config["misc"]["device"])
from sacred import Experiment from sacred.observers import FileStorageObserver from lib.datasets.config import get_output_dir, cfg from lib.datasets.datasets import Datasets from lib.utils import plot_sequence, plot_sequence_ids_frames, plot_sequence_ids_frames_kalman, get_mot_accum, evaluate_mot_accums from lib.detector.fake_detector import FakeDetector17, FakeDetector20, FakeDetector15 from lib.my_tracker import MyTracker from lib.network.complete_net import completeNet ex = Experiment('Tracking') ex.observers.append( FileStorageObserver('../gdrive/MyDrive/KamMOT_trenowanie/logs')) ex.add_config('config/tracking.yaml') @ex.automain def my_main(_config, _log, _run, tracking): #torch.manual_seed(tracking['seed']) #torch.cuda.manual_seed(tracking['seed']) #np.random.seed(tracking['seed']) #torch.backends.cudnn.deterministic = True print(_config) output_dir = osp.join( get_output_dir(tracking['name']) + '_runID_' + _run._id) if not osp.exists(output_dir):
conf_name = config.conf_name valid_fold = int(config.fold) train(valid_fold, conf_name) if __name__ == '__main__': """" python -u customer/reg.py main with conf_name=5cls_resnet34 fold=0 """ from sacred.arg_parser import get_config_updates import sys config_updates, named_configs = get_config_updates(sys.argv[1:]) conf_name = config_updates.get('conf_name') fold = config_updates.get('fold') locker = task_locker( 'mongodb://*****:*****@10.10.20.103:27017/db?authSource=admin', remove_failed=9, version=version) task_id = f'lung_{conf_name}_{fold}' #pydevd_pycharm.settrace('192.168.1.101', port=1234, stdoutToServer=True, stderrToServer=True) with locker.lock_block(task_id=task_id) as lock_id: if lock_id is not None: ex.add_config({ 'lock_id': lock_id, 'lock_name': task_id, 'version': version, }) res = ex.run_commandline()
model_saved_path = cp.save_model( exp_path, policy, total_timesteps, episode_num, num_samples, replay_buffer, envs_train_names, args, ) print("*** training finished and model saved to {} ***".format( model_saved_path)) if __name__ == "__main__": args = get_args() if not args.debug: # I was using mongodb to store the results, insert your credentials if you want to use it as well. # For code release, I switched that to a FileStorage observer, but I haven't tested it properly. # db_url = "mongodb://{user}:{pwd}@DBSERVER:DBPORT/{dbname}?".format( # user="******", # pwd="DBPASSWORD", # dbname="DBNAME", # ) # mongo_client = setup_mongodb(db_url, DBNAME) ex.observers.append(FileStorageObserver(f"{DATA_DIR}/{args.expID}")) ex.add_config(vars(args)) ex.run() else: train()
def run_experiment(config, reproduce_result=None): # interactive mode for jupyter notebooks, DEACTIVATES REPODUCIBILITY SAFEGUARDS ex = Experiment('DAM') # mongo uri is in a non-pushed login.py file # make a new login.py file with a mongo URI that looks like this: # mongodb+srv://my-username:[email protected]/sacred?retryWrites=true # connect to client client = pymongo.MongoClient(MONGO_URI) ex.observers.append(MongoObserver.create(client=client)) print("config debug", config) # add config to ex manually, instead of creating @ex.config functions ex.add_config({'seed': SEED}) ex.add_config(config) tf.compat.v1.random.set_random_seed(SEED) @ex.capture def my_metrics(_run, logs): if not config.get('use_capsnet'): _run.log_scalar("loss", float(logs.get('loss'))) _run.log_scalar("acc", float(logs.get('accuracy'))) _run.log_scalar("val_loss", float(logs.get('val_loss'))) _run.log_scalar("val_acc", float(logs.get('val_accuracy'))) _run.result = float(logs.get('val_accuracy')) # a callback to log to Sacred, found here: https://www.hhllcks.de/blog/2018/5/4/version-your-machine--models-with-sacred class LogMetrics(Callback): def on_epoch_end(self, _, logs={}): my_metrics(logs=logs) print("ran config, now initializating run") @ex.main def run(_run): # Load configs, if parameters are unspecified, fill in a default config = _run.config run = config.get('fit_params') model_params = config.get('model_params') data_params = config.get('data_params') batch_size = data_params.get('batch_size') augmentations = data_params.get('augmentations') buffer_size = data_params.get('buffer_size') # the buffer sizes for shuffling use_sampling = data_params.get('use_sampling') class_target_prob = 1 / model_params.get('num_classes') print("[!] list of parameter configurations") pprint(config) # Load data and define generators ------------------------------------------ print("[!] loading datasets \n") x_train, x_val, x_test, probs = load_data() # get a rough estimate: there are 100 files per TFRecord # except for one TFRecord per item, so this estimate might not be 100% correct num_training = len(x_train) * 100 # TF parsing functions print("[!] Creating dataset iterators \n") # Load the dataset iterators train_dataset = create_training_dataset(x_train, batch_size, buffer_size, augmentations, use_sampling, probs, class_target_prob, **model_params) val_dataset = validate(x_val, batch_size, **model_params) test_dataset = validate(x_test, batch_size, **model_params) # we need the actual labels from the TFRecords, but they take INCREDIBLY long to parse # parse through them once, and create a csv file with a list of all the labels # note: the tf parsing requires that there is no randomness (shuffling) in the validation/test labels if not os.path.exists('../datasets/data/valid/val_labels.csv'): print(os.path.exists('../datasets/data/valid/val_labels.csv')) print("[!] creating validation label file in ../datasets/data/valid/val_labels.csv") create_label_csv(val_dataset,'../datasets/data/valid/val_labels.csv') else: print("[!] validation labels csv exist") if not os.path.exists('../datasets/data/test/test_labels.csv'): print("[!] creating test label file in ../datasets/data/test/test_labels.csv") create_label_csv(test_dataset,'../datasets/data/test/test_labels.csv') else: print("[!] test labels csv exist") # load the file with validation labels # getting labels from a TFRecords with lots of other data is horribly slow... print("[!] Loading validation labels for callbacks") val_labels = pd.read_csv('../datasets/data/valid/val_labels.csv') val_labels = np.squeeze(val_labels.to_numpy()) # Model definitions -------------------------------------------------------- print("[!] compiling model and adding callbacks \n") # function for building the model model_func = model_dict[run.get('model')] # invoke the user function model = model_func(**model_params) model.summary() # compile the model with catcrossentropy: one hot encoded labels!! model.compile(optimizer= tf.keras.optimizers.Adam(run.get('lr')), loss= 'categorical_crossentropy', metrics=['accuracy']) # Model callbacks ---------------------------------------------------------- # ReduceLRonPlateau if run.get('reduce_lr_on_plateau'): reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, min_lr=10e-7, verbose=1) else: reduce_lr = Callback() # Model checkpoints now = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") aug_string = 'aug' if augmentations==True else 'noaug' modelcheckpoint_name= lambda x: "checkpoints/model-{}-{}-{}-{}-{}.hdf5".format(run.get('model'), x, aug_string, 'ch_' + str(len(model_params.get('channels'))), now) modelcheckpoint = ModelCheckpoint(modelcheckpoint_name('best_loss'), monitor = 'val_loss', verbose=1, save_best_only=True, save_weights_only=True) # Model early stopping earlystopping = EarlyStopping(monitor='val_loss', patience=10) # tensorboard and metric callbacks log_dir = "logs/fit/{}-{}-{}-{}".format(run.get('model'), aug_string, 'ch_' + str(len(model_params.get('channels'))), now) file_writer = tfsum.create_file_writer(log_dir) tensorboard_cb = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, profile_batch=0) f1_metric = Metrics(val_dataset, val_labels, save_best=True, save_name= modelcheckpoint_name('best_f1'), writer=file_writer) # Model Training and evaluation -------------------------------------------- print("[!] fitting model \n") model.fit( train_dataset.repeat(), epochs=run.get('epochs'), steps_per_epoch= int(num_training / batch_size), validation_data=val_dataset, validation_steps = None, shuffle=True, verbose= 1, callbacks = [tensorboard_cb, f1_metric, LogMetrics(), modelcheckpoint, earlystopping, reduce_lr, MemoryCallback()] ) print("[!] done running, terminating program") ''' optional: run test set evaluation within sacred/omniboard workflow # Model evaluation print("[!] predicting test set") # load optimal weights model.load_weights(modelcheckpoint_name) # evaluate works like a charm results = model.evaluate(test_dataset) print("results are", results) print("[!] predicting confusion matrix") preds = model.predict(test_dataset) labels = [label for img,label in test_dataset] labels = [np.argmax(item.numpy()) for sublist in labels for item in sublist] labels = np.array(labels) confusion_matrix = tf.math.confusion_matrix(labels, np.argmax(preds, axis=1)) print(confusion_matrix) _run.log_scalar("test_loss", float(results[0])) _run.log_scalar("test_acc", float(results[1])) ''' runner = ex.run() return runner
from wireless.agents.bosch_agent import BoschAgent from wireless.agents.time_freq_resource_allocation_v0.round_robin_agent import * from wireless.agents.time_freq_resource_allocation_v0.proportional_fair import * from wireless.agents.noma_ul_time_freq_resource_allocation_v0.noma_ul_proportional_fair import * # Load agent parameters with open('../../config/config_agent.json') as f: ac = json.load(f) # Configure experiment with open('../../config/config_sacred.json') as f: sc = json.load(f) # Sacred Configuration ns = sc["sacred"][ "n_metrics_points"] # Number of points per episode to log in Sacred ex = Experiment(ac["agent"]["agent_type"], save_git_info=False) ex.add_config(sc) ex.add_config(ac) mongo_db_url = f'mongodb://{sc["sacred"]["sacred_user"]}:{sc["sacred"]["sacred_pwd"]}@' +\ f'{sc["sacred"]["sacred_host"]}:{sc["sacred"]["sacred_port"]}/{sc["sacred"]["sacred_db"]}' # ex.observers.append(MongoObserver(url=mongo_db_url, db_name=sc["sacred"]["sacred_db"])) # Uncomment to save to DB # Load environment parameters with open('../../config/config_environment.json') as f: ec = json.load(f) ex.add_config(ec) @ex.automain def main(_run): n_eps = _run.config["agent"]["n_episodes"] t_max = _run.config['agent']['t_max']
if __name__ == '__main__': params = deepcopy(sys.argv) # Get the defaults from default.yaml with open( os.path.join(os.path.dirname(__file__), "config", "default.yaml"), "r") as f: # src/config/default.yaml try: config_dict = yaml.load(f) except yaml.YAMLError as exc: assert False, "default.yaml error: {}".format(exc) # Load algorithm and env base configs env_config = _get_config(params, "--env-config", "envs") alg_config = _get_config(params, "--config", "algs") # config_dict = {**config_dict, **env_config, **alg_config} config_dict = recursive_dict_update(config_dict, env_config) config_dict = recursive_dict_update(config_dict, alg_config) # now add all the config to sacred ex.add_config(config_dict) # Save to disk by default for sacred logger.info("Saving to FileStorageObserver in results/sacred.") file_obs_path = os.path.join(results_path, "sacred") ex.observers.append(FileStorageObserver.create(file_obs_path)) ex.run_commandline(params)
import torch import yaml from torch.utils.data import DataLoader import sys root_pth = '/'.join(osp.dirname(__file__).split('/')[:-2]) sys.path.insert(1, root_pth) from sacred import Experiment from src.tracktor.datasets.factory import Datasets from src.tracktor.resnet import resnet50 from src.tracktor.tracker import Tracker from src.tracktor.utils import interpolate, plot_img ex = Experiment() ex.add_config(root_pth + '/experiments/cfgs/tracktor_pub.yaml') # hacky workaround to load the corresponding configs and not having to hardcode paths here ex.add_config('/'.join(osp.dirname(__file__).split('/')[:-3]) + ex.configurations[0]._conf['tracktor']['reid_network_config']) ex.add_config(root_pth + ex.configurations[0]._conf['tracktor']['obj_detect_config']) # Tracker = ex.capture(Tracker, prefix='tracker.tracker') @ex.automain def my_main(tracktor, siamese, _config): # set all seeds torch.manual_seed(tracktor['seed']) torch.cuda.manual_seed(tracktor['seed'])
import os import cPickle as pickle import numpy as np import cv2 from spikefuel import tools, gui, helpers exp = Experiment("DVS Recording - Caltech-256") exp.add_config({ "caltech256_dir": "", "caltech256_stats_path": "", "recording_save_path": "", "viewer_id": 1, "screen_height": 0, "screen_width": 0, "saccade_size": 0, "work_win_scale": 0.9, "bg_color": [255, 0, 0], "fps": 0, "start_class": 0 }) @exp.automain def dvs_ucf50_exp(caltech256_dir, caltech256_stats_path, recording_save_path, viewer_id, screen_height, screen_width,
from feature_extractor import FeatureExtractor from sacred import Experiment from sacred.observers import MongoObserver ex = Experiment('tsss') URL = r'mongodb://*****:*****@ds115124.mlab.com:15124/intsys' ex.observers.append(MongoObserver.create(url=URL, db_name='intsys')) ex.add_config({ 'max_df': 0.50, # ignore terms that appear in more than 50% of the documents 'min_df': 0.01, # ignore terms that appear in less than 1% of the documents 'max_features': 1000, 'lowercase': True, 'stop_words': 'english', 'analyzer': 'word', 'strip_accents': 'unicode' }) ex.add_config({'use_idf': True, 'sublinear_tf': True, 'norm': None}) # ex.add_config({ # 'data_provider': 'reuters', # }) @ex.named_config def variant_reuters(): data_provider = 'reuters'
INFO - hello_config_dict - Completed after 0:00:00 The message can also easily be changed using the ``with`` command-line argument:: $ ./02_hello_config_dict.py with message='Ciao world!' INFO - hello_config_dict - Running command 'main' INFO - hello_config_dict - Started Ciao world! INFO - hello_config_dict - Completed after 0:00:00 """ from __future__ import division, print_function, unicode_literals from sacred import Experiment ex = Experiment('hello_config_dict') # We add message to the configuration of the experiment here ex.add_config({ "message": "Hello world!" }) # Equivalent: # ex.add_config( # message="Hello world!" # ) # notice how we can access the message here by taking it as an argument @ex.automain def main(message): print(message)
def fetch_parents(current_path, parents=[]): tmp_ex = Experiment('treeqn') tmp_ex.add_config(current_path) with suppress_stdout(): tmp_ex.run("print_config") if tmp_ex.current_run is not None and "parent_config" in tmp_ex.current_run.config: return fetch_parents(tmp_ex.current_run.config["parent_config"], [current_path] + parents) else: return [current_path] + parents configs = fetch_parents(path) ex = Experiment('treeqn') for path in configs: ex.add_config(path) ex.logger = logger ex.observers.append(FileStorageObserver.create('./results')) @ex.config def my_config(save_folder, env_id, architecture, label, name): pytorch_version = torch.__version__ # Timestamp experiment directory save_folder = get_timestamped_dir(save_folder) # Environment switches # obs_dtype as str does the job and plays nice with sacred obs_dtype, input_mode = 'uint8', "atari"