def config(): debug = False batch_size = 6 train_dataset = "mix_2_spk_min_tr" validate_dataset = "mix_2_spk_min_cv" # Start with an empty dict to allow tracking by Sacred trainer = { "model": { "factory": pt.models.bss.PermutationInvariantTrainingModel, "dropout_input": 0., "dropout_hidden": 0., "dropout_linear": 0. }, "storage_dir": None, "optimizer": { "factory": pt.optimizer.Adam, "gradient_clipping": 1 }, "summary_trigger": (1000, "iteration"), "stop_trigger": (300_000, "iteration"), "loss_weights": { "pit_ips_loss": 1.0, "pit_mse_loss": 0.0, } } pt.Trainer.get_config(trainer) if trainer['storage_dir'] is None: trainer['storage_dir'] = get_new_folder(path_template, mkdir=False) ex.observers.append( FileStorageObserver.create(Path(trainer['storage_dir']) / 'sacred'))
def test_fs_observer_equality(dir_obs): basedir, obs = dir_obs obs2 = FileStorageObserver.create(obs.basedir) assert obs == obs2 assert not obs != obs2 assert not obs == "foo" assert obs != "foo"
def test_fs_observer_equality(dir_obs): basedir, obs = dir_obs obs2 = FileStorageObserver.create(obs.basedir) assert obs == obs2 assert not obs != obs2 assert not obs == 'foo' assert obs != 'foo'
def test_fs_observer_resource_event_does_not_duplicate(dir_obs, sample_run, tmpfile): basedir, obs = dir_obs obs2 = FileStorageObserver.create(obs.basedir) obs.started_event(**sample_run) obs.resource_event(tmpfile.name) # let's have another run from a different observer sample_run["_id"] = None _id = obs2.started_event(**sample_run) run_dir = basedir.join(str(_id)) obs2.resource_event(tmpfile.name) res_dir = basedir.join("_resources") assert res_dir.exists() assert len(res_dir.listdir()) == 1 assert res_dir.listdir()[0].read() == tmpfile.content run = json.loads(run_dir.join("run.json").read()) assert len(run["resources"]) == 1 assert run["resources"][0] == [tmpfile.name, res_dir.listdir()[0].strpath]
def test_fs_observer_resource_event_does_not_duplicate(dir_obs, sample_run, tmpfile): basedir, obs = dir_obs obs2 = FileStorageObserver.create(obs.basedir) obs.started_event(**sample_run) obs.resource_event(tmpfile.name) # let's have another run from a different observer sample_run['_id'] = None _id = obs2.started_event(**sample_run) run_dir = basedir.join(str(_id)) obs2.resource_event(tmpfile.name) res_dir = basedir.join('_resources') assert res_dir.exists() assert len(res_dir.listdir()) == 1 assert res_dir.listdir()[0].read() == tmpfile.content run = json.loads(run_dir.join('run.json').read()) assert len(run['resources']) == 1 assert run['resources'][0] == [tmpfile.name, res_dir.listdir()[0].strpath]
def config(): debug = False batch_size = 6 database_json = "" # Path to WSJ0_2mix .json if "WSJ0_2MIX" in os.environ: database_json = os.environ.get("WSJ0_2MIX") assert len(database_json) > 0, 'Set path to database Json on the command line or set environment variable WSJ0_2MIX' train_dataset = "mix_2_spk_min_tr" validate_dataset = "mix_2_spk_min_cv" # dict describing the model parameters, to allow changing the paramters from the command line. # Configurable automatically inserts default values of not mentioned parameters to the config.json trainer = { "model": { "factory": pt.contrib.examples.pit.model.PermutationInvariantTrainingModel, "dropout_input": 0., "dropout_hidden": 0., "dropout_linear": 0. }, "storage_dir": None, "optimizer": { "factory": pt.optimizer.Adam, "gradient_clipping": 1 }, "summary_trigger": (1000, "iteration"), "stop_trigger": (300_000, "iteration"), "loss_weights": { "pit_ips_loss": 1.0, "pit_mse_loss": 0.0, } } pt.Trainer.get_config(trainer) if trainer['storage_dir'] is None: trainer['storage_dir'] = get_new_folder(path_template, mkdir=False) ex.observers.append(FileStorageObserver.create( Path(trainer['storage_dir']) / 'sacred') )
def test_observer_equality(tmpdir): observer_1 = FileStorageObserver.create(str(tmpdir / "a")) observer_2 = FileStorageObserver.create(str(tmpdir / "b")) observer_3 = FileStorageObserver.create(str(tmpdir / "a")) assert observer_1 == observer_3 assert observer_1 != observer_2
def dir_obs(tmpdir): basedir = tmpdir.join("file_storage") return basedir, FileStorageObserver.create(basedir.strpath)
def dir_obs(tmpdir): basedir = tmpdir.join('file_storage') return basedir, FileStorageObserver.create(basedir.strpath)
def dir_obs(tmpdir): return tmpdir, FileStorageObserver.create(tmpdir.strpath)
import sacred from sacred.utils import apply_backspaces_and_linefeeds from sacred.observers.file_storage import FileStorageObserver GRAMTOOLS_INSTALL_PATH = '/home/robyn/Documents/gramtools' GENERATE_PRG_SCRIPT_PATH = os.path.join(GRAMTOOLS_INSTALL_PATH, 'utils/vcf_to_linear_prg.pl') GENERATE_KMERS_SCRIPT_PATH = os.path.join(GRAMTOOLS_INSTALL_PATH, 'utils/variantKmers.py') MAP_READS_PATH = os.path.join(GRAMTOOLS_INSTALL_PATH, 'bin', 'gramtools') experiment = sacred.Experiment() experiment.captured_out_filter = apply_backspaces_and_linefeeds file_observer = FileStorageObserver.create('gramtools_runs') file_observer.save_sources = lambda x: None experiment.observers.append(file_observer) def generate_paths(vcf_path, fasta_path): """Generate and return all file paths associated with experiment.""" vcf_path = os.path.abspath(vcf_path) file_observer.run_entry['artifacts'].append(vcf_path) fasta_path = os.path.abspath(fasta_path) file_observer.run_entry['artifacts'].append(fasta_path) run_path = os.path.abspath(file_observer.dir) data_path = os.path.join(run_path, 'data')