def experiment_iterator(name):
for root, _, files in os.walk(get_experiment_save_directory(name)):
for fname in files:
if ".pkl" in fname:
with open(os.path.join(root, fname), "rb") as f:
yield pickle.load(f)
if ".json" in fname and fname != "specification.json":
with open(os.path.join(root, fname), "r") as f:
yield json.load(f)
def experiment_iterator(name, use_tqdm=False):
iterator = os.walk(get_experiment_save_directory(name))
if use_tqdm:
iterator = tqdm(list(iterator), desc="Loading Experiments")
for root, _, files in iterator:
for fname in files:
if ".pkl" in fname:
with open(os.path.join(root, fname), "rb") as f:
yield pickle.load(f)
if ".json" in fname and fname != "specification.json":
with open(os.path.join(root, fname), "r") as f:
yield json.load(f)
def _find_uncompleted_specifications(self, name, specifications):
already_completed_specifications = []
for root, _, files in os.walk(get_experiment_save_directory(name)):
for fname in files:
if ".pkl" in fname:
with open(os.path.join(root, fname), "rb") as f:
completed = dill.load(f)
already_completed_specifications.append(completed["specification"])
if ".json" in fname and fname != 'specification.json':
with open(os.path.join(root, fname), "r") as f:
completed = json.load(f)
already_completed_specifications.append(completed["specification"])
need_to_run_specifications = []
for specification in specifications:
if specification in already_completed_specifications:
logging.getLogger("smallab.runner").info("Skipping: " + str(specification))
else:
need_to_run_specifications.append(specification)
return need_to_run_specifications
# Read https://stackoverflow.com/questions/5619914/sendmail-errno61-connection-refused about how to start an stmp serevr
from smallab.utilities.email_hooks import EmailCallbackBatchOnly
runner.attach_callbacks([EmailCallbackBatchOnly("*****@*****.**", 40)])
# Take it back off since we don't actually want to bother Mr. Test
runner.attach_callbacks([])
# Set the specifications for our experiments, the author reccomends reading this from a json file!
specifications = [{"seed": 1, "num_calls": 1}, {"seed": 2, "num_calls": 1}]
# Fire off the experiment
runner.run("random_number", specifications, SimpleExperiment())
# Read back our results. Smallab will attempt to save the file in json format so you can easily read
# it but will fall back to pickle if necessary.
for root, _, files in os.walk(get_experiment_save_directory("random_number")):
for fname in files:
if ".json" in fname:
with open(os.path.join(root, fname), "r") as f:
results = json.load(f)
print(results["specification"]["seed"])
print(results["result"]["number"])
from smallab.specification_generator import SpecificationGenerator
# If you want to run a lot of experiments but not manual write out each one, use the specification generator.
# Note: This is also JSON serializable, so you could store this in a json file
generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [1, 2, 3]}
# Call the generate method. Will create the cross product.
specifications = SpecificationGenerator().generate(generation_specification)