def do_experiments(config_list, runner, writer, dirname='./', mapper=map):
"""Runs and writes provided 'config's using provided runner, writer, mapper
Same as do_experiment but takes list of 'config's and optional mapper
argument. Optional mapper argument allows multiprocessing or
IPython.parallel
Args:
config_list: (list of 'config's) 'config's to run with runner
runner: ('config' -> 'result') function that takes config and returns
result. This is where the computation occurs.
writer: ('result' -> None) function that takes single result and writes
it to local filesystem
dirname: (string) local filesystem directory to write serialize
'result's to
mapper: (function, args -> outputs) mapper to use. If runner spawns
daemonic processes, mapper must be non-daemonic.
Returns:
None
"""
ensure_dir(dirname)
config_list = ensure_listlike(config_list)
_do_experiment = partial(do_experiment, runner=runner,
writer=writer, dirname=dirname)
mapper(_do_experiment, config_list)
return
def s3_read_results(bucket_str, config_to_result_path, configs, dirname=''):
bucket = s3_utils.get_bucket(bucket_str)
configs = ensure_listlike(configs)
_read_result = partial(s3_read_result, bucket, config_to_result_path,
dirname=dirname)
results = map(_read_result, configs)
return results
def read_results(reader, config_list, dirname='./'):
"""Reads and extracts 'result's from all files that contain 'result's in a
directory
Args:
config_list: (list of 'config's) list of 'config's to read using reader
reader: ('config', string -> 'result') function to read 'result' from
persistent storage.
given a
'config' and a dirname
dirname: (string) local filesystem directory to look in
Returns:
results: (list of 'result's) list of all 'result's found
"""
config_list = ensure_listlike(config_list)
_read_result = partial(reader, dirname=dirname)
results = map(_read_result, config_list)
return results
