def from_file(filename: str) -> 'Result':
"""Create a Result from a transaction file."""
#Why is this here??? This is really confusing in practice
#if filename is None or not Path(filename).exists(): return Result()
json_encode = Cartesian(JsonEncode())
json_decode = Cartesian(JsonDecode())
Pipe.join(DiskSource(filename), [json_decode, ResultPromote(), json_encode], DiskSink(filename, 'w')).run()
return Result.from_transactions(Pipe.join(DiskSource(filename), [json_decode]).read())
def test_simple_sans_gz(self):
sink = DiskSink("coba/tests/.temp/test.log")
sink.write("a")
sink.write("b")
sink.write("c")
self.assertEqual(
["a", "b", "c"],
Path("coba/tests/.temp/test.log").read_text().splitlines())
def test_bad_config_file2(self):
CobaContext.search_paths = ["coba/tests/.temp/"]
DiskSink("coba/tests/.temp/.coba").write('[1,2,3]')
with self.assertRaises(CobaExit) as e:
CobaContext.cacher
lines = str(e.exception).splitlines()
self.assertEqual('', lines[0])
self.assertIn("ERROR: An error occured while initializing CobaContext",
lines[1])
self.assertIn(f"Expecting a JSON object (i.e., {{}}) ", lines[2])
self.assertIn(f" in coba{os.sep}tests{os.sep}.temp{os.sep}.coba.",
lines[2])
self.assertTrue(str(e.exception).endswith("\n"))
def test_config_file_logger(self):
CobaContext.search_paths = ["coba/tests/.temp/"]
DiskSink("coba/tests/.temp/.coba").write(JsonEncode().filter(
{"logger": "NullLogger"}))
self.assertIsInstance(CobaContext.cacher, DiskCacher)
self.assertIsInstance(CobaContext.logger, NullLogger)
self.assertIsInstance(CobaContext.logger.sink, NullSink)
self.assertEqual(CobaContext.cacher.cache_directory, None)
self.assertEqual(CobaContext.experiment.processes, 1)
self.assertEqual(CobaContext.experiment.maxchunksperchild, 0)
self.assertEqual(CobaContext.experiment.chunk_by, 'source')
self.assertEqual(CobaContext.api_keys, {})
self.assertEqual(CobaContext.store, {})
def test_simple_with_gz(self):
sink = DiskSink("coba/tests/.temp/test.gz")
sink.write("a")
sink.write("b")
sink.write("c")
lines = gzip.decompress(
Path("coba/tests/.temp/test.gz").read_bytes()).decode(
'utf-8').splitlines()
self.assertEqual(["a", "b", "c"], lines)
def test_bad_config_file1(self):
CobaContext.search_paths = ["coba/tests/.temp/"]
DiskSink("coba/tests/.temp/.coba").write(
'{ "cacher": { "DiskCacher": "~"')
with self.assertRaises(CobaExit) as e:
CobaContext.cacher
lines = str(e.exception).splitlines()
self.assertEqual('', lines[0])
self.assertIn("ERROR: An error occured while initializing CobaContext",
lines[1])
self.assertIn(f"Expecting ',' delimiter: line 2 column 1 (char 32) ",
lines[2])
self.assertIn(f" in coba{os.sep}tests{os.sep}.temp{os.sep}.coba.",
lines[2])
self.assertTrue(str(e.exception).endswith("\n"))
def test_config_file_disk_cacher_current_directory(self):
CobaContext.search_paths = ["coba/tests/.temp/"]
DiskSink("coba/tests/.temp/.coba").write(JsonEncode().filter(
{"cacher": {
"DiskCacher": "./"
}}))
self.assertIsInstance(CobaContext.cacher, DiskCacher)
self.assertIsInstance(CobaContext.logger, IndentLogger)
self.assertIsInstance(CobaContext.logger.sink, ConsoleSink)
self.assertEqual(CobaContext.cacher.cache_directory,
str(Path("coba/tests/.temp").resolve()))
self.assertEqual(CobaContext.experiment.processes, 1)
self.assertEqual(CobaContext.experiment.maxchunksperchild, 0)
self.assertEqual(CobaContext.experiment.chunk_by, 'source')
self.assertEqual(CobaContext.api_keys, {})
self.assertEqual(CobaContext.store, {})
def test_config_file_experiment(self):
CobaContext.search_paths = ["coba/tests/.temp/"]
DiskSink("coba/tests/.temp/.coba").write(JsonEncode().filter({
"experiment": {
"processes": 2,
"maxtasksperchild": 3,
"chunk_by": "task"
}
}))
self.assertIsInstance(CobaContext.cacher, DiskCacher)
self.assertIsInstance(CobaContext.logger, IndentLogger)
self.assertIsInstance(CobaContext.logger.sink, ConsoleSink)
self.assertEqual(CobaContext.cacher.cache_directory, None)
self.assertEqual(CobaContext.experiment.processes, 2)
self.assertEqual(CobaContext.experiment.maxchunksperchild, 3)
self.assertEqual(CobaContext.experiment.chunk_by, 'task')
self.assertEqual(CobaContext.api_keys, {})
self.assertEqual(CobaContext.store, {})
def __init__(self, log_file: Optional[str] = None, minify:bool=True) -> None:
self._log_file = log_file
self._minify = minify
self._source = DiskSource(log_file) if log_file else ListSource()
self._sink = DiskSink(log_file) if log_file else ListSink(self._source.items)
class TransactionIO_V4(Source['Result'], Sink[Any]):
def __init__(self, log_file: Optional[str] = None, minify:bool=True) -> None:
self._log_file = log_file
self._minify = minify
self._source = DiskSource(log_file) if log_file else ListSource()
self._sink = DiskSink(log_file) if log_file else ListSink(self._source.items)
def write(self, item: Any) -> None:
if isinstance(self._sink, ListSink):
self._sink.write(self._encode(item))
else:
if not Path(self._sink._filename).exists():self._sink.write('["version",4]')
self._sink.write(JsonEncode(self._minify).filter(self._encode(item)))
def read(self) -> 'Result':
n_lrns = None
n_sims = None
lrn_rows = {}
env_rows = {}
int_rows = {}
if isinstance(self._source, ListSource):
decoded_source = self._source
else:
decoded_source = Pipes.join(self._source, Foreach(JsonDecode()))
for trx in decoded_source.read():
if not trx: continue
if trx[0] == "experiment":
n_lrns = trx[1]["n_learners"]
n_sims = trx[1]["n_environments"]
if trx[0] == "E":
env_rows[trx[1]] = trx[2]
if trx[0] == "L":
lrn_rows[trx[1]] = trx[2]
if trx[0] == "I":
int_rows[tuple(trx[1])] = trx[2]
return Result(n_lrns, n_sims, env_rows, lrn_rows, int_rows)
def _encode(self,item):
if item[0] == "T0":
return ['experiment', {"n_learners":item[1], "n_environments":item[2]}]
if item[0] == "T1":
return ["L", item[1], item[2]]
if item[0] == "T2":
return ["E", item[1], item[2]]
if item[0] == "T3":
rows_T = collections.defaultdict(list)
for row in item[2]:
for col,val in row.items():
if col == "rewards" : col="reward"
if col == "reveals" : col="reveal"
rows_T[col].append(val)
return ["I", item[1], { "_packed": rows_T }]
return None
def test_is_picklable(self):
pickle.dumps(DiskSink("coba/tests/.temp/test.gz"))