def test_write_db(self):
"""writing with overwrite in MPI should reset db"""
dstore = io_app.get_data_store("data", suffix="fasta")
members = dstore.filtered(
callback=lambda x: "brca1.fasta" not in x.split("/"))
reader = io_app.load_unaligned()
aligner = align_app.align_to_ref()
writer = write_db("delme.tinydb", create=True, if_exists="overwrite")
process = reader + aligner + writer
r = process.apply_to(
members,
logger=False,
show_progress=False,
parallel=True,
par_kw=dict(use_mpi=True),
)
expect = [str(m) for m in process.data_store]
process.data_store.close()
# now get read only and check what's in there
result = io_app.get_data_store("delme.tinydb")
got = [str(m) for m in result]
assert got == expect
def _get_all_composables(tmp_dir_name):
test_model1 = evo.model("HKY85")
test_model2 = evo.model("GN")
test_hyp = evo.hypothesis(test_model1, test_model2)
test_num_reps = 100
applications = [
align.align_to_ref(),
align.progressive_align(model="GY94"),
evo.ancestral_states(),
evo.bootstrap(hyp=test_hyp, num_reps=test_num_reps),
evo.hypothesis(test_model1, test_model2),
evo.model("GN"),
evo.tabulate_stats(),
sample.fixed_length(100),
sample.min_length(100),
io.write_db(tmp_dir_name, create=True),
io.write_json(tmp_dir_name, create=True),
io.write_seqs(tmp_dir_name, create=True),
sample.omit_bad_seqs(),
sample.omit_degenerates(),
sample.omit_duplicated(),
sample.take_codon_positions(1),
sample.take_named_seqs(),
sample.trim_stop_codons(gc=1),
translate.select_translatable(),
tree.quick_tree(),
tree.scale_branches(),
tree.uniformize_tree(),
]
return applications
def test_apply_to_not_partially_done(self):
"""correctly applies process when result already partially done"""
dstore = io_app.get_data_store("data", suffix="fasta")
num_records = len(dstore)
with TemporaryDirectory(dir=".") as dirname:
dirname = pathlib.Path(dirname)
reader = io_app.load_aligned(format="fasta", moltype="dna")
outpath = dirname / "delme.tinydb"
writer = io_app.write_db(outpath)
_ = writer(reader(dstore[0]))
writer.data_store.close()
writer = io_app.write_db(outpath, if_exists="ignore")
process = reader + writer
_ = process.apply_to(dstore, show_progress=False)
writer.data_store.close()
dstore = io_app.get_data_store(outpath)
self.assertEqual(len(dstore), num_records)
def test_write_db_invalid(self):
"""value error if identifier does not match data.info.source"""
with TemporaryDirectory(dir=".") as dirname:
outpath = join(dirname, "delme")
writer = write_db(outpath, create=True, if_exists="ignore")
data = UnionDict(a=[1, 2], b="string", source="delme2.json")
got = writer(data, identifier=join("blah", "delme.json"))
self.assertTrue("ValueError" in got.message)
writer.data_store.db.close()
def test_bootstrap_composability(self):
"""can be composed with load_db and write_db"""
m1 = evo_app.model("F81")
m2 = evo_app.model("HKY85")
hyp = evo_app.hypothesis(m1, m2)
with TemporaryDirectory(dir=".") as dirname:
path = join(dirname, "delme.tinydb")
_ = io.load_db() + evo_app.bootstrap(
hyp, num_reps=2) + io.write_db(path)
def test_apply_to_not_completed(self):
"""correctly creates notcompleted"""
dstore = io_app.get_data_store("data", suffix="fasta", limit=3)
with TemporaryDirectory(dir=".") as dirname:
reader = io_app.load_aligned(format="fasta", moltype="dna")
# trigger creation of notcompleted
min_length = sample_app.min_length(3000)
outpath = os.path.join(os.getcwd(), dirname, "delme.tinydb")
writer = io_app.write_db(outpath)
process = reader + min_length + writer
r = process.apply_to(dstore, show_progress=False)
self.assertEqual(len(process.data_store.incomplete), 3)
process.data_store.close()
def test_write_db_load_db2(self):
"""correctly write/load built-in python from tinydb"""
with TemporaryDirectory(dir=".") as dirname:
outpath = join(dirname, "delme")
writer = write_db(outpath, create=True, if_exists="ignore")
data = dict(a=[1, 2], b="string")
m = writer(data, identifier=join("blah", "delme.json"))
writer.data_store.db.close()
dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json")
reader = io_app.load_db()
got = reader(dstore[0])
dstore.close()
self.assertEqual(got, data)
def test_apply_to_strings(self):
"""apply_to handles strings as paths"""
dstore = io_app.get_data_store("data", suffix="fasta", limit=3)
dstore = [str(m) for m in dstore]
with TemporaryDirectory(dir=".") as dirname:
reader = io_app.load_aligned(format="fasta", moltype="dna")
min_length = sample_app.min_length(10)
outpath = os.path.join(os.getcwd(), dirname, "delme.tinydb")
writer = io_app.write_db(outpath)
process = reader + min_length + writer
# create paths as strings
r = process.apply_to(dstore, show_progress=False)
self.assertEqual(len(process.data_store.logs), 1)
process.data_store.close()
def test_restricted_usage_of_tinydb_suffix(self):
"""can only use tinydb in a load_db, write_db context"""
with TemporaryDirectory(dir=".") as dirname:
outdir = join(dirname, "delme.tinydb")
for writer_class in (
io_app.write_seqs,
io_app.write_json,
io_app.write_tabular,
):
with self.assertRaises(ValueError):
writer_class(outdir, create=True, if_exists="skip")
# but OK for write_db
w = io_app.write_db(outdir, create=True, if_exists="skip")
w.data_store.close()
def test_write_db_load_db(self):
"""correctly write/load from tinydb"""
# straight directory
with TemporaryDirectory(dir=".") as dirname:
outpath = join(dirname, "delme")
writer = write_db(outpath, create=True, if_exists="ignore")
gr = _get_generic_result(join("blah", "delme.json"))
got = writer(gr)
writer.data_store.db.close()
dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json")
reader = io_app.load_db()
got = reader(dstore[0])
dstore.close()
got.deserialised_values()
self.assertIsInstance(got["dna"], DNA.__class__)
self.assertEqual(got["dna"], DNA)
def test_write_db_load_db(self):
"""correctly write/load from tinydb"""
# straight directory
with TemporaryDirectory(dir=".") as dirname:
outpath = join(dirname, "delme")
writer = write_db(outpath, create=True, if_exists="ignore")
mock = patch("data.source", autospec=True)
mock.to_json = DNA.to_json
mock.source = join("blah", "delme.json")
got = writer(mock)
writer.data_store.db.close()
dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json")
reader = io_app.load_db()
got = reader(dstore[0])
dstore.close()
self.assertIsInstance(got, DNA.__class__)
self.assertEqual(got, DNA)
def test_load_db_failure_json_file(self):
"""informative load_db error message when given a json file path"""
# todo this test has a trapped exception about being unable to delete
# a file
with TemporaryDirectory(dir=".") as dirname:
outpath = join(dirname, "delme")
writer = write_db(outpath, create=True, if_exists="ignore")
gr = _get_generic_result(join("blah", "delme.json"))
got = writer(gr)
writer.data_store.db.close()
dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json")
reader = io_app.load_db()
outpath = join(dirname, "dummy.json")
with open(outpath, mode="w") as outfile:
outfile.write("\n\n")
got = reader(outpath)
self.assertIsInstance(got, NotCompleted)
self.assertTrue("json" in got.message)
