def test_write_db(self): """writing with overwrite in MPI should reset db""" dstore = io_app.get_data_store("data", suffix="fasta") members = dstore.filtered( callback=lambda x: "brca1.fasta" not in x.split("/")) reader = io_app.load_unaligned() aligner = align_app.align_to_ref() writer = write_db("delme.tinydb", create=True, if_exists="overwrite") process = reader + aligner + writer r = process.apply_to( members, logger=False, show_progress=False, parallel=True, par_kw=dict(use_mpi=True), ) expect = [str(m) for m in process.data_store] process.data_store.close() # now get read only and check what's in there result = io_app.get_data_store("delme.tinydb") got = [str(m) for m in result] assert got == expect
def _get_all_composables(tmp_dir_name): test_model1 = evo.model("HKY85") test_model2 = evo.model("GN") test_hyp = evo.hypothesis(test_model1, test_model2) test_num_reps = 100 applications = [ align.align_to_ref(), align.progressive_align(model="GY94"), evo.ancestral_states(), evo.bootstrap(hyp=test_hyp, num_reps=test_num_reps), evo.hypothesis(test_model1, test_model2), evo.model("GN"), evo.tabulate_stats(), sample.fixed_length(100), sample.min_length(100), io.write_db(tmp_dir_name, create=True), io.write_json(tmp_dir_name, create=True), io.write_seqs(tmp_dir_name, create=True), sample.omit_bad_seqs(), sample.omit_degenerates(), sample.omit_duplicated(), sample.take_codon_positions(1), sample.take_named_seqs(), sample.trim_stop_codons(gc=1), translate.select_translatable(), tree.quick_tree(), tree.scale_branches(), tree.uniformize_tree(), ] return applications
def test_apply_to_not_partially_done(self): """correctly applies process when result already partially done""" dstore = io_app.get_data_store("data", suffix="fasta") num_records = len(dstore) with TemporaryDirectory(dir=".") as dirname: dirname = pathlib.Path(dirname) reader = io_app.load_aligned(format="fasta", moltype="dna") outpath = dirname / "delme.tinydb" writer = io_app.write_db(outpath) _ = writer(reader(dstore[0])) writer.data_store.close() writer = io_app.write_db(outpath, if_exists="ignore") process = reader + writer _ = process.apply_to(dstore, show_progress=False) writer.data_store.close() dstore = io_app.get_data_store(outpath) self.assertEqual(len(dstore), num_records)
def test_write_db_invalid(self): """value error if identifier does not match data.info.source""" with TemporaryDirectory(dir=".") as dirname: outpath = join(dirname, "delme") writer = write_db(outpath, create=True, if_exists="ignore") data = UnionDict(a=[1, 2], b="string", source="delme2.json") got = writer(data, identifier=join("blah", "delme.json")) self.assertTrue("ValueError" in got.message) writer.data_store.db.close()
def test_bootstrap_composability(self): """can be composed with load_db and write_db""" m1 = evo_app.model("F81") m2 = evo_app.model("HKY85") hyp = evo_app.hypothesis(m1, m2) with TemporaryDirectory(dir=".") as dirname: path = join(dirname, "delme.tinydb") _ = io.load_db() + evo_app.bootstrap( hyp, num_reps=2) + io.write_db(path)
def test_apply_to_not_completed(self): """correctly creates notcompleted""" dstore = io_app.get_data_store("data", suffix="fasta", limit=3) with TemporaryDirectory(dir=".") as dirname: reader = io_app.load_aligned(format="fasta", moltype="dna") # trigger creation of notcompleted min_length = sample_app.min_length(3000) outpath = os.path.join(os.getcwd(), dirname, "delme.tinydb") writer = io_app.write_db(outpath) process = reader + min_length + writer r = process.apply_to(dstore, show_progress=False) self.assertEqual(len(process.data_store.incomplete), 3) process.data_store.close()
def test_write_db_load_db2(self): """correctly write/load built-in python from tinydb""" with TemporaryDirectory(dir=".") as dirname: outpath = join(dirname, "delme") writer = write_db(outpath, create=True, if_exists="ignore") data = dict(a=[1, 2], b="string") m = writer(data, identifier=join("blah", "delme.json")) writer.data_store.db.close() dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json") reader = io_app.load_db() got = reader(dstore[0]) dstore.close() self.assertEqual(got, data)
def test_apply_to_strings(self): """apply_to handles strings as paths""" dstore = io_app.get_data_store("data", suffix="fasta", limit=3) dstore = [str(m) for m in dstore] with TemporaryDirectory(dir=".") as dirname: reader = io_app.load_aligned(format="fasta", moltype="dna") min_length = sample_app.min_length(10) outpath = os.path.join(os.getcwd(), dirname, "delme.tinydb") writer = io_app.write_db(outpath) process = reader + min_length + writer # create paths as strings r = process.apply_to(dstore, show_progress=False) self.assertEqual(len(process.data_store.logs), 1) process.data_store.close()
def test_restricted_usage_of_tinydb_suffix(self): """can only use tinydb in a load_db, write_db context""" with TemporaryDirectory(dir=".") as dirname: outdir = join(dirname, "delme.tinydb") for writer_class in ( io_app.write_seqs, io_app.write_json, io_app.write_tabular, ): with self.assertRaises(ValueError): writer_class(outdir, create=True, if_exists="skip") # but OK for write_db w = io_app.write_db(outdir, create=True, if_exists="skip") w.data_store.close()
def test_write_db_load_db(self): """correctly write/load from tinydb""" # straight directory with TemporaryDirectory(dir=".") as dirname: outpath = join(dirname, "delme") writer = write_db(outpath, create=True, if_exists="ignore") gr = _get_generic_result(join("blah", "delme.json")) got = writer(gr) writer.data_store.db.close() dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json") reader = io_app.load_db() got = reader(dstore[0]) dstore.close() got.deserialised_values() self.assertIsInstance(got["dna"], DNA.__class__) self.assertEqual(got["dna"], DNA)
def test_write_db_load_db(self): """correctly write/load from tinydb""" # straight directory with TemporaryDirectory(dir=".") as dirname: outpath = join(dirname, "delme") writer = write_db(outpath, create=True, if_exists="ignore") mock = patch("data.source", autospec=True) mock.to_json = DNA.to_json mock.source = join("blah", "delme.json") got = writer(mock) writer.data_store.db.close() dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json") reader = io_app.load_db() got = reader(dstore[0]) dstore.close() self.assertIsInstance(got, DNA.__class__) self.assertEqual(got, DNA)
def test_load_db_failure_json_file(self): """informative load_db error message when given a json file path""" # todo this test has a trapped exception about being unable to delete # a file with TemporaryDirectory(dir=".") as dirname: outpath = join(dirname, "delme") writer = write_db(outpath, create=True, if_exists="ignore") gr = _get_generic_result(join("blah", "delme.json")) got = writer(gr) writer.data_store.db.close() dstore = io_app.get_data_store(f"{outpath}.tinydb", suffix="json") reader = io_app.load_db() outpath = join(dirname, "dummy.json") with open(outpath, mode="w") as outfile: outfile.write("\n\n") got = reader(outpath) self.assertIsInstance(got, NotCompleted) self.assertTrue("json" in got.message)