def test_analyse(utils, additionals): config = {"elements_in": ["sel1"]} dummyName = "dummyAnalyser" checkUserExceptionAnalyser = EmptyAnalyser( {**config, "dev": True}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR) ) dummyAnalyser = TxtCopyAnalyser( config, dummyName, LocalStorage(folder=utils.TEMP_ELEMENT_DIR) ) # TODO: work out whether this test is needed with the new format # test it calls the user-defined `analyse_element` # with pytest.raises(Exception, match="is the user-defined func!"): # checkUserExceptionAnalyser.start_analysing(in_parallel=False) # try again with a text el mocking selection completed # TODO: fix these tests- adding casting throws errors in some cases, as well as extra log. for el in additionals.sel1_elements: with open( f"{dummyAnalyser.disk.base_dir}/sel1/{dummyAnalyser.disk.RETRIEVED_EXT}/{el}/anitem.txt", "w+", ) as f: f.write("Hello") dummyAnalyser.start_analysing() # confirm txt has carried for el in additionals.sel1_elements: with open( f"{dummyAnalyser.disk.base_dir}/sel1/{dummyAnalyser.disk.ANALYSED_EXT}/{dummyName}/{el}/copy.txt", "r", ) as f: lines = f.readlines() assert len(lines) == 1 assert lines[0] == "Hello"
def test_phase_decorator(additionals): class BadClass: @MTModule.phase("somekey") def improper_func(self): pass class GoodClass(MTModule): @MTModule.phase("somekey") def proper_func(self): self.logger("we did something.") return "no error" # test that a decorated method carries through its return value gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR)) # test that a decorated method carries through its return value gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR)) assert gc.proper_func() == "no error" with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f: lines = f.readlines() assert len(lines) == 1 assert lines[0] == "my_good_mod: somekey: we did something.\n" # check that logs were cleared after phase assert gc._MTModule__LOGS == []
def test_bad_init_error(utils): bad0 = {} bad1 = {"elements_in": []} bad2 = {"elements_in": None} good = {"elements_in": ["selname"]} with pytest.raises( InvalidAnalyserConfigError, match= "must contain an 'elements_in' indicating the analyser's input", ): no_elements_in = ErrorThrowingAnalyser( bad0, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) with pytest.raises( InvalidAnalyserConfigError, match= "The 'elements_in' must be a list containing at least one string", ): empty_elements_in = ErrorThrowingAnalyser( bad1, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) with pytest.raises( InvalidAnalyserConfigError, match= "The 'elements_in' must be a list containing at least one string", ): empty_elements_in = ErrorThrowingAnalyser( bad2, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) with pytest.raises(InvalidAnalyserConfigError, match="You must provide a name for your analyser"): badan2 = ErrorThrowingAnalyser( good, "", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
def test_bad_whitelist(utils): badConfig = {"elements_in": ["sel1/an1/el1"]} badAn = EmptyAnalyser(badConfig, "whitelistErrorAnalyser", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) with pytest.raises(InvalidAnalyserElements, match="'elements_in' you specified does not exist"): badAn.start_analysing()
def additionals(utils): obj = lambda: None obj.emptySelector = EmptySelector( {"dev": True}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) utils.setup() yield obj utils.cleanup()
def additionals(utils): obj = lambda: None obj.maxDiff = None obj.emptyAnalyserName = "empty" obj.WHITELIST = ["sel1/an1", "sel1/an2", "sel2"] obj.sel1 = "sel1" obj.sel2 = "sel2" obj.sel1_elements = ["el1", "el2"] obj.sel2_elements = ["el4", "el5", "el6"] utils.scaffold_empty(obj.sel1, elements=obj.sel1_elements, analysers=["an1", "an2"]) utils.scaffold_empty(obj.sel2, elements=obj.sel2_elements) os.rmdir(utils.get_element_path(obj.sel1, "el1", analyser="an2")) obj.config = {"elements_in": obj.WHITELIST} obj.emptyAnalyser = EmptyAnalyser( obj.config, obj.emptyAnalyserName, storage=LocalStorage(folder=utils.TEMP_ELEMENT_DIR), ) utils.setup() yield obj utils.cleanup()
def basic(utils): global base base = utils.TEMP_ELEMENT_DIR utils.scaffold_empty("Youtube", elements=["el1"], analysers=["Me"]) utils.setup() yield LocalStorage(folder=base) utils.cleanup()
def additionals(utils): obj = lambda: None indexModule = "indexErrorSelector" indexConfig = {"error": "index", "dev": True} obj.indexErrorSelector = BasicErrorSelector( indexConfig, indexModule, LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) castModule = "castErrorSelector" castConfig = {"dev": True} obj.castErrorSelector = BasicErrorSelector( castConfig, castModule, LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) retrieveModule = "retrieveErrorSelector" retrieveConfig = {"dev": True} obj.retrieveErrorSelector = RetrieveErrorSelector( retrieveConfig, retrieveModule, LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) yield obj utils.cleanup()
def test_parallel_phase_decorator(additionals): class GoodClass(MTModule): @MTModule.phase("somekey") def func(self, gen): self.logger("This function only takes a generator of elements.") return "no error" @MTModule.phase("somekey", remove_db=False) def func_no_remove(self, gen): return "no error" @MTModule.phase("secondkey") def func_w_arg(self, gen, extra): self.logger(f"Running func with {list(gen)}, with extra arg {extra}.") return "no error" # test that a decorated method carries through its return value gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR)) # test parallel logs eg_gen = (a for a in range(0, 100)) assert gc.func(eg_gen) == "no error" with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f: lines = f.readlines() assert len(lines) == 100 # test db file generation eg_gen = (a for a in range(0, 100)) assert gc.func_no_remove(eg_gen) == "no error" dbfile = f"{gc.disk.base_dir}/{gc.UNIQUE_ID}.db" with open(dbfile, "rb") as f: _bytes = f.read() assert len(_bytes) == 800 # 2 4-byte entries per item for 100 items os.remove(dbfile) # test that a function is resumed properly eg_gen = (a for a in range(0, 50)) assert gc.func_no_remove(eg_gen) == "no error" eg_gen = (a for a in range(0, 100)) assert gc.func(eg_gen) == "no error" with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f: lines = f.readlines() assert len(lines) == 150 # test function with argument eg_gen = (a for a in range(0, 100)) assert gc.func_w_arg(eg_gen, 10) == "no error"
def additionals(utils): obj = lambda: None obj.selname = "stub_sel" elements = ["skip", "retry3", "retryN", "pass"] utils.scaffold_empty(obj.selname, elements=elements) for element in elements: with open(f"{utils.get_element_path(obj.selname, element)}/out.txt", "w") as f: f.write("something") goodConfig = {"elements_in": [obj.selname], "dev": True} obj.an = ErrorThrowingAnalyser( goodConfig, "analyserErrorSelector", LocalStorage(folder=utils.TEMP_ELEMENT_DIR) ) yield obj utils.cleanup()
def additionals(utils): obj = lambda: None obj.BASE_DIR = utils.TEMP_ELEMENT_DIR obj.mod = EmptyMTModule({}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)) yield obj utils.cleanup()
def make_storage(cfg: dict) -> LocalStorage: # TODO: generalise `folder` here to a `storage` var that is passed from YAML return LocalStorage(folder=cfg["folder"])