def test_analyse(utils, additionals):
config = {"elements_in": ["sel1"]}
dummyName = "dummyAnalyser"
checkUserExceptionAnalyser = EmptyAnalyser(
{**config, "dev": True}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)
)
dummyAnalyser = TxtCopyAnalyser(
config, dummyName, LocalStorage(folder=utils.TEMP_ELEMENT_DIR)
)
# TODO: work out whether this test is needed with the new format
# test it calls the user-defined `analyse_element`
# with pytest.raises(Exception, match="is the user-defined func!"):
# checkUserExceptionAnalyser.start_analysing(in_parallel=False)
# try again with a text el mocking selection completed
# TODO: fix these tests- adding casting throws errors in some cases, as well as extra log.
for el in additionals.sel1_elements:
with open(
f"{dummyAnalyser.disk.base_dir}/sel1/{dummyAnalyser.disk.RETRIEVED_EXT}/{el}/anitem.txt",
"w+",
) as f:
f.write("Hello")
dummyAnalyser.start_analysing()
# confirm txt has carried
for el in additionals.sel1_elements:
with open(
f"{dummyAnalyser.disk.base_dir}/sel1/{dummyAnalyser.disk.ANALYSED_EXT}/{dummyName}/{el}/copy.txt",
"r",
) as f:
lines = f.readlines()
assert len(lines) == 1
assert lines[0] == "Hello"
def test_phase_decorator(additionals):
class BadClass:
@MTModule.phase("somekey")
def improper_func(self):
pass
class GoodClass(MTModule):
@MTModule.phase("somekey")
def proper_func(self):
self.logger("we did something.")
return "no error"
# test that a decorated method carries through its return value
gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR))
# test that a decorated method carries through its return value
gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR))
assert gc.proper_func() == "no error"
with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f:
lines = f.readlines()
assert len(lines) == 1
assert lines[0] == "my_good_mod: somekey: we did something.\n"
# check that logs were cleared after phase
assert gc._MTModule__LOGS == []
def test_bad_init_error(utils):
bad0 = {}
bad1 = {"elements_in": []}
bad2 = {"elements_in": None}
good = {"elements_in": ["selname"]}
with pytest.raises(
InvalidAnalyserConfigError,
match=
"must contain an 'elements_in' indicating the analyser's input",
):
no_elements_in = ErrorThrowingAnalyser(
bad0, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
with pytest.raises(
InvalidAnalyserConfigError,
match=
"The 'elements_in' must be a list containing at least one string",
):
empty_elements_in = ErrorThrowingAnalyser(
bad1, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
with pytest.raises(
InvalidAnalyserConfigError,
match=
"The 'elements_in' must be a list containing at least one string",
):
empty_elements_in = ErrorThrowingAnalyser(
bad2, "stub", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
with pytest.raises(InvalidAnalyserConfigError,
match="You must provide a name for your analyser"):
badan2 = ErrorThrowingAnalyser(
good, "", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
def test_bad_whitelist(utils):
badConfig = {"elements_in": ["sel1/an1/el1"]}
badAn = EmptyAnalyser(badConfig, "whitelistErrorAnalyser",
LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
with pytest.raises(InvalidAnalyserElements,
match="'elements_in' you specified does not exist"):
badAn.start_analysing()
def additionals(utils):
obj = lambda: None
obj.emptySelector = EmptySelector(
{"dev": True}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
utils.setup()
yield obj
utils.cleanup()
def additionals(utils):
obj = lambda: None
obj.maxDiff = None
obj.emptyAnalyserName = "empty"
obj.WHITELIST = ["sel1/an1", "sel1/an2", "sel2"]
obj.sel1 = "sel1"
obj.sel2 = "sel2"
obj.sel1_elements = ["el1", "el2"]
obj.sel2_elements = ["el4", "el5", "el6"]
utils.scaffold_empty(obj.sel1,
elements=obj.sel1_elements,
analysers=["an1", "an2"])
utils.scaffold_empty(obj.sel2, elements=obj.sel2_elements)
os.rmdir(utils.get_element_path(obj.sel1, "el1", analyser="an2"))
obj.config = {"elements_in": obj.WHITELIST}
obj.emptyAnalyser = EmptyAnalyser(
obj.config,
obj.emptyAnalyserName,
storage=LocalStorage(folder=utils.TEMP_ELEMENT_DIR),
)
utils.setup()
yield obj
utils.cleanup()
def basic(utils):
global base
base = utils.TEMP_ELEMENT_DIR
utils.scaffold_empty("Youtube", elements=["el1"], analysers=["Me"])
utils.setup()
yield LocalStorage(folder=base)
utils.cleanup()
def additionals(utils):
obj = lambda: None
indexModule = "indexErrorSelector"
indexConfig = {"error": "index", "dev": True}
obj.indexErrorSelector = BasicErrorSelector(
indexConfig, indexModule, LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
castModule = "castErrorSelector"
castConfig = {"dev": True}
obj.castErrorSelector = BasicErrorSelector(
castConfig, castModule, LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
retrieveModule = "retrieveErrorSelector"
retrieveConfig = {"dev": True}
obj.retrieveErrorSelector = RetrieveErrorSelector(
retrieveConfig, retrieveModule,
LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
yield obj
utils.cleanup()
def test_parallel_phase_decorator(additionals):
class GoodClass(MTModule):
@MTModule.phase("somekey")
def func(self, gen):
self.logger("This function only takes a generator of elements.")
return "no error"
@MTModule.phase("somekey", remove_db=False)
def func_no_remove(self, gen):
return "no error"
@MTModule.phase("secondkey")
def func_w_arg(self, gen, extra):
self.logger(f"Running func with {list(gen)}, with extra arg {extra}.")
return "no error"
# test that a decorated method carries through its return value
gc = GoodClass({}, "my_good_mod", storage=LocalStorage(folder=additionals.BASE_DIR))
# test parallel logs
eg_gen = (a for a in range(0, 100))
assert gc.func(eg_gen) == "no error"
with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f:
lines = f.readlines()
assert len(lines) == 100
# test db file generation
eg_gen = (a for a in range(0, 100))
assert gc.func_no_remove(eg_gen) == "no error"
dbfile = f"{gc.disk.base_dir}/{gc.UNIQUE_ID}.db"
with open(dbfile, "rb") as f:
_bytes = f.read()
assert len(_bytes) == 800 # 2 4-byte entries per item for 100 items
os.remove(dbfile)
# test that a function is resumed properly
eg_gen = (a for a in range(0, 50))
assert gc.func_no_remove(eg_gen) == "no error"
eg_gen = (a for a in range(0, 100))
assert gc.func(eg_gen) == "no error"
with open(f"{additionals.BASE_DIR}/logs/logs.txt", "r") as f:
lines = f.readlines()
assert len(lines) == 150
# test function with argument
eg_gen = (a for a in range(0, 100))
assert gc.func_w_arg(eg_gen, 10) == "no error"
def additionals(utils):
obj = lambda: None
obj.selname = "stub_sel"
elements = ["skip", "retry3", "retryN", "pass"]
utils.scaffold_empty(obj.selname, elements=elements)
for element in elements:
with open(f"{utils.get_element_path(obj.selname, element)}/out.txt", "w") as f:
f.write("something")
goodConfig = {"elements_in": [obj.selname], "dev": True}
obj.an = ErrorThrowingAnalyser(
goodConfig, "analyserErrorSelector", LocalStorage(folder=utils.TEMP_ELEMENT_DIR)
)
yield obj
utils.cleanup()
def additionals(utils):
obj = lambda: None
obj.BASE_DIR = utils.TEMP_ELEMENT_DIR
obj.mod = EmptyMTModule({}, "empty", LocalStorage(folder=utils.TEMP_ELEMENT_DIR))
yield obj
utils.cleanup()
def make_storage(cfg: dict) -> LocalStorage:
# TODO: generalise `folder` here to a `storage` var that is passed from YAML
return LocalStorage(folder=cfg["folder"])
