def test_report_requires_schema(self, results_file_path):
""" Report fails if schema not provided, even for a file backend """
psm = PipestatManager(name="test", results_file=results_file_path)
with pytest.raises(SchemaNotFoundError):
psm.report(result_identifier="name_of_something",
record_identifier="sample1",
value="test_name")
def test_report_requires_schema(
self,
rec_id,
val,
config_no_schema_file_path,
results_file_path,
backend,
):
"""
If schema is not provided at object instantiation stage, SchemaNotFondError
is raised if report method is called with file as a backend.
In case of the DB as a backend, the error is raised at object
instantiation stage since there is no way to init relational DB table
with no columns predefined
"""
args = dict(namespace="test")
backend_data = ({
"config": config_no_schema_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
if backend == "file":
with pytest.raises(SchemaNotFoundError):
psm.report(record_identifier=rec_id, values=val)
def test_connection_checker(self, config_file_path, schema_file_path):
pm = PipestatManager(
config=config_file_path,
database_only=True,
schema_path=schema_file_path,
namespace="test",
)
assert pm.is_db_connected()
def test_remove_basic(self, rec_id, res_id, val, config_file_path,
results_file_path, schema_file_path, backend):
args = dict(schema_path=schema_file_path, name="test")
backend_data = {"database_config": config_file_path} if backend == "db"\
else {"results_file": results_file_path}
args.update(backend_data)
psm = PipestatManager(**args)
psm.remove(result_identifier=res_id, record_identifier=rec_id)
assert res_id not in psm.data["test"][rec_id]
def test_retrieve_whole_record(self, rec_id, config_file_path,
results_file_path, schema_file_path,
backend):
args = dict(schema_path=schema_file_path, name="test")
backend_data = {"database_config": config_file_path} if backend == "db"\
else {"results_file": results_file_path}
args.update(backend_data)
psm = PipestatManager(**args)
assert isinstance(psm.retrieve(record_identifier=rec_id), Mapping)
def test_connection_overwrite_error(self, config_file_path, schema_file_path):
pm = PipestatManager(
config=config_file_path,
database_only=True,
schema_path=schema_file_path,
namespace="test",
)
with pytest.raises(PipestatDatabaseError):
pm.establish_db_connection()
def test_remove_nonexistent_record(self, rec_id, schema_file_path,
config_file_path, results_file_path,
backend):
args = dict(schema_path=schema_file_path, name="test")
backend_data = {"database_config": config_file_path} if backend == "db"\
else {"results_file": results_file_path}
args.update(backend_data)
psm = PipestatManager(**args)
assert not psm.remove(record_identifier=rec_id)
def test_retrieve_nonexistent(self, rec_id, res_id, config_file_path,
results_file_path, schema_file_path,
backend):
args = dict(schema_path=schema_file_path, name="test")
backend_data = {"database_config": config_file_path} if backend == "db"\
else {"results_file": results_file_path}
args.update(backend_data)
psm = PipestatManager(**args)
with pytest.raises(PipestatDatabaseError):
psm.retrieve(result_identifier=res_id, record_identifier=rec_id)
def test_use_other_namespace_file(self, schema_file_path):
""" Results file can be used with just one namespace """
tmp_res_file = os.path.join(mkdtemp(), "res.yml")
print(f"Temporary results file: {tmp_res_file}")
assert not os.path.exists(tmp_res_file)
PipestatManager(name="test",
results_file=tmp_res_file,
schema_path=schema_file_path)
assert os.path.exists(tmp_res_file)
with pytest.raises(PipestatDatabaseError):
PipestatManager(name="new_test",
results_file=tmp_res_file,
schema_path=schema_file_path)
def test_select_offset(
self,
config_file_path,
schema_file_path,
offset,
):
args = dict(
schema_path=schema_file_path, namespace="test", config=config_file_path
)
psm = PipestatManager(**args)
result = psm.select(offset=offset)
print(result)
assert len(result) == max((psm.record_count - offset), 0)
def test_status_not_configured(
self, schema_file_path, config_file_path, results_file_path, backend, status_id
):
"""status management works even in case it has not been configured"""
args = dict(schema_path=schema_file_path, namespace="test")
backend_data = (
{"config": config_file_path}
if backend == "db"
else {"results_file_path": results_file_path}
)
args.update(backend_data)
psm = PipestatManager(**args)
psm.set_status(record_identifier="sample1", status_identifier=status_id)
assert psm.get_status(record_identifier="sample1") == status_id
def test_remove_record(self, rec_id, schema_file_path, config_file_path,
results_file_path, backend):
args = dict(schema_path=schema_file_path,
namespace="test",
database_only=False)
backend_data = ({
"config": config_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
psm.remove(record_identifier=rec_id)
assert rec_id not in psm.data["test"]
def test_remove(self, val, config_file_path, schema_file_path,
results_file_path, backend):
REC_ID = "constant_record_id"
args = dict(schema_path=schema_file_path,
namespace="test",
record_identifier=REC_ID)
backend_data = ({
"config": config_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
assert psm.remove(result_identifier=list(val.keys())[0])
def test_report_overwrite(self, rec_id, res_id, val, config_file_path,
schema_file_path, results_file_path, backend):
args = dict(schema_path=schema_file_path, name="test")
backend_data = {"database_config": config_file_path} if backend == "db"\
else {"results_file": results_file_path}
args.update(backend_data)
psm = PipestatManager(**args)
psm.report(result_identifier=res_id,
record_identifier=rec_id,
value=val,
force_overwrite=True)
assert rec_id in psm.data["test"]
assert res_id in psm.data["test"][rec_id]
if backend == "file":
is_in_file(results_file_path, str(val))
def test_schema_req_for_db(self, config_file_path):
"""
Object constructor raises exception if schema is not provided
with database configuration file
"""
with pytest.raises(SchemaNotFoundError):
PipestatManager(name="test", database_config=config_file_path)
def test_select_invalid_filter_structure(
self,
res_id,
config_file_path,
schema_file_path,
filter,
):
args = dict(
schema_path=schema_file_path, namespace="test", config=config_file_path
)
psm = PipestatManager(**args)
with pytest.raises((ValueError, TypeError)):
psm.select(
filter_conditions=[filter],
columns=[res_id],
)
def test_select_invalid_filter_column(
self,
rec_id,
res_id,
config_file_path,
schema_file_path,
):
args = dict(
schema_path=schema_file_path, namespace="test", config=config_file_path
)
psm = PipestatManager(**args)
with pytest.raises(ValueError):
psm.select(
filter_conditions=[("bogus_column", "eq", rec_id)],
columns=[res_id],
)
def test_results_file_contents_loaded(self, results_file_path,
schema_file_path):
""" Contents of the results file are present after loading """
psm = PipestatManager(name="test",
results_file=results_file_path,
schema_path=schema_file_path)
assert "test" in psm.data
def test_config(self, monkeypatch, config_file_path):
"""
test that the object can be created if the arguments are
provided in a config that is provided as env vars
"""
monkeypatch.setenv(ENV_VARS["config"], config_file_path)
PipestatManager()
def test_str_representation(self, results_file_path, schema_file_path):
"""Test string representation identifies number of records"""
psm = PipestatManager(
namespace="test",
results_file_path=results_file_path,
schema_path=schema_file_path,
)
assert f"Records count: {len(psm.data[psm.namespace])}" in str(psm)
def test_select_limit(
self,
rec_id,
res_id,
config_file_path,
schema_file_path,
limit,
):
args = dict(
schema_path=schema_file_path, namespace="test", config=config_file_path
)
psm = PipestatManager(**args)
result = psm.select(
filter_conditions=[(RECORD_ID, "eq", rec_id)],
columns=[res_id],
limit=limit,
)
assert len(result) <= limit
def test_status_file_defult_location(self, schema_file_path, results_file_path):
"""status file location is set to the results file dir
if not specified"""
psm = PipestatManager(
namespace="test",
results_file_path=results_file_path,
schema_path=schema_file_path,
)
assert psm[STATUS_FILE_DIR] == os.path.dirname(psm.file)
def test_create_results_file(self, schema_file_path):
""" Results file is created if a nonexistent path provided """
tmp_res_file = os.path.join(mkdtemp(), "res.yml")
print(f"Temporary results file: {tmp_res_file}")
assert not os.path.exists(tmp_res_file)
PipestatManager(name="test",
results_file=tmp_res_file,
schema_path=schema_file_path)
assert os.path.exists(tmp_res_file)
def test_missing_cfg_data(self, schema_file_path):
"""Object constructor raises exception if cfg is missing data"""
tmp_pth = os.path.join(mkdtemp(), "res.yml")
with open(tmp_pth, "w") as file:
dump({"database": {"host": "localhost"}}, file)
with pytest.raises(MissingConfigDataError):
PipestatManager(
namespace="test", config=tmp_pth, schema_path=schema_file_path
)
def test_report(
self,
val,
config_file_path,
schema_file_path,
):
REC_ID = "constant_record_id"
psm = PipestatManager(
schema_path=schema_file_path,
namespace="test",
record_identifier=REC_ID,
database_only=True,
config=config_file_path,
)
psm.report(values=val)
assert len(psm.data) == 0
val_name = list(val.keys())[0]
assert psm.select(filter_conditions=[(val_name, "eq", str(val[val_name]))])
def test_obj_creation_file(self, schema_file_path, results_file_path):
"""Object constructor works with file as backend"""
assert isinstance(
PipestatManager(
namespace="test",
results_file_path=results_file_path,
schema_path=schema_file_path,
),
PipestatManager,
)
def test_remove_nonexistent_result(
self,
rec_id,
res_id,
schema_file_path,
config_file_path,
results_file_path,
backend,
):
args = dict(schema_path=schema_file_path, namespace="test")
backend_data = ({
"config": config_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
assert not psm.remove(record_identifier=rec_id,
result_identifier=res_id)
def test_no_config(self, monkeypatch, results_file_path, schema_file_path):
"""
test that the object can be created if the arguments
are provided as env vars
"""
monkeypatch.setenv(ENV_VARS["namespace"], "test")
monkeypatch.setenv(ENV_VARS["record_identifier"], "sample1")
monkeypatch.setenv(ENV_VARS["results_file"], results_file_path)
monkeypatch.setenv(ENV_VARS["schema"], schema_file_path)
PipestatManager()
def test_retrieve_basic(
self,
rec_id,
val,
config_file_path,
results_file_path,
schema_file_path,
backend,
):
args = dict(schema_path=schema_file_path, namespace="test")
backend_data = ({
"config": config_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
retrieved_val = psm.retrieve(record_identifier=rec_id,
result_identifier=list(val.keys())[0])
assert str(retrieved_val) == str(list(val.values())[0])
def test_report(self, val, config_file_path, schema_file_path,
results_file_path, backend):
REC_ID = "constant_record_id"
args = dict(
schema_path=schema_file_path,
namespace="test",
record_identifier=REC_ID,
database_only=False,
)
backend_data = ({
"config": config_file_path
} if backend == "db" else {
"results_file_path": results_file_path
})
args.update(backend_data)
psm = PipestatManager(**args)
psm.report(values=val)
assert REC_ID in psm.data["test"]
assert list(val.keys())[0] in psm.data["test"][REC_ID]
if backend == "file":
is_in_file(results_file_path, str(list(val.values())[0]))