def test_sync_studies_from_disk() -> None:
ma = RawStudy(id="a", path="a")
fa = StudyFolder(path=Path("a"), workspace="", groups=[])
mb = RawStudy(id="b", path="b")
mc = RawStudy(
id="c",
path="c",
name="c",
content_status=StudyContentStatus.WARNING,
workspace=DEFAULT_WORKSPACE_NAME,
owner=User(id=0),
)
fc = StudyFolder(path=Path("c"),
workspace=DEFAULT_WORKSPACE_NAME,
groups=[])
repository = Mock()
repository.get_all.side_effect = [[ma, mb], [ma]]
service = StorageService(
study_service=Mock(),
importer_service=Mock(),
exporter_service=Mock(),
user_service=Mock(),
repository=repository,
event_bus=Mock(),
)
service.sync_studies_on_disk([fa, fc])
repository.delete.assert_called_once_with(mb.id)
repository.save.assert_called_once()
def test_save_metadata() -> None:
# Mock
repository = Mock()
uuid = str(uuid4())
study_service = Mock()
study_service.get_study_information.return_value = {
"antares": {
"caption": "CAPTION",
"version": "VERSION",
"author": "AUTHOR",
"created": 1234,
"lastsave": 9876,
}
}
# Input
jwt = JWTUser(
id=0,
impersonator=0,
type="users",
groups=[JWTGroup(id="my-group", name="group", role=RoleType.ADMIN)],
)
user = User(id=0, name="user")
group = Group(id="my-group", name="group")
# Expected
study = RawStudy(
id=uuid,
name="CAPTION",
version="VERSION",
author="AUTHOR",
created_at=datetime.fromtimestamp(1234),
updated_at=datetime.fromtimestamp(9876),
content_status=StudyContentStatus.VALID,
workspace=DEFAULT_WORKSPACE_NAME,
owner=user,
groups=[group],
)
service = StorageService(
study_service=study_service,
importer_service=Mock(),
exporter_service=Mock(),
user_service=Mock(),
repository=repository,
event_bus=Mock(),
)
service._save_study(
RawStudy(id=uuid, workspace=DEFAULT_WORKSPACE_NAME),
owner=jwt,
)
repository.save.assert_called_once_with(study)
def test_copy_study(
tmp_path: str,
clean_ini_writer: Callable,
storage_service_builder,
) -> None:
path_studies = Path(tmp_path)
source_name = "study1"
path_study = path_studies / source_name
path_study.mkdir()
path_study_info = path_study / "study.antares"
path_study_info.touch()
value = {
"study": {
"antares": {
"caption": "ex1",
"created": 1480683452,
"lastsave": 1602678639,
"author": "unknown",
},
"output": [],
}
}
study = Mock()
study.get.return_value = value
study_factory = Mock()
config = Mock()
study_factory.create_from_fs.return_value = config, study
study_factory.create_from_config.return_value = study
url_engine = Mock()
url_engine.resolve.return_value = None, None, None
study_service = StudyService(
config=build_config(path_studies),
study_factory=study_factory,
path_resources=Path(),
)
src_md = RawStudy(id=source_name,
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_study))
dest_md = RawStudy(
id="study2",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(study_service.get_default_workspace_path() / "study2"),
)
md = study_service.copy_study(src_md, dest_md)
assert str(md.path) == f"{tmp_path}{os.sep}study2"
study.get.assert_called_once_with()
def _save_study(
self,
study: RawStudy,
owner: Optional[JWTUser] = None,
group_ids: List[str] = list(),
content_status: StudyContentStatus = StudyContentStatus.VALID,
) -> None:
if not owner:
raise UserHasNotPermissionError
info = self.study_service.get_study_information(study)["antares"]
study.name = info["caption"]
study.version = info["version"]
study.author = info["author"]
study.created_at = datetime.fromtimestamp(info["created"])
study.updated_at = datetime.fromtimestamp(info["lastsave"])
study.content_status = content_status
if owner:
study.owner = User(id=owner.impersonator)
groups = []
for gid in group_ids:
group = next(filter(lambda g: g.id == gid, owner.groups), None)
if group is None or not group.role.is_higher_or_equals(
RoleType.WRITER):
raise UserHasNotPermissionError()
groups.append(Group(id=group.id, name=group.name))
study.groups = groups
self.repository.save(study)
def create_study(self, study_name: str, group_ids: List[str],
params: RequestParameters) -> str:
sid = str(uuid4())
study_path = str(self.study_service.get_default_workspace_path() / sid)
raw = RawStudy(
id=sid,
name=study_name,
workspace=DEFAULT_WORKSPACE_NAME,
path=study_path,
)
raw = self.study_service.create_study(raw)
self._save_study(raw, params.user, group_ids)
self.event_bus.push(
Event(EventType.STUDY_CREATED, raw.to_json_summary()))
return str(raw.id)
def test_assert_study_not_exist(tmp_path: str, project_path) -> None:
# Create folders
tmp = Path(tmp_path)
(tmp / "study1").mkdir()
(tmp / "myfile").touch()
path_study2 = tmp / "study2.py"
path_study2.mkdir()
(path_study2 / "settings").mkdir()
# Input
study_name = "study3"
path_to_studies = Path(tmp_path)
# Test & Verify
study_service = StudyService(
config=build_config(path_to_studies),
study_factory=Mock(),
path_resources=project_path / "resources",
)
metadata = RawStudy(id=study_name,
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_study2))
with pytest.raises(StudyNotFoundError):
study_service.check_study_exists(metadata)
def test_exporter_file(tmp_path: Path, sta_mini_zip_path: Path):
path_studies = tmp_path / "studies"
with ZipFile(sta_mini_zip_path) as zip_output:
zip_output.extractall(path=path_studies)
config = Config(
resources_path=Path(),
security=SecurityConfig(disabled=True),
storage=StorageConfig(
workspaces={
DEFAULT_WORKSPACE_NAME: WorkspaceConfig(path=path_studies)
}
),
)
md = RawStudy(
id="STA-mini",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_studies / "STA-mini"),
)
repo = Mock()
repo.get.return_value = md
service = build_storage(
application=Mock(),
config=config,
session=Mock(),
user_service=Mock(),
metadata_repository=repo,
)
data = assert_url_content(service, url="/studies/STA-mini/export")
assert_data(data)
def test_create_study(tmp_path: str, storage_service_builder,
project_path) -> None:
path_studies = Path(tmp_path)
study = Mock()
data = {"study": {"antares": {"caption": None}}}
study.get.return_value = data
study_factory = Mock()
study_factory.create_from_fs.return_value = (None, study)
study_service = StudyService(
config=build_config(path_studies),
study_factory=study_factory,
path_resources=project_path / "resources",
)
metadata = RawStudy(
id="study1",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(study_service.get_default_workspace_path() / "study1"),
)
md = study_service.create_study(metadata)
assert md.path == f"{tmp_path}{os.sep}study1"
path_study = path_studies / md.id
assert path_study.exists()
path_study_antares_infos = path_study / "study.antares"
assert path_study_antares_infos.is_file()
def test_edit_study(tmp_path: Path, storage_service_builder) -> None:
# Mock
(tmp_path / "my-uuid").mkdir()
(tmp_path / "my-uuid/study.antares").touch()
study = Mock()
study_factory = Mock()
study_factory.create_from_fs.return_value = None, study
study_service = StudyService(
config=build_config(tmp_path),
study_factory=study_factory,
path_resources=Path(),
)
# Input
url = "url/to/change"
new = {"Hello": "World"}
md = RawStudy(
id="my-uuid",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(tmp_path / "my-uuid"),
)
res = study_service.edit_study(md, url, new)
assert new == res
study.save.assert_called_once_with(new, ["url", "to", "change"])
def test_study_inheritance():
engine = create_engine("sqlite:///:memory:", echo=True)
sess = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=engine))
user = User(id=0, name="admin")
group = Group(id="my-group", name="group")
Base.metadata.create_all(engine)
repo = StudyMetadataRepository(session=sess)
a = RawStudy(
name="a",
version="42",
author="John Smith",
created_at=datetime.now(),
updated_at=datetime.now(),
public_mode=PublicMode.FULL,
owner=user,
groups=[group],
workspace=DEFAULT_WORKSPACE_NAME,
path="study",
content_status=StudyContentStatus.WARNING,
)
repo.save(a)
b = repo.get(a.id)
assert isinstance(b, RawStudy)
assert b.path == "study"
def import_study(
self,
stream: IO[bytes],
group_ids: List[str],
params: RequestParameters,
) -> str:
sid = str(uuid4())
path = str(self.study_service.get_default_workspace_path() / sid)
study = RawStudy(id=sid, workspace=DEFAULT_WORKSPACE_NAME, path=path)
study = self.importer_service.import_study(study, stream)
status = self._analyse_study(study)
self._save_study(
study,
owner=params.user,
group_ids=group_ids,
content_status=status,
)
self.event_bus.push(
Event(EventType.STUDY_CREATED, study.to_json_summary()))
return str(study.id)
def sync_studies_on_disk(self, folders: List[StudyFolder]) -> None:
# delete orphan studies on database
paths = [str(f.path) for f in folders]
for study in self.repository.get_all():
if isinstance(
study,
RawStudy) and (study.workspace != DEFAULT_WORKSPACE_NAME
and study.path not in paths):
logger.info(
f"Study={study.id} is not present in disk and will be deleted"
)
self.event_bus.push(
Event(EventType.STUDY_DELETED, study.to_json_summary()))
self.repository.delete(study.id)
# Add new studies
paths = [
study.path for study in self.repository.get_all()
if isinstance(study, RawStudy)
]
for folder in folders:
if str(folder.path) not in paths:
study = RawStudy(
id=str(uuid4()),
name=folder.path.name,
path=str(folder.path),
workspace=folder.workspace,
owner=None,
groups=folder.groups,
public_mode=PublicMode.FULL
if len(folder.groups) == 0 else PublicMode.NONE,
)
study.content_status = self._analyse_study(study)
logger.info(
f"Study={study.id} appears on disk and will be added")
self.event_bus.push(
Event(EventType.STUDY_CREATED, study.to_json_summary()))
self.repository.save(study)
def storage_service(tmp_path: str, project_path: Path,
sta_mini_zip_path: Path) -> StorageService:
path_studies = Path(tmp_path) / "studies"
path_resources = project_path / "resources"
with ZipFile(sta_mini_zip_path) as zip_output:
zip_output.extractall(path=path_studies)
md = RawStudy(
id="STA-mini",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_studies / "STA-mini"),
)
repo = Mock()
repo.get.side_effect = lambda name: RawStudy(
id=name,
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_studies / name),
)
repo.get_all.return_value = [md]
config = Config(
resources_path=path_resources,
security=SecurityConfig(disabled=True),
storage=StorageConfig(
workspaces={
DEFAULT_WORKSPACE_NAME: WorkspaceConfig(path=path_studies)
}),
)
storage_service = build_storage(
application=Mock(),
session=Mock(),
user_service=Mock(),
config=config,
metadata_repository=repo,
)
return storage_service
def test_export_matrix(tmp_path: Path) -> None:
file = tmp_path / "file.txt"
file.write_bytes(b"Hello World")
service = Mock()
service.get_study_path.return_value = tmp_path
exporter = ExporterService(study_service=service,
study_factory=Mock(),
exporter=Mock())
md = RawStudy(id="id", workspace=DEFAULT_WORKSPACE_NAME)
assert exporter.get_matrix(md, "file.txt") == b"Hello World"
def create_study(self, metadata: RawStudy) -> RawStudy:
empty_study_zip = self.path_resources / "empty-study.zip"
path_study = self.get_study_path(metadata)
path_study.mkdir()
with ZipFile(empty_study_zip) as zip_output:
zip_output.extractall(path=path_study)
study_data = self.get(metadata, url="", depth=10)
StorageServiceUtils.update_antares_info(metadata, study_data)
_, study = self.study_factory.create_from_fs(path_study)
study.save(study_data)
metadata.path = str(path_study)
return metadata
def test_check_errors():
study = Mock()
study.check_errors.return_value = ["Hello"]
factory = Mock()
factory.create_from_fs.return_value = None, study
study_service = StudyService(
config=build_config(Path()),
study_factory=factory,
path_resources=Path(),
)
metadata = RawStudy(
id="study",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(study_service.get_default_workspace_path() / "study"),
)
assert study_service.check_errors(metadata) == ["Hello"]
def test_delete_study(tmp_path: Path, storage_service_builder) -> None:
name = "my-study"
study_path = tmp_path / name
study_path.mkdir()
(study_path / "study.antares").touch()
study_service = StudyService(
config=build_config(tmp_path),
study_factory=Mock(),
path_resources=Path(),
)
md = RawStudy(id=name,
workspace=DEFAULT_WORKSPACE_NAME,
path=str(study_path))
study_service.delete_study(md)
assert not study_path.exists()
def test_get(tmp_path: str, project_path) -> None:
"""
path_to_studies
|_study1 (d)
|_ study2.py
|_ settings (d)
|_myfile (f)
"""
# Create folders
path_to_studies = Path(tmp_path)
(path_to_studies / "study1").mkdir()
(path_to_studies / "myfile").touch()
path_study = path_to_studies / "study2.py"
path_study.mkdir()
(path_study / "settings").mkdir()
(path_study / "study.antares").touch()
data = {"titi": 43}
sub_route = "settings"
path = path_study / "settings"
key = "titi"
study = Mock()
study.get.return_value = data
study_factory = Mock()
study_factory.create_from_fs.return_value = (None, study)
study_service = StudyService(
config=build_config(path_to_studies),
study_factory=study_factory,
path_resources=project_path / "resources",
)
metadata = RawStudy(id="study2.py",
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_study))
output = study_service.get(metadata=metadata, url=sub_route, depth=2)
assert output == data
study.get.assert_called_once_with(["settings"], depth=2)
def copy_study(self, src_meta: RawStudy, dest_meta: RawStudy) -> RawStudy:
self.check_study_exists(src_meta)
src_path = self.get_study_path(src_meta)
config, study = self.study_factory.create_from_fs(src_path)
data_source = study.get()
del study
dest_meta.path = self.get_study_path(dest_meta)
config.path = dest_meta.path
data_destination = copy.deepcopy(data_source)
StorageServiceUtils.update_antares_info(dest_meta, data_destination)
if "output" in data_destination:
del data_destination["output"]
config.outputs = {}
study = self.study_factory.create_from_config(config)
study.save(data_destination)
del study
return dest_meta
def test_export_file(tmp_path: Path):
name = "my-study"
study_path = tmp_path / name
study_path.mkdir()
(study_path / "study.antares").touch()
exporter = Mock()
exporter.export_file.return_value = b"Hello"
study_service = Mock()
study_service.check_study_exist.return_value = None
exporter_service = ExporterService(
study_service=build_storage_service(tmp_path, name),
study_factory=Mock(),
exporter=exporter,
)
# Test good study
md = RawStudy(id=name, workspace=DEFAULT_WORKSPACE_NAME)
assert b"Hello" == exporter_service.export_study(md)
exporter.export_file.assert_called_once_with(study_path, True)
def test_assert_study_exist(tmp_path: str, project_path) -> None:
tmp = Path(tmp_path)
(tmp / "study1").mkdir()
(tmp / "study.antares").touch()
path_study2 = tmp / "study2.py"
path_study2.mkdir()
(path_study2 / "settings").mkdir()
(path_study2 / "study.antares").touch()
# Input
study_name = "study2.py"
path_to_studies = Path(tmp_path)
# Test & Verify
study_service = StudyService(
config=build_config(path_to_studies),
study_factory=Mock(),
path_resources=project_path / "resources",
)
metadata = RawStudy(id=study_name,
workspace=DEFAULT_WORKSPACE_NAME,
path=str(path_study2))
study_service.check_study_exists(metadata)
def test_create_study() -> None:
# Mock
repository = Mock()
# Input
user = User(id=0, name="user")
group = Group(id="my-group", name="group")
expected = RawStudy(
id=str(uuid4()),
name="new-study",
version="VERSION",
author="AUTHOR",
created_at=datetime.fromtimestamp(1234),
updated_at=datetime.fromtimestamp(9876),
content_status=StudyContentStatus.VALID,
workspace=DEFAULT_WORKSPACE_NAME,
owner=user,
groups=[group],
)
study_service = Mock()
study_service.get_default_workspace_path.return_value = Path("")
study_service.get_study_information.return_value = {
"antares": {
"caption": "CAPTION",
"version": "VERSION",
"author": "AUTHOR",
"created": 1234,
"lastsave": 9876,
}
}
study_service.create_study.return_value = expected
service = StorageService(
study_service=study_service,
importer_service=Mock(),
exporter_service=Mock(),
user_service=Mock(),
repository=repository,
event_bus=Mock(),
)
with pytest.raises(UserHasNotPermissionError):
service.create_study(
"new-study",
["my-group"],
RequestParameters(JWTUser(id=0, impersonator=0, type="users")),
)
service.create_study(
"new-study",
["my-group"],
RequestParameters(
JWTUser(
id=0,
impersonator=0,
type="users",
groups=[
JWTGroup(id="my-group", name="group", role=RoleType.WRITER)
],
)),
)
study_service.create_study.assert_called()
repository.save.assert_called_once_with(expected)
