async def test_create_snapshot_workflow(client, user_project: ProjectDict):
project_uuid = user_project["uuid"]
# get existing project
resp = await client.get(f"/{vtag}/projects/{project_uuid}")
data, _ = await assert_status(resp, web.HTTPOk)
assert data
project = Project.parse_obj(data)
# list snapshots -> None
resp = await client.get(f"/{vtag}/projects/{project_uuid}/snapshots")
data, _ = await assert_status(resp, web.HTTPOk)
assert data == []
# create snapshot
resp = await client.post(f"/{vtag}/projects/{project_uuid}/snapshots")
data, _ = await assert_status(resp, web.HTTPCreated)
assert data
snapshot = SnapshotItem.parse_obj(data)
assert snapshot.parent_uuid == project.uuid
# snapshot has an associated project
resp = await client.get(f"/{vtag}/projects/{snapshot.project_uuid}")
data, _ = await assert_status(resp, web.HTTPOk)
assert data
snapshot_project = Project.parse_obj(data)
# FIXME: project is None and snapshot_project is {}
project.ui.workbench = {}
project.ui.slideshow = {}
different_fields = {"name", "uuid", "creation_date", "last_change_date"}
assert snapshot_project.dict(
exclude=different_fields, exclude_none=True, exclude_unset=True
) == project.dict(exclude=different_fields, exclude_none=True, exclude_unset=True)
# snapshot projects are hidden, and therefore NOT listed
resp = await client.get(f"/{vtag}/projects")
data, _ = await assert_status(resp, web.HTTPOk)
assert len(data) == 1
# FIXME:
project.ui.workbench = None
project.ui.slideshow = None
assert project == Project.parse_obj(data[0])
# now it has one snapshot
resp = await client.get(f"/{vtag}/projects/{project_uuid}/snapshots")
data, _ = await assert_status(resp, web.HTTPOk)
assert len(data) == 1
assert snapshot == SnapshotItem.parse_obj(data[0])
async def add_new_project(app: web.Application, project: Project,
user: UserInfo):
# TODO: move this to projects_api
# TODO: this piece was taking fromt the end of projects.projects_handlers.create_projects
from ..projects.projects_db import APP_PROJECT_DBAPI
from ..director_v2 import create_or_update_pipeline
db = app[APP_PROJECT_DBAPI]
# validated project is transform in dict via json to use only primitive types
project_in: Dict = json.loads(
project.json(exclude_none=True, by_alias=True))
# update metadata (uuid, timestamps, ownership) and save
_project_db: Dict = await db.add_project(project_in,
user.id,
force_as_template=False)
assert _project_db["uuid"] == str(project.uuid) # nosec
# This is a new project and every new graph needs to be reflected in the pipeline db
#
# TODO: Ensure this user has access to these services!
#
await create_or_update_pipeline(app, user.id, project.uuid)
def test_project_with_thumbnail_as_empty_string(minimal_project: Dict[str,
Any]):
thumbnail_empty_string = deepcopy(minimal_project)
thumbnail_empty_string.update({"thumbnail": ""})
project = Project.parse_obj(thumbnail_empty_string)
assert project
assert project.thumbnail == None
async def take_snapshot(
parent: ProjectDict,
snapshot_label: Optional[str] = None,
) -> Tuple[ProjectDict, Snapshot]:
assert Project.parse_obj(parent) # nosec
# Clones parent's project document
snapshot_timestamp: datetime = parent["lastChangeDate"]
snapshot_project_uuid: UUID = Snapshot.compose_project_uuid(
parent["uuid"], snapshot_timestamp
)
child: ProjectDict
child, _ = projects_utils.clone_project_document(
project=parent,
forced_copy_project_id=snapshot_project_uuid,
)
assert child # nosec
assert Project.parse_obj(child) # nosec
child["name"] += snapshot_label or f" [snapshot {snapshot_timestamp}]"
# creation_date = state of parent upon copy! WARNING: changes can be state changes and not project definition?
child["creationDate"] = snapshot_timestamp
child["hidden"] = True
child["published"] = False
snapshot = Snapshot(
name=snapshot_label or f"Snapshot {snapshot_timestamp} [{parent['name']}]",
created_at=snapshot_timestamp,
parent_uuid=parent["uuid"],
project_uuid=child["uuid"],
)
return (child, snapshot)
async def add_new_project(app: web.Application, project: Project, user_id: int):
# TODO: move this to projects_api
# TODO: this piece was taking fromt the end of projects.projects_handlers.create_projects
db = app[APP_PROJECT_DBAPI]
# validated project is transform in dict via json to use only primitive types
project_in: Dict = json.loads(
project.json(exclude_none=True, by_alias=True, exclude_unset=True)
)
# update metadata (uuid, timestamps, ownership) and save
_project_db: Dict = await db.add_project(
project_in, user_id, force_as_template=False
)
if _project_db["uuid"] != str(project.uuid):
raise ExporterException("Project uuid dose nto match after validation")
await create_or_update_pipeline(app, user_id, project.uuid)
async def test_create_snapshot(client, user_project: ProjectDict):
project_uuid = user_project["uuid"]
# create snapshot
resp = await client.post(f"/{vtag}/projects/{project_uuid}/snapshots")
data, _ = await assert_status(resp, web.HTTPCreated)
assert data
snapshot = SnapshotItem.parse_obj(data)
assert str(snapshot.parent_uuid) == project_uuid
# snapshot project can be now retrieved
resp = await client.get(f"/{vtag}/projects/{snapshot.project_uuid}")
data, _ = await assert_status(resp, web.HTTPOk)
assert data
snapshot_project = Project.parse_obj(data)
assert snapshot_project.uuid == snapshot.project_uuid
async def test_take_snapshot(user_project: ProjectDict):
project, snapshot = await take_snapshot(parent=user_project,
snapshot_label="some snapshot")
assert isinstance(project, dict)
assert isinstance(snapshot, Snapshot)
# project overrides ORM-only fields
assert project["hidden"]
assert not project["published"]
# domain models
parent = Project.parse_obj(user_project)
# snapshot timestamp corresponds to the last change of the project
def to_dt(timestamp):
return datetime.fromisoformat(
timestamp[:-1]).replace(tzinfo=timezone.utc)
assert snapshot.created_at == to_dt(parent.last_change_date)
assert to_dt(project["creationDate"]) == snapshot.created_at
async def import_files_and_validate_project(
app: web.Application,
user_id: int,
root_folder: Path,
manifest_root_folder: Optional[Path],
) -> str:
project_file = await ProjectFile.model_from_file(root_dir=root_folder)
shuffled_data: ShuffledData = project_file.get_shuffled_uuids()
# replace shuffled_data in project
# NOTE: there is no reason to write the shuffled data to file
log.debug("Loaded project data: %s", project_file)
shuffled_project_file = project_file.new_instance_from_shuffled_data(
shuffled_data=shuffled_data
)
# creating an unique name to help the user distinguish
# between the original and new study
shuffled_project_file.name = "%s %s" % (
shuffled_project_file.name,
datetime.datetime.utcnow().strftime("%Y:%m:%d:%H:%M:%S"),
)
log.debug("Shuffled project data: %s", shuffled_project_file)
# NOTE: it is not necessary to apply data shuffling to the manifest
manifest_file = await ManifestFile.model_from_file(
root_dir=manifest_root_folder or root_folder
)
user: Dict = await get_user(app=app, user_id=user_id)
# create and add the project
project = Project(
uuid=shuffled_project_file.uuid,
name=shuffled_project_file.name,
description=shuffled_project_file.description,
thumbnail=shuffled_project_file.thumbnail,
prjOwner=user["email"],
accessRights={
user["primary_gid"]: AccessRights(read=True, write=True, delete=True)
},
creationDate=now_str(),
lastChangeDate=now_str(),
dev=shuffled_project_file.dev,
workbench=shuffled_project_file.workbench,
ui=shuffled_project_file.ui,
quality=shuffled_project_file.quality,
)
project_uuid = str(project.uuid)
try:
await _remove_runtime_states(project)
await add_new_project(app, project, user_id)
# upload files to storage
links_to_new_e_tags = await _upload_files_to_storage(
app=app,
user_id=user_id,
root_folder=root_folder,
manifest_file=manifest_file,
shuffled_data=shuffled_data,
)
# fix etags
await _fix_file_e_tags(project, links_to_new_e_tags)
# NOTE: first fix the file eTags, and then the run hashes
await _fix_node_run_hashes_based_on_old_project(
project, project_file, shuffled_data
)
except Exception as e:
log.warning(
"The below error occurred during import\n%s", traceback.format_exc()
)
log.warning(
"Removing project %s, because there was an error while importing it."
)
try:
await delete_project(app=app, project_uuid=project_uuid, user_id=user_id)
except ProjectsException:
# no need to raise an error here
log.exception(
"Could not find project %s while trying to revert actions", project_uuid
)
raise e
return project_uuid
async def test_workflow(
client: TestClient,
user_project: ProjectDict,
do_update_user_project: Callable[[UUID], Awaitable],
):
project_uuid = user_project["uuid"]
# get existing project
resp = await client.get(f"/{vtag}/projects/{project_uuid}")
data, _ = await assert_status(resp, web.HTTPOk)
project = Project.parse_obj(data)
assert project.uuid == UUID(project_uuid)
#
# list repos i.e. versioned projects
resp = await client.get(f"/{vtag}/repos/projects")
data, _ = await assert_status(resp, web.HTTPOk)
assert data == []
#
# CREATE a checkpoint
resp = await client.post(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints",
json={
"tag": "v1",
"message": "init"
},
)
data, _ = await assert_status(resp, web.HTTPCreated)
assert data
checkpoint1 = CheckpointApiModel.parse_obj(
data) # NOTE: this is NOT API model
#
# this project now has a repo
resp = await client.get(f"/{vtag}/repos/projects")
page = await assert_resp_page(resp, expected_total=1, expected_count=1)
repo = RepoApiModel.parse_obj(page.data[0])
assert repo.project_uuid == UUID(project_uuid)
# GET checkpoint with HEAD
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/HEAD")
data, _ = await assert_status(resp, web.HTTPOk)
assert CheckpointApiModel.parse_obj(data) == checkpoint1
# TODO: GET checkpoint with tag
with pytest.raises(aiohttp.ClientResponseError) as excinfo:
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/v1")
resp.raise_for_status()
assert CheckpointApiModel.parse_obj(data) == checkpoint1
assert excinfo.value.status == web.HTTPNotImplemented.status_code
# GET checkpoint with id
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}")
assert str(resp.url) == checkpoint1.url
assert CheckpointApiModel.parse_obj(data) == checkpoint1
# LIST checkpoints
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints")
page = await assert_resp_page(resp, expected_total=1, expected_count=1)
assert CheckpointApiModel.parse_obj(page.data[0]) == checkpoint1
# UPDATE checkpoint annotations
resp = await client.patch(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}",
json={
"message": "updated message",
"tag": "Version 1"
},
)
data, _ = await assert_status(resp, web.HTTPOk)
checkpoint1_updated = CheckpointApiModel.parse_obj(data)
assert checkpoint1.id == checkpoint1_updated.id
assert checkpoint1.checksum == checkpoint1_updated.checksum
assert checkpoint1_updated.tags == ("Version 1", )
assert checkpoint1_updated.message == "updated message"
# GET view
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/HEAD/workbench/view"
)
data, _ = await assert_status(resp, web.HTTPOk)
assert (data["workbench"] == project.dict(exclude_none=True,
exclude_unset=True)["workbench"])
# do some changes in project
await do_update_user_project(project.uuid)
# CREATE new checkpoint
resp = await client.post(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints",
json={
"tag": "v2",
"message": "new commit"
},
)
data, _ = await assert_status(resp, web.HTTPCreated)
checkpoint2 = CheckpointApiModel.parse_obj(data)
assert checkpoint2.tags == ("v2", )
# GET checkpoint with HEAD
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/HEAD")
data, _ = await assert_status(resp, web.HTTPOk)
assert CheckpointApiModel.parse_obj(data) == checkpoint2
# CHECKOUT
resp = await client.post(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}:checkout"
)
data, _ = await assert_status(resp, web.HTTPOk)
assert CheckpointApiModel.parse_obj(data) == checkpoint1_updated
# GET checkpoint with HEAD
resp = await client.get(
f"/{vtag}/repos/projects/{project_uuid}/checkpoints/HEAD")
data, _ = await assert_status(resp, web.HTTPOk)
assert CheckpointApiModel.parse_obj(data) == checkpoint1_updated
# get working copy
resp = await client.get(f"/{vtag}/projects/{project_uuid}")
data, _ = await assert_status(resp, web.HTTPOk)
project_wc = Project.parse_obj(data)
assert project_wc.uuid == UUID(project_uuid)
assert project_wc != project
def test_project_minimal_model(minimal_project: Dict[str, Any]):
project = Project.parse_obj(minimal_project)
assert project
assert project.thumbnail == None
def create_viewer_project_model(
project_id: str,
file_picker_id: str,
viewer_id: str,
owner: UserInfo,
download_link: str,
viewer_info: ViewerInfo,
) -> Project:
file_picker_output_id = "outFile"
file_picker = Node(
key="simcore/services/frontend/file-picker",
version="1.0.0", # TODO: check with odeimaiz about version here
label="File Picker",
inputs={},
inputNodes=[],
outputs={
# TODO: check with odeimaiz what is the meaning and relevance of label
file_picker_output_id:
DownloadLink(downloadLink=str(download_link), label="")
},
progress=0,
)
viewer_service = Node(
key=viewer_info.key,
version=viewer_info.version,
label=viewer_info.label,
inputs={
viewer_info.input_port_key:
PortLink(nodeUuid=file_picker_id, output=file_picker_output_id)
},
inputNodes=[
file_picker_id,
],
)
# Access rights policy
access_rights = AccessRights(read=True, write=True,
delete=True) # will keep a copy
if owner.is_guest:
access_rights.write = access_rights.delete = False
# Assambles project instance
project = Project(
uuid=project_id,
name=f"Viewer {viewer_info.title}",
description="Temporary study to visualize downloaded file",
thumbnail="https://placeimg.com/171/96/tech/grayscale/?0.jpg",
prjOwner=owner.email,
accessRights={owner.primary_gid: access_rights},
creationDate=now_str(),
lastChangeDate=now_str(),
workbench={
file_picker_id: file_picker,
viewer_id: viewer_service
},
ui=StudyUI(
workbench={
file_picker_id: {
"position": {
"x": 305,
"y": 229
}
},
viewer_id: {
"position": {
"x": 633,
"y": 229
}
},
}),
)
return project
results = Page[ProjectIterationResultItem].parse_obj(body).data
# GET project and MODIFY iterator values----------------------------------------------
# - Change iterations from 0:4 -> HEAD+1
resp = await client.get(f"/v0/projects/{project_uuid}")
assert resp.status == HTTPStatus.OK, await resp.text()
body = await resp.json()
# NOTE: updating a project fields can be daunting because
# it combines nested field attributes with dicts and from the
# json you cannot distinguish easily what-is-what automatically
# Dict keys are usually some sort of identifier, typically a UUID or
# and index but nothing prevents a dict from using any other type of key types
#
project = Project.parse_obj(body["data"])
new_project = project.copy(
update={
# TODO: HACK to overcome export from None -> string
# SOLUTION 1: thumbnail should not be required (check with team!)
# SOLUTION 2: make thumbnail nullable
"thumbnail": faker.image_url(),
}
)
assert new_project.workbench is not None
assert new_project.workbench
node = new_project.workbench["fc9208d9-1a0a-430c-9951-9feaf1de3368"]
assert node.inputs
node.inputs["linspace_stop"] = 4
resp = await client.put(