def test_sync_with_empty_database(prepare_environment: Any) -> None:
"""Test for configuration synchronization with empty database."""
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- KIDNEYS_SEGMENTATION
tasks:
- key: KIDNEYS_SEGMENTATION
name: Kidneys segmentation
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: LEFT_KIDNEY
name: Left Kidney
tools:
- CHAIN
- BRUSH
- key: RIGHT_KIDNEY
name: Right Kidney
tools:
- CHAIN
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 1
kidneys = DatasetsRepository.get_dataset_by_key('KIDNEYS')
assert kidneys.name == 'Kidneys'
assert not kidneys.disabled
assert {task.key for task in kidneys.tasks} == {'KIDNEYS_SEGMENTATION'}
# Check if Tasks were synchronized properly
tasks = TasksRepository.get_all_tasks(include_disabled=True)
assert len(tasks) == 1
kidneys_segmentation = TasksRepository.get_task_by_key('KIDNEYS_SEGMENTATION')
assert kidneys_segmentation.name == 'Kidneys segmentation'
assert kidneys_segmentation.image_path == 'assets/icon/kidneys_dataset_icon.svg'
assert kidneys_segmentation.description == 'This is a test task'
assert kidneys_segmentation.label_examples == ['assets/example_1', 'assets/example_2']
assert not kidneys_segmentation.disabled
assert len(kidneys_segmentation.available_tags) == 2
assert {tag.key for tag in kidneys_segmentation.available_tags} == {'LEFT_KIDNEY', 'RIGHT_KIDNEY'}
# Check if Label Tags were synchronized properly
tags = LabelTagsRepository.get_all_tags(include_disabled=True)
assert len(tags) == 2
left_kidney = LabelTagsRepository.get_label_tag_by_key('LEFT_KIDNEY')
assert left_kidney.name == 'Left Kidney'
assert not left_kidney.disabled
assert {tool.name for tool in left_kidney.tools} == {'CHAIN', 'BRUSH'}
right_kidney = LabelTagsRepository.get_label_tag_by_key('RIGHT_KIDNEY')
assert right_kidney.name == 'Right Kidney'
assert not right_kidney.disabled
assert {tool.name for tool in right_kidney.tools} == {'CHAIN'}
def _sync_datasets(configuration: Dict) -> None:
"""Synchronize Datasets from configuration file with database entries.
Example DataSets in the configuration file:
```
datasets:
- name: Kidneys
key: KIDNEYS
tasks:
- KIDNEYS_SEGMENTATION
```
:param configuration: content of YAML configuration file
"""
datasets = configuration.get('datasets', []) or []
configuration_datasets_keys = {dataset['key'] for dataset in datasets}
database_datasets_keys = {dataset.key for dataset in DatasetsRepository.get_all_datasets(include_disabled=True)}
datasets_to_add = configuration_datasets_keys - database_datasets_keys
datasets_to_disable = database_datasets_keys - configuration_datasets_keys
datasets_to_enable = database_datasets_keys & configuration_datasets_keys
for dataset_key in datasets_to_add:
dataset = next(dataset for dataset in datasets if dataset['key'] == dataset_key)
DatasetsRepository.add_new_dataset(dataset['key'], dataset['name'])
logger.info('New DataSet added: %s', dataset['key'])
for dataset_key in datasets_to_enable:
dataset = next(dataset for dataset in datasets if dataset['key'] == dataset_key)
DatasetsRepository.enable(dataset['key'])
DatasetsRepository.update(dataset['key'], dataset['name'])
logger.info('DataSet enabled: %s', dataset['key'])
for dataset_key in datasets_to_disable:
DatasetsRepository.disable(dataset_key)
logger.info('DataSet disabled: %s', dataset_key)
def test_sync_with_changed_dataset_and_reused_task(prepare_environment: Any) -> None:
"""Test for changing DataSet and reusing Task."""
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- FIND_NODULES
tasks:
- key: FIND_NODULES
name: Find nodules
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: NODULE
name: Nodule
tools:
- BRUSH
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 1
kidneys = DatasetsRepository.get_dataset_by_key('KIDNEYS')
assert kidneys.name == 'Kidneys'
assert not kidneys.disabled
assert {task.key for task in kidneys.tasks} == {'FIND_NODULES'}
# Now, let's change Tools for Kidneys Segmentation
configuration = yaml.load("""
datasets:
- key: LUNGS
name: Lungs
tasks:
- FIND_NODULES
tasks:
- key: FIND_NODULES
name: Find nodules
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task with updated description
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: NODULE
name: Nodule
tools:
- BRUSH
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 2
kidneys = DatasetsRepository.get_dataset_by_key('KIDNEYS')
assert kidneys.name == 'Kidneys'
assert kidneys.disabled
assert {task.key for task in kidneys.tasks} == set()
lungs = DatasetsRepository.get_dataset_by_key('LUNGS')
assert lungs.name == 'Lungs'
assert not lungs.disabled
assert {task.key for task in lungs.tasks} == {'FIND_NODULES'}
def test_sync_with_changed_task_in_dataset(mocker: Any, prepare_environment: Any) -> None:
"""Test for adding and removing Task in DataSet."""
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- KIDNEYS_SEGMENTATION
tasks:
- key: KIDNEYS_SEGMENTATION
name: Kidneys segmentation
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: LEFT_KIDNEY
name: Left Kidney
tools:
- CHAIN
- BRUSH
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 1
# Check if Tasks were synchronized properly
tasks = TasksRepository.get_all_tasks(include_disabled=True)
assert len(tasks) == 1
kidneys_segmentation = TasksRepository.get_task_by_key('KIDNEYS_SEGMENTATION')
assert kidneys_segmentation.name == 'Kidneys segmentation'
assert not kidneys_segmentation.disabled
# Now, let's change Tasks for Kidneys
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- KIDNEYS_SEGMENTATION
tasks:
- key: FIND_NODULES
name: Find nodules
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: NODULE
name: Nodule
tools:
- BRUSH
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 1
# Check if Tasks were synchronized properly
tasks = TasksRepository.get_all_tasks(include_disabled=True)
assert len(tasks) == 2
kidneys_segmentation = TasksRepository.get_task_by_key('KIDNEYS_SEGMENTATION')
assert kidneys_segmentation.name == 'Kidneys segmentation'
assert kidneys_segmentation.disabled
find_nodules = TasksRepository.get_task_by_key('FIND_NODULES')
assert find_nodules.name == 'Find nodules'
assert not find_nodules.disabled
# Now, let's change let's get back to older Tasks (with different values) for Kidneys
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- KIDNEYS_SEGMENTATION
tasks:
- key: KIDNEYS_SEGMENTATION
name: New Kidneys segmentation
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: NEW_LEFT_KIDNEY
name: New Left Kidney
tools:
- RECTANGLE
""")
script.sync_configuration(configuration)
# Check if Datasets were synchronized properly
datasets = DatasetsRepository.get_all_datasets(include_disabled=True)
assert len(datasets) == 1
# Check if Tasks were synchronized properly
tasks = TasksRepository.get_all_tasks(include_disabled=True)
assert len(tasks) == 2
find_nodules = TasksRepository.get_task_by_key('FIND_NODULES')
assert find_nodules.name == 'Find nodules'
assert find_nodules.disabled
kidneys_segmentation = TasksRepository.get_task_by_key('KIDNEYS_SEGMENTATION')
assert kidneys_segmentation.name == 'New Kidneys segmentation'
assert kidneys_segmentation.image_path == 'assets/icon/kidneys_dataset_icon.svg'
assert kidneys_segmentation.description == 'This is a test task'
assert kidneys_segmentation.label_examples == ['assets/example_1', 'assets/example_2']
assert not kidneys_segmentation.disabled
assert len(kidneys_segmentation.available_tags) == 1
assert {tag.key for tag in kidneys_segmentation.available_tags} == {'NEW_LEFT_KIDNEY'}
# Check if Label Tags were synchronized properly
tags = LabelTagsRepository.get_all_tags(include_disabled=True)
assert len(tags) == 3
left_kidney = LabelTagsRepository.get_label_tag_by_key('LEFT_KIDNEY')
assert left_kidney.disabled
nodule = LabelTagsRepository.get_label_tag_by_key('NODULE')
assert nodule.disabled
new_left_kidney = LabelTagsRepository.get_label_tag_by_key('NEW_LEFT_KIDNEY')
assert not new_left_kidney.disabled
# Now, let's change let's get back to the original Tasks for Kidneys
mocked_logger = mocker.patch.object(script, 'logger')
configuration = yaml.load("""
datasets:
- key: KIDNEYS
name: Kidneys
tasks:
- KIDNEYS_SEGMENTATION
tasks:
- key: KIDNEYS_SEGMENTATION
name: Kidneys segmentation
image_path: assets/icon/kidneys_dataset_icon.svg
description: This is a test task
label_examples: ['assets/example_1', 'assets/example_2']
tags:
- key: LEFT_KIDNEY
name: Left Kidney
tools:
- CHAIN
- BRUSH
""")
script.sync_configuration(configuration)
# Check if user was warned about possible data inconsistency
mocked_logger.warning.assert_called_once_with(
'Reusing previously existing Label Tag (%s)! This may cause data inconsistency! '
'Make sure you know what you are doing and clear database entries if necessary!',
'LEFT_KIDNEY',
)
# Check if Label Tags were synchronized properly
tags = LabelTagsRepository.get_all_tags(include_disabled=True)
assert len(tags) == 3
left_kidney = LabelTagsRepository.get_label_tag_by_key('LEFT_KIDNEY')
assert not left_kidney.disabled
nodule = LabelTagsRepository.get_label_tag_by_key('NODULE')
assert nodule.disabled
new_left_kidney = LabelTagsRepository.get_label_tag_by_key('NEW_LEFT_KIDNEY')
assert new_left_kidney.disabled
def get_available_datasets() -> List[Dataset]:
"""Fetch list of all available Datasets.
:return: list of Datasets
"""
return DatasetsRepository.get_all_datasets()
'active': user.active,
'role': user.role.name,
})
print('\nSaving Roles and Users to file: {}.'.format(ROLES_AND_USERS_FILE))
with open(ROLES_AND_USERS_FILE, 'w') as json_file:
json.dump(_users_and_roles, json_file)
# Backup all Scans and its Labels with DICOM files
_scans = {
'datasets': [],
'label_tags': [],
'scans': [],
} # type: Dict[str, List[Dict]]
print('\nFetching Datasets...')
for dataset in DatasetsRepository.get_all_datasets():
print('Saving Dataset: {}'.format(dataset.key))
_scans['datasets'].append({
'id': dataset.id,
'key': dataset.key,
'name': dataset.name,
})
print('\nFetching Label Tags...')
for label_tag in LabelTagsRepository.get_all_tags():
print('Saving Label Tag: {}'.format(label_tag.key))
_scans['label_tags'].append({
'id': label_tag.id,
'key': label_tag.key,
'name': label_tag.name,
})
print('\nSaving all DICOMs to directory...')
