def test_import_projects(self):
"""Import a set of new projects, stacks and stack groups. This tests
only the actual import. Retrieving the data to import from different
sources is not part of this test.
"""
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
existing_projects = list(Project.objects.all())
existing_project_ids = [p.id for p in existing_projects]
p1_config = {
'project': {
'title': 'test-no-stacks',
}
}
p2_config = {
'project': {
'title':
'test-two-stacks',
'stacks': [
{
# A basic stack, only with required information
'title':
'test-stack-1',
'dimension':
'(7, 17, 23)',
'resolution':
'(2, 3, 5)',
'zoomlevels':
-1,
'mirrors': [{
'title': 'test-mirror-1',
'fileextension': 'jpg'
}]
},
{
# A basic stack with a little more information
'title':
'test-stack-2',
'dimension':
'(7, 17, 23)',
'resolution':
'(2, 3, 5)',
'zoomlevels':
-1,
'mirrors': [{
'title': 'test-mirror-2',
'fileextension': 'jpg',
'url': 'https://this.is.my.stack/'
}]
},
{
# A stack with all optional properties
'title':
'test-stack-3',
'dimension':
'(4, 34, 9)',
'resolution':
'(1, 2, 3)',
'metadata':
'Test meta data',
'zoomlevels':
-1,
'translation':
'(10, 20, 30)',
'mirrors': [{
'title': 'test-mirror-3',
'folder': 'abc/',
'fileextension': 'jpg',
'tile_width': 123,
'tile_height': 456,
'tile_source_type': 2,
}],
'stackgroups': [{
# Add a single stack group with only this stack
# in it.
'title': 'Test group 1',
'relation': 'view',
}],
}
]
}
}
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 5
default_position = 0
cls_graph_ids_to_link = []
remove_unref_stack_data = False
imported, not_imported = importer.import_projects(
self.user, pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertListEqual(pre_projects, imported)
self.assertListEqual([], not_imported)
new_projects = list(
Project.objects.exclude(
id__in=existing_project_ids).order_by('title'))
self.assertEqual(2, len(new_projects))
# Projects should be ordered by name, so the first project will be based
# on p1_config. Test p1 first, it is not expected to have any stacks.
p1 = new_projects[0]
self.assertEqual(p1_config['project']['title'], p1.title)
self.assertEqual(0, p1.stacks.all().count())
# Test p2.
p2 = new_projects[1]
self.assertEqual(p2_config['project']['title'], p2.title)
p2_stacks = p2.stacks.all().order_by('title')
self.assertEqual(3, len(p2_stacks))
p2cfg_stacks = p2_config['project']['stacks']
for n, p2s in enumerate(p2_stacks):
stack = p2cfg_stacks[n]
# Test required fields
self.assertEqual(stack['title'], p2s.title)
six.assertCountEqual(self, literal_eval(stack['dimension']),
literal_eval(str(p2s.dimension)))
six.assertCountEqual(self, literal_eval(stack['resolution']),
literal_eval(str(p2s.resolution)))
self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels)
# Test mirrors
mirrors = p2s.stackmirror_set.all().order_by('title')
self.assertEqual(len(stack['mirrors']), len(mirrors))
for m, omirror in enumerate(mirrors):
mirror = stack['mirrors'][m]
self.assertEqual(mirror['title'], omirror.title)
self.assertEqual(mirror['fileextension'],
omirror.file_extension)
# Test fields with potential default values
self.assertEqual(mirror.get('position', default_position),
omirror.position)
self.assertEqual(mirror.get('tile_width', default_tile_width),
omirror.tile_width)
self.assertEqual(
mirror.get('tile_height', default_tile_height),
omirror.tile_height)
self.assertEqual(
mirror.get('tile_source_type', default_tile_source_type),
omirror.tile_source_type)
if 'url' in mirror:
image_base = mirror['url']
else:
image_base = urljoin(
project_url,
urljoin(mirror.get('path', ''),
mirror.get('folder', '')))
self.assertEqual(image_base, omirror.image_base)
# Test project-stack link
ps = ProjectStack.objects.get(project=p2.id, stack=p2s)
six.assertCountEqual(
self, literal_eval(stack.get('translation', '(0,0,0)')),
literal_eval(str(ps.translation)))
# Test stack groups
ostack_group_links = StackStackGroup.objects.filter(
stack=p2s).order_by('stack__title')
stack_groups = stack.get('stackgroups', [])
self.assertEqual(len(ostack_group_links), len(stack_groups))
for m, sg_cfg in enumerate(stack_groups):
ostack_group_link = ostack_group_links[m]
ostack_group = ostack_group_link.stack_group
self.assertEqual(sg_cfg['title'], ostack_group.title)
self.assertEqual(sg_cfg['relation'],
ostack_group_link.group_relation.name)
self.assertEqual(sg_cfg.get('position', default_position),
ostack_group_link.position)
def test_import_export_projects(self):
"""Export all projects, stacks and stack groups (without class instance
and tracing data). Make then sure, they match the fixture.
"""
p1_config = {
'project': {
'title': 'test-no-stacks',
'stacks': tuple(),
}
}
p2_config = {
'project': {
'title':
'test-two-stacks',
'stacks': [{
'broken_sections': [],
'title':
'test-stack-1',
'dimension':
'(7,17,23)',
'resolution':
'(2,3,5)',
'zoomlevels':
-1,
'orientation':
0,
'translation':
'(0,0,0)',
'metadata':
'',
'comment':
'Test comment',
'attribution':
'Test attribution',
'description':
'Simple test data',
'canary_location':
'(0,0,0)',
'placeholder_color':
'(0,0,0,1)',
'mirrors': [{
'title': 'test-mirror-1',
'url': 'https://catmaid-test/',
'tile_height': 512,
'tile_width': 256,
'fileextension': 'jpg',
'tile_source_type': 5,
'position': 2
}]
}, {
'broken_sections': [],
'comment':
None,
'title':
'test-stack-2',
'dimension':
'(7,17,23)',
'metadata':
'',
'resolution':
'(2,3,5)',
'zoomlevels':
-1,
'orientation':
0,
'translation':
'(0,0,0)',
'attribution':
None,
'description':
'',
'canary_location':
'(0,0,0)',
'placeholder_color':
'(0.5,0.4,0.3,1)',
'mirrors': [{
'title': 'test-mirror-2',
'position': 0,
'url': 'https://this.is.my.stack/',
'tile_height': 400,
'tile_width': 300,
'fileextension': 'jpg',
'tile_source_type': 5,
}]
}, {
'broken_sections': [],
'comment':
None,
'title':
'test-stack-3',
'dimension':
'(4,34,9)',
'metadata':
'Test meta data',
'resolution':
'(1,2,3)',
'zoomlevels':
-1,
'orientation':
0,
'translation':
'(0,0,0)',
'attribution':
None,
'description':
'',
'canary_location':
'(1,2,3)',
'placeholder_color':
'(0,0,0.3,0.1)',
'mirrors': [{
'title': 'test-mirror-3',
'position': 0,
'url': 'https://catmaid-test/abc/',
'tile_height': 456,
'tile_width': 123,
'fileextension': 'jpg',
'tile_source_type': 2,
}],
'stackgroups': [{
'relation': 'view',
'title': u'Test group 1'
}],
}]
}
}
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
config = [p1_config, p2_config]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 1
cls_graph_ids_to_link = []
remove_unref_stack_data = False
# Make sure there are no existing projects or stacks
Project.objects.all().delete()
Stack.objects.all().delete()
imported, not_imported = importer.import_projects(
self.user, pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertEqual(0, len(not_imported))
self.assertEqual(len(config), len(imported))
# Make sure we can see all projects
for p in Project.objects.all():
assign_perm('can_browse', self.user, p)
# Export imported data
self.fake_authentication()
response = self.client.get('/projects/export')
self.assertEqual(response.status_code, 200)
result = yaml.load(response.content.decode('utf-8'))
def strip_ids(d):
""" Recursively, strip all 'id' fields of dictionaries.
"""
if type(d) == dict:
if 'id' in d:
d.pop('id')
for _, v in d.items():
strip_ids(v)
if type(d) == list:
for v in d:
strip_ids(v)
# Results come with IDs, which we don't have in our input data. Strip
# them to be able to simply compare dictionaries.
strip_ids(result)
for cp, p in zip(config, result):
self.assertDictEqual(cp, p)
def test_import_projects(self):
"""Import a set of new projects, stacks and stack groups. This tests
only the actual import. Retrieving the data to import from different
sources is not part of this test.
"""
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
existing_projects = list(Project.objects.all())
existing_project_ids = [p.id for p in existing_projects]
p1_config = {
'project': {
'title': 'test-no-stacks',
}
}
p2_config = {
'project': {
'title': 'test-two-stacks',
'stacks': [
{
# A basic stack, only with required information
'title': 'test-stack-1',
'dimension': '(7, 17, 23)',
'resolution': '(2, 3, 5)',
'zoomlevels': -1,
'mirrors': [{
'title': 'test-mirror-1',
'fileextension': 'jpg'
}]
},
{
# A basic stack with a little more information
'title': 'test-stack-2',
'dimension': '(7, 17, 23)',
'resolution': '(2, 3, 5)',
'zoomlevels': -1,
'mirrors': [{
'title': 'test-mirror-2',
'fileextension': 'jpg',
'url': 'https://this.is.my.stack/'
}]
},
{
# A stack with all optional properties
'title': 'test-stack-3',
'dimension': '(4, 34, 9)',
'resolution': '(1, 2, 3)',
'metadata': 'Test meta data',
'zoomlevels': -1,
'translation': '(10, 20, 30)',
'mirrors': [{
'title': 'test-mirror-3',
'folder': 'abc/',
'fileextension': 'jpg',
'tile_width': 123,
'tile_height': 456,
'tile_source_type': 2,
}],
'stackgroups': [{
# Add a single stack group with only this stack
# in it.
'title': 'Test group 1',
'relation': 'view',
}],
}
]
}
}
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 5
default_position = 0
cls_graph_ids_to_link = []
remove_unref_stack_data = False
imported, not_imported = importer.import_projects(self.user,
pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertListEqual(pre_projects, imported)
self.assertListEqual([], not_imported)
new_projects = list(Project.objects.exclude(id__in=existing_project_ids).order_by('title'))
self.assertEqual(2, len(new_projects))
# Projects should be ordered by name, so the first project will be based
# on p1_config. Test p1 first, it is not expected to have any stacks.
p1 = new_projects[0]
self.assertEqual(p1_config['project']['title'], p1.title)
self.assertEqual(0, p1.stacks.all().count())
# Test p2.
p2 = new_projects[1]
self.assertEqual(p2_config['project']['title'], p2.title)
p2_stacks = p2.stacks.all().order_by('title')
self.assertEqual(3, len(p2_stacks))
p2cfg_stacks = p2_config['project']['stacks']
for n, p2s in enumerate(p2_stacks):
stack = p2cfg_stacks[n]
# Test required fields
self.assertEqual(stack['title'], p2s.title)
six.assertCountEqual(self, literal_eval(stack['dimension']),
literal_eval(str(p2s.dimension)))
six.assertCountEqual(self, literal_eval(stack['resolution']),
literal_eval(str(p2s.resolution)))
self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels)
# Test mirrors
mirrors = p2s.stackmirror_set.all().order_by('title')
self.assertEqual(len(stack['mirrors']), len(mirrors))
for m, omirror in enumerate(mirrors):
mirror = stack['mirrors'][m]
self.assertEqual(mirror['title'], omirror.title)
self.assertEqual(mirror['fileextension'], omirror.file_extension)
# Test fields with potential default values
self.assertEqual(mirror.get('position', default_position),
omirror.position)
self.assertEqual(mirror.get('tile_width', default_tile_width),
omirror.tile_width)
self.assertEqual(mirror.get('tile_height', default_tile_height),
omirror.tile_height)
self.assertEqual(mirror.get('tile_source_type', default_tile_source_type),
omirror.tile_source_type)
if 'url' in mirror:
image_base = mirror['url']
else:
image_base = urljoin(project_url,
urljoin(mirror.get('path', ''), mirror.get('folder', '')))
self.assertEqual(image_base, omirror.image_base)
# Test project-stack link
ps = ProjectStack.objects.get(project=p2.id, stack=p2s)
six.assertCountEqual(self, literal_eval(stack.get('translation', '(0,0,0)')),
literal_eval(str(ps.translation)))
# Test stack groups
ostack_group_links = StackStackGroup.objects.filter(stack=p2s).order_by('stack__title')
stack_groups = stack.get('stackgroups', [])
self.assertEqual(len(ostack_group_links), len(stack_groups))
for m, sg_cfg in enumerate(stack_groups):
ostack_group_link = ostack_group_links[m]
ostack_group = ostack_group_link.stack_group
self.assertEqual(sg_cfg['title'], ostack_group.title)
self.assertEqual(sg_cfg['relation'],
ostack_group_link.group_relation.name)
self.assertEqual(sg_cfg.get('position', default_position),
ostack_group_link.position)
def handle(self, *args, **options):
ignore_same_name_projects = options['ignore_same_name_projects']
ignore_same_name_stacks = options['ignore_same_name_stacks']
ignore_empty_projects = options['ignore_empty_projects']
project_id = options['project_id']
default_tile_width = options['default_tile_width']
default_tile_height = options['default_tile_height']
default_tile_source_type = options['default_tile_source_type']
remove_unref_stack_data = options['remove_unref_stack_data']
image_base = options['image_base']
if ignore_same_name_projects:
logger.info("Ignoring all loaded projects that have same name as "
"existing projects")
if ignore_same_name_stacks:
logger.info("Ignoring all loaded stacks that have same name as "
"existing stacks")
# Parse permissions
permissions:List = []
for p in map(lambda x: x.split(':'), options['permissions']):
if len(p) != 3:
raise CommandError('Invalid permission format, expected: type:name:permission')
p_type, obj_name, p_name = p[0].lower(), p[1], p[2]
if p_type == 'user':
target = User.objects.get(username=obj_name)
elif p_type == 'group':
target = Group.objects.get(groupname=obj_name)
else:
raise CommandError(f'Unknown permission target type: {p_type}')
logger.info(f'Setting {p_name} permissions for {p_type} {obj_name}')
permissions.append((target, p_name))
# This will read from either stdin or a provided text file
if options['input'].isatty():
raise CommandError('Please provide either the --input argument '
'with a file path or provide data on stdin.')
input_data = options['input'].read()
options['input'].close()
project_configs = json.loads(input_data)
pre_projects:List = []
for project_config in project_configs:
title = project_config['project']['title']
if ignore_same_name_projects and \
Project.objects.filter(title=title).count() > 0:
logger.info(f"Skipping project {title}, a project with the same name exists alrady")
continue
logger.info(f"Parsing project {title}")
pre_project = PreProject(project_config, image_base, None)
stacks_to_remove = []
for pre_stack in pre_project.stacks:
if Stack.objects.filter(title=pre_stack.title).count() > 0:
stacks_to_remove.append(pre_stack)
if stacks_to_remove:
stack_titles = ', '.join(map(lambda x: x.title, stacks_to_remove))
logger.info(f"Skipping stacks {stack_titles} in project {title}, "
"because stacks with these names exist alrady")
for stack_to_remove in stacks_to_remove:
pre_project.stacks.remove(stack_to_remove)
if ignore_empty_projects and not pre_project.stacks:
logger.info(f"Skipping project {title}, because it has no stacks")
continue
pre_projects.append(pre_project)
tags:List = []
cls_graph_ids_to_link:List = []
user = get_system_user()
logger.info(f'Importing {len(pre_projects)} projects')
imported, not_imported = import_projects(user,
pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
logger.info(f'Imported {len(imported)} projects')
if not_imported:
logger.info("Encountered the following problems during import:\n" +
'\n'.join(map(lambda x: f'{x[0]}: {x[1]}', not_imported)))
def test_import_export_projects(self):
"""Export all projects, stacks and stack groups (without class instance
and tracing data). Make then sure, they match the fixture.
"""
p1_config = {
'project': {
'title': 'test-no-stacks',
'stacks': list(),
}
}
p2_config = {
'project': {
'title': 'test-two-stacks',
'stacks': [{
'broken_sections': [],
'title': 'test-stack-1',
'dimension': '(7, 17, 23)',
'resolution': '(2,3,5)',
'downsample_factors': None,
'orientation': 0,
'translation': '(0,0,0)',
'metadata': '',
'comment': 'Test comment',
'attribution': 'Test attribution',
'description': 'Simple test data',
'canary_location': '(0, 0, 0)',
'placeholder_color': '(0,0,0,1)',
'mirrors': [{
'title': 'test-mirror-1',
'url': 'https://catmaid-test/',
'tile_height': 512,
'tile_width': 256,
'fileextension': 'jpg',
'tile_source_type': 5,
'position': 2
}]
}, {
'broken_sections': [],
'comment': None,
'title': 'test-stack-2',
'dimension': '(7, 17, 23)',
'metadata': '',
'resolution': '(2,3,5)',
'downsample_factors': None,
'orientation': 0,
'translation': '(0,0,0)',
'attribution': None,
'description': '',
'canary_location': '(0, 0, 0)',
'placeholder_color': '(0.5,0.4,0.3,1)',
'mirrors': [{
'title': 'test-mirror-2',
'position': 0,
'url': 'https://this.is.my.stack/',
'tile_height': 400,
'tile_width': 300,
'fileextension': 'jpg',
'tile_source_type': 5,
}]
}, {
'broken_sections': [],
'comment': None,
'title': 'test-stack-3',
'dimension': '(4, 34, 9)',
'metadata': 'Test meta data',
'resolution': '(1,2,3)',
'downsample_factors': None,
'orientation': 0,
'translation': '(0,0,0)',
'attribution': None,
'description': '',
'canary_location': '(1, 2, 3)',
'placeholder_color': '(0,0,0.3,0.1)',
'mirrors': [{
'title': 'test-mirror-3',
'position': 0,
'url': 'https://catmaid-test/abc/',
'tile_height': 456,
'tile_width': 123,
'fileextension': 'jpg',
'tile_source_type': 2,
}],
'stackgroups': [{
'relation': 'view',
'title': u'Test group 1'
}],
}]
}
}
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
config = [p1_config, p2_config]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 1
cls_graph_ids_to_link = []
remove_unref_stack_data = False
# Make sure there are no existing projects or stacks
Project.objects.all().delete()
Stack.objects.all().delete()
imported, not_imported = importer.import_projects(self.user,
pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertEqual(0, len(not_imported))
self.assertEqual(len(config), len(imported))
# Make sure we can see all projects
for p in Project.objects.all():
assign_perm('can_browse', self.user, p)
def strip_ids(d):
""" Recursively, strip all 'id' fields of dictionaries.
"""
if type(d) == dict:
if 'id' in d:
d.pop('id')
for _,v in d.items():
strip_ids(v)
if type(d) == list:
for v in d:
strip_ids(v)
def test_result(result):
# Results come with IDs, which we don't have in our input data. Strip
# them to be able to simply compare dictionaries.
strip_ids(result)
for cp, p in zip(config, result):
# Convert potential stack tuples into lists (YAML parsing
# results in tuples).
if 'project' in p:
if 'stacks' in p['project']:
if type(p['project']['stacks']) == tuple:
p['project']['stacks'] = list(p['project']['stacks'])
self.assertDictEqual(cp, p)
self.fake_authentication()
def parse_list(d):
for k in d:
if type(d[k]) == tuple:
d[k] = list(d[k])
return d
# Export imported YAML data
response = self.client.get('/projects/export')
self.assertEqual(response.status_code, 200)
result_yaml = yaml.load(response.content.decode('utf-8'))
test_result(result_yaml)
# Export imported JSON data
response = self.client.get('/projects/export', HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
result_json = json.loads(response.content.decode('utf-8'),
object_hook=parse_list)
test_result(result_json)
