def test_import_projects(self): """Import a set of new projects, stacks and stack groups. This tests only the actual import. Retrieving the data to import from different sources is not part of this test. """ project_url = 'https://catmaid-test/' data_folder = '/tmp/catmaid-test/' existing_projects = list(Project.objects.all()) existing_project_ids = [p.id for p in existing_projects] p1_config = { 'project': { 'title': 'test-no-stacks', } } p2_config = { 'project': { 'title': 'test-two-stacks', 'stacks': [ { # A basic stack, only with required information 'title': 'test-stack-1', 'dimension': '(7, 17, 23)', 'resolution': '(2, 3, 5)', 'zoomlevels': -1, 'mirrors': [{ 'title': 'test-mirror-1', 'fileextension': 'jpg' }] }, { # A basic stack with a little more information 'title': 'test-stack-2', 'dimension': '(7, 17, 23)', 'resolution': '(2, 3, 5)', 'zoomlevels': -1, 'mirrors': [{ 'title': 'test-mirror-2', 'fileextension': 'jpg', 'url': 'https://this.is.my.stack/' }] }, { # A stack with all optional properties 'title': 'test-stack-3', 'dimension': '(4, 34, 9)', 'resolution': '(1, 2, 3)', 'metadata': 'Test meta data', 'zoomlevels': -1, 'translation': '(10, 20, 30)', 'mirrors': [{ 'title': 'test-mirror-3', 'folder': 'abc/', 'fileextension': 'jpg', 'tile_width': 123, 'tile_height': 456, 'tile_source_type': 2, }], 'stackgroups': [{ # Add a single stack group with only this stack # in it. 'title': 'Test group 1', 'relation': 'view', }], } ] } } pre_projects = [ importer.PreProject(p1_config, project_url, data_folder), importer.PreProject(p2_config, project_url, data_folder), ] tags = [] permissions = [] default_tile_width = 256 default_tile_height = 512 default_tile_source_type = 5 default_position = 0 cls_graph_ids_to_link = [] remove_unref_stack_data = False imported, not_imported = importer.import_projects( self.user, pre_projects, tags, permissions, default_tile_width, default_tile_height, default_tile_source_type, cls_graph_ids_to_link, remove_unref_stack_data) self.assertListEqual(pre_projects, imported) self.assertListEqual([], not_imported) new_projects = list( Project.objects.exclude( id__in=existing_project_ids).order_by('title')) self.assertEqual(2, len(new_projects)) # Projects should be ordered by name, so the first project will be based # on p1_config. Test p1 first, it is not expected to have any stacks. p1 = new_projects[0] self.assertEqual(p1_config['project']['title'], p1.title) self.assertEqual(0, p1.stacks.all().count()) # Test p2. p2 = new_projects[1] self.assertEqual(p2_config['project']['title'], p2.title) p2_stacks = p2.stacks.all().order_by('title') self.assertEqual(3, len(p2_stacks)) p2cfg_stacks = p2_config['project']['stacks'] for n, p2s in enumerate(p2_stacks): stack = p2cfg_stacks[n] # Test required fields self.assertEqual(stack['title'], p2s.title) six.assertCountEqual(self, literal_eval(stack['dimension']), literal_eval(str(p2s.dimension))) six.assertCountEqual(self, literal_eval(stack['resolution']), literal_eval(str(p2s.resolution))) self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels) # Test mirrors mirrors = p2s.stackmirror_set.all().order_by('title') self.assertEqual(len(stack['mirrors']), len(mirrors)) for m, omirror in enumerate(mirrors): mirror = stack['mirrors'][m] self.assertEqual(mirror['title'], omirror.title) self.assertEqual(mirror['fileextension'], omirror.file_extension) # Test fields with potential default values self.assertEqual(mirror.get('position', default_position), omirror.position) self.assertEqual(mirror.get('tile_width', default_tile_width), omirror.tile_width) self.assertEqual( mirror.get('tile_height', default_tile_height), omirror.tile_height) self.assertEqual( mirror.get('tile_source_type', default_tile_source_type), omirror.tile_source_type) if 'url' in mirror: image_base = mirror['url'] else: image_base = urljoin( project_url, urljoin(mirror.get('path', ''), mirror.get('folder', ''))) self.assertEqual(image_base, omirror.image_base) # Test project-stack link ps = ProjectStack.objects.get(project=p2.id, stack=p2s) six.assertCountEqual( self, literal_eval(stack.get('translation', '(0,0,0)')), literal_eval(str(ps.translation))) # Test stack groups ostack_group_links = StackStackGroup.objects.filter( stack=p2s).order_by('stack__title') stack_groups = stack.get('stackgroups', []) self.assertEqual(len(ostack_group_links), len(stack_groups)) for m, sg_cfg in enumerate(stack_groups): ostack_group_link = ostack_group_links[m] ostack_group = ostack_group_link.stack_group self.assertEqual(sg_cfg['title'], ostack_group.title) self.assertEqual(sg_cfg['relation'], ostack_group_link.group_relation.name) self.assertEqual(sg_cfg.get('position', default_position), ostack_group_link.position)
def test_import_export_projects(self): """Export all projects, stacks and stack groups (without class instance and tracing data). Make then sure, they match the fixture. """ p1_config = { 'project': { 'title': 'test-no-stacks', 'stacks': tuple(), } } p2_config = { 'project': { 'title': 'test-two-stacks', 'stacks': [{ 'broken_sections': [], 'title': 'test-stack-1', 'dimension': '(7,17,23)', 'resolution': '(2,3,5)', 'zoomlevels': -1, 'orientation': 0, 'translation': '(0,0,0)', 'metadata': '', 'comment': 'Test comment', 'attribution': 'Test attribution', 'description': 'Simple test data', 'canary_location': '(0,0,0)', 'placeholder_color': '(0,0,0,1)', 'mirrors': [{ 'title': 'test-mirror-1', 'url': 'https://catmaid-test/', 'tile_height': 512, 'tile_width': 256, 'fileextension': 'jpg', 'tile_source_type': 5, 'position': 2 }] }, { 'broken_sections': [], 'comment': None, 'title': 'test-stack-2', 'dimension': '(7,17,23)', 'metadata': '', 'resolution': '(2,3,5)', 'zoomlevels': -1, 'orientation': 0, 'translation': '(0,0,0)', 'attribution': None, 'description': '', 'canary_location': '(0,0,0)', 'placeholder_color': '(0.5,0.4,0.3,1)', 'mirrors': [{ 'title': 'test-mirror-2', 'position': 0, 'url': 'https://this.is.my.stack/', 'tile_height': 400, 'tile_width': 300, 'fileextension': 'jpg', 'tile_source_type': 5, }] }, { 'broken_sections': [], 'comment': None, 'title': 'test-stack-3', 'dimension': '(4,34,9)', 'metadata': 'Test meta data', 'resolution': '(1,2,3)', 'zoomlevels': -1, 'orientation': 0, 'translation': '(0,0,0)', 'attribution': None, 'description': '', 'canary_location': '(1,2,3)', 'placeholder_color': '(0,0,0.3,0.1)', 'mirrors': [{ 'title': 'test-mirror-3', 'position': 0, 'url': 'https://catmaid-test/abc/', 'tile_height': 456, 'tile_width': 123, 'fileextension': 'jpg', 'tile_source_type': 2, }], 'stackgroups': [{ 'relation': 'view', 'title': u'Test group 1' }], }] } } project_url = 'https://catmaid-test/' data_folder = '/tmp/catmaid-test/' pre_projects = [ importer.PreProject(p1_config, project_url, data_folder), importer.PreProject(p2_config, project_url, data_folder), ] config = [p1_config, p2_config] tags = [] permissions = [] default_tile_width = 256 default_tile_height = 512 default_tile_source_type = 1 cls_graph_ids_to_link = [] remove_unref_stack_data = False # Make sure there are no existing projects or stacks Project.objects.all().delete() Stack.objects.all().delete() imported, not_imported = importer.import_projects( self.user, pre_projects, tags, permissions, default_tile_width, default_tile_height, default_tile_source_type, cls_graph_ids_to_link, remove_unref_stack_data) self.assertEqual(0, len(not_imported)) self.assertEqual(len(config), len(imported)) # Make sure we can see all projects for p in Project.objects.all(): assign_perm('can_browse', self.user, p) # Export imported data self.fake_authentication() response = self.client.get('/projects/export') self.assertEqual(response.status_code, 200) result = yaml.load(response.content.decode('utf-8')) def strip_ids(d): """ Recursively, strip all 'id' fields of dictionaries. """ if type(d) == dict: if 'id' in d: d.pop('id') for _, v in d.items(): strip_ids(v) if type(d) == list: for v in d: strip_ids(v) # Results come with IDs, which we don't have in our input data. Strip # them to be able to simply compare dictionaries. strip_ids(result) for cp, p in zip(config, result): self.assertDictEqual(cp, p)
def test_import_projects(self): """Import a set of new projects, stacks and stack groups. This tests only the actual import. Retrieving the data to import from different sources is not part of this test. """ project_url = 'https://catmaid-test/' data_folder = '/tmp/catmaid-test/' existing_projects = list(Project.objects.all()) existing_project_ids = [p.id for p in existing_projects] p1_config = { 'project': { 'title': 'test-no-stacks', } } p2_config = { 'project': { 'title': 'test-two-stacks', 'stacks': [ { # A basic stack, only with required information 'title': 'test-stack-1', 'dimension': '(7, 17, 23)', 'resolution': '(2, 3, 5)', 'zoomlevels': -1, 'mirrors': [{ 'title': 'test-mirror-1', 'fileextension': 'jpg' }] }, { # A basic stack with a little more information 'title': 'test-stack-2', 'dimension': '(7, 17, 23)', 'resolution': '(2, 3, 5)', 'zoomlevels': -1, 'mirrors': [{ 'title': 'test-mirror-2', 'fileextension': 'jpg', 'url': 'https://this.is.my.stack/' }] }, { # A stack with all optional properties 'title': 'test-stack-3', 'dimension': '(4, 34, 9)', 'resolution': '(1, 2, 3)', 'metadata': 'Test meta data', 'zoomlevels': -1, 'translation': '(10, 20, 30)', 'mirrors': [{ 'title': 'test-mirror-3', 'folder': 'abc/', 'fileextension': 'jpg', 'tile_width': 123, 'tile_height': 456, 'tile_source_type': 2, }], 'stackgroups': [{ # Add a single stack group with only this stack # in it. 'title': 'Test group 1', 'relation': 'view', }], } ] } } pre_projects = [ importer.PreProject(p1_config, project_url, data_folder), importer.PreProject(p2_config, project_url, data_folder), ] tags = [] permissions = [] default_tile_width = 256 default_tile_height = 512 default_tile_source_type = 5 default_position = 0 cls_graph_ids_to_link = [] remove_unref_stack_data = False imported, not_imported = importer.import_projects(self.user, pre_projects, tags, permissions, default_tile_width, default_tile_height, default_tile_source_type, cls_graph_ids_to_link, remove_unref_stack_data) self.assertListEqual(pre_projects, imported) self.assertListEqual([], not_imported) new_projects = list(Project.objects.exclude(id__in=existing_project_ids).order_by('title')) self.assertEqual(2, len(new_projects)) # Projects should be ordered by name, so the first project will be based # on p1_config. Test p1 first, it is not expected to have any stacks. p1 = new_projects[0] self.assertEqual(p1_config['project']['title'], p1.title) self.assertEqual(0, p1.stacks.all().count()) # Test p2. p2 = new_projects[1] self.assertEqual(p2_config['project']['title'], p2.title) p2_stacks = p2.stacks.all().order_by('title') self.assertEqual(3, len(p2_stacks)) p2cfg_stacks = p2_config['project']['stacks'] for n, p2s in enumerate(p2_stacks): stack = p2cfg_stacks[n] # Test required fields self.assertEqual(stack['title'], p2s.title) six.assertCountEqual(self, literal_eval(stack['dimension']), literal_eval(str(p2s.dimension))) six.assertCountEqual(self, literal_eval(stack['resolution']), literal_eval(str(p2s.resolution))) self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels) # Test mirrors mirrors = p2s.stackmirror_set.all().order_by('title') self.assertEqual(len(stack['mirrors']), len(mirrors)) for m, omirror in enumerate(mirrors): mirror = stack['mirrors'][m] self.assertEqual(mirror['title'], omirror.title) self.assertEqual(mirror['fileextension'], omirror.file_extension) # Test fields with potential default values self.assertEqual(mirror.get('position', default_position), omirror.position) self.assertEqual(mirror.get('tile_width', default_tile_width), omirror.tile_width) self.assertEqual(mirror.get('tile_height', default_tile_height), omirror.tile_height) self.assertEqual(mirror.get('tile_source_type', default_tile_source_type), omirror.tile_source_type) if 'url' in mirror: image_base = mirror['url'] else: image_base = urljoin(project_url, urljoin(mirror.get('path', ''), mirror.get('folder', ''))) self.assertEqual(image_base, omirror.image_base) # Test project-stack link ps = ProjectStack.objects.get(project=p2.id, stack=p2s) six.assertCountEqual(self, literal_eval(stack.get('translation', '(0,0,0)')), literal_eval(str(ps.translation))) # Test stack groups ostack_group_links = StackStackGroup.objects.filter(stack=p2s).order_by('stack__title') stack_groups = stack.get('stackgroups', []) self.assertEqual(len(ostack_group_links), len(stack_groups)) for m, sg_cfg in enumerate(stack_groups): ostack_group_link = ostack_group_links[m] ostack_group = ostack_group_link.stack_group self.assertEqual(sg_cfg['title'], ostack_group.title) self.assertEqual(sg_cfg['relation'], ostack_group_link.group_relation.name) self.assertEqual(sg_cfg.get('position', default_position), ostack_group_link.position)
def handle(self, *args, **options): ignore_same_name_projects = options['ignore_same_name_projects'] ignore_same_name_stacks = options['ignore_same_name_stacks'] ignore_empty_projects = options['ignore_empty_projects'] project_id = options['project_id'] default_tile_width = options['default_tile_width'] default_tile_height = options['default_tile_height'] default_tile_source_type = options['default_tile_source_type'] remove_unref_stack_data = options['remove_unref_stack_data'] image_base = options['image_base'] if ignore_same_name_projects: logger.info("Ignoring all loaded projects that have same name as " "existing projects") if ignore_same_name_stacks: logger.info("Ignoring all loaded stacks that have same name as " "existing stacks") # Parse permissions permissions:List = [] for p in map(lambda x: x.split(':'), options['permissions']): if len(p) != 3: raise CommandError('Invalid permission format, expected: type:name:permission') p_type, obj_name, p_name = p[0].lower(), p[1], p[2] if p_type == 'user': target = User.objects.get(username=obj_name) elif p_type == 'group': target = Group.objects.get(groupname=obj_name) else: raise CommandError(f'Unknown permission target type: {p_type}') logger.info(f'Setting {p_name} permissions for {p_type} {obj_name}') permissions.append((target, p_name)) # This will read from either stdin or a provided text file if options['input'].isatty(): raise CommandError('Please provide either the --input argument ' 'with a file path or provide data on stdin.') input_data = options['input'].read() options['input'].close() project_configs = json.loads(input_data) pre_projects:List = [] for project_config in project_configs: title = project_config['project']['title'] if ignore_same_name_projects and \ Project.objects.filter(title=title).count() > 0: logger.info(f"Skipping project {title}, a project with the same name exists alrady") continue logger.info(f"Parsing project {title}") pre_project = PreProject(project_config, image_base, None) stacks_to_remove = [] for pre_stack in pre_project.stacks: if Stack.objects.filter(title=pre_stack.title).count() > 0: stacks_to_remove.append(pre_stack) if stacks_to_remove: stack_titles = ', '.join(map(lambda x: x.title, stacks_to_remove)) logger.info(f"Skipping stacks {stack_titles} in project {title}, " "because stacks with these names exist alrady") for stack_to_remove in stacks_to_remove: pre_project.stacks.remove(stack_to_remove) if ignore_empty_projects and not pre_project.stacks: logger.info(f"Skipping project {title}, because it has no stacks") continue pre_projects.append(pre_project) tags:List = [] cls_graph_ids_to_link:List = [] user = get_system_user() logger.info(f'Importing {len(pre_projects)} projects') imported, not_imported = import_projects(user, pre_projects, tags, permissions, default_tile_width, default_tile_height, default_tile_source_type, cls_graph_ids_to_link, remove_unref_stack_data) logger.info(f'Imported {len(imported)} projects') if not_imported: logger.info("Encountered the following problems during import:\n" + '\n'.join(map(lambda x: f'{x[0]}: {x[1]}', not_imported)))
def test_import_export_projects(self): """Export all projects, stacks and stack groups (without class instance and tracing data). Make then sure, they match the fixture. """ p1_config = { 'project': { 'title': 'test-no-stacks', 'stacks': list(), } } p2_config = { 'project': { 'title': 'test-two-stacks', 'stacks': [{ 'broken_sections': [], 'title': 'test-stack-1', 'dimension': '(7, 17, 23)', 'resolution': '(2,3,5)', 'downsample_factors': None, 'orientation': 0, 'translation': '(0,0,0)', 'metadata': '', 'comment': 'Test comment', 'attribution': 'Test attribution', 'description': 'Simple test data', 'canary_location': '(0, 0, 0)', 'placeholder_color': '(0,0,0,1)', 'mirrors': [{ 'title': 'test-mirror-1', 'url': 'https://catmaid-test/', 'tile_height': 512, 'tile_width': 256, 'fileextension': 'jpg', 'tile_source_type': 5, 'position': 2 }] }, { 'broken_sections': [], 'comment': None, 'title': 'test-stack-2', 'dimension': '(7, 17, 23)', 'metadata': '', 'resolution': '(2,3,5)', 'downsample_factors': None, 'orientation': 0, 'translation': '(0,0,0)', 'attribution': None, 'description': '', 'canary_location': '(0, 0, 0)', 'placeholder_color': '(0.5,0.4,0.3,1)', 'mirrors': [{ 'title': 'test-mirror-2', 'position': 0, 'url': 'https://this.is.my.stack/', 'tile_height': 400, 'tile_width': 300, 'fileextension': 'jpg', 'tile_source_type': 5, }] }, { 'broken_sections': [], 'comment': None, 'title': 'test-stack-3', 'dimension': '(4, 34, 9)', 'metadata': 'Test meta data', 'resolution': '(1,2,3)', 'downsample_factors': None, 'orientation': 0, 'translation': '(0,0,0)', 'attribution': None, 'description': '', 'canary_location': '(1, 2, 3)', 'placeholder_color': '(0,0,0.3,0.1)', 'mirrors': [{ 'title': 'test-mirror-3', 'position': 0, 'url': 'https://catmaid-test/abc/', 'tile_height': 456, 'tile_width': 123, 'fileextension': 'jpg', 'tile_source_type': 2, }], 'stackgroups': [{ 'relation': 'view', 'title': u'Test group 1' }], }] } } project_url = 'https://catmaid-test/' data_folder = '/tmp/catmaid-test/' pre_projects = [ importer.PreProject(p1_config, project_url, data_folder), importer.PreProject(p2_config, project_url, data_folder), ] config = [p1_config, p2_config] tags = [] permissions = [] default_tile_width = 256 default_tile_height = 512 default_tile_source_type = 1 cls_graph_ids_to_link = [] remove_unref_stack_data = False # Make sure there are no existing projects or stacks Project.objects.all().delete() Stack.objects.all().delete() imported, not_imported = importer.import_projects(self.user, pre_projects, tags, permissions, default_tile_width, default_tile_height, default_tile_source_type, cls_graph_ids_to_link, remove_unref_stack_data) self.assertEqual(0, len(not_imported)) self.assertEqual(len(config), len(imported)) # Make sure we can see all projects for p in Project.objects.all(): assign_perm('can_browse', self.user, p) def strip_ids(d): """ Recursively, strip all 'id' fields of dictionaries. """ if type(d) == dict: if 'id' in d: d.pop('id') for _,v in d.items(): strip_ids(v) if type(d) == list: for v in d: strip_ids(v) def test_result(result): # Results come with IDs, which we don't have in our input data. Strip # them to be able to simply compare dictionaries. strip_ids(result) for cp, p in zip(config, result): # Convert potential stack tuples into lists (YAML parsing # results in tuples). if 'project' in p: if 'stacks' in p['project']: if type(p['project']['stacks']) == tuple: p['project']['stacks'] = list(p['project']['stacks']) self.assertDictEqual(cp, p) self.fake_authentication() def parse_list(d): for k in d: if type(d[k]) == tuple: d[k] = list(d[k]) return d # Export imported YAML data response = self.client.get('/projects/export') self.assertEqual(response.status_code, 200) result_yaml = yaml.load(response.content.decode('utf-8')) test_result(result_yaml) # Export imported JSON data response = self.client.get('/projects/export', HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, 200) result_json = json.loads(response.content.decode('utf-8'), object_hook=parse_list) test_result(result_json)