def check_http_accessibility( image_base, file_extension ):
""" Returns true if data below this image base can be accessed through HTTP.
"""
slice_zero_url = urljoin(image_base, "0")
first_file_url = urljoin(slice_zero_url, "0_0_0." + file_extension)
code = urllib.urlopen(first_file_url).getcode()
return code == 200
def export_skeleton_as_nrrd_async(skeleton_id,
source_ref,
target_ref,
user_id,
mirror=True,
create_message=True):
result = export_skeleton_as_nrrd(skeleton_id, source_ref, target_ref,
user_id, mirror)
if create_message:
msg = Message()
msg.user = User.objects.get(pk=int(user_id))
msg.read = False
if result['errors']:
msg.title = "No NRRD file could be creaed for skeleton {}".format(
skeleton_id)
msg.text = "There was at least one error during the NRRD export: {}".format(
'\n'.join(result['errors']))
msg.action = ""
else:
url = urljoin(
urljoin(settings.MEDIA_URL,
settings.MEDIA_EXPORT_SUBDIRECTORY),
result['nrrd_name'])
msg.title = "Exported skeleton {} as NRRD file".format(skeleton_id)
msg.text = "The requested skeleton was exported as NRRD file. You " \
"can download it from this location: <a href='{}'>{}</a>".format(url, url)
msg.action = url
msg.save()
return "Errors: {}".format('\n'.join(
result['errors'])) if result['errors'] else result['nrrd_path']
def check_http_accessibility(image_base, file_extension):
""" Returns true if data below this image base can be accessed through HTTP.
"""
slice_zero_url = urljoin(image_base, "0")
first_file_url = urljoin(slice_zero_url, "0_0_0." + file_extension)
try:
code = urllib.urlopen(first_file_url).getcode()
except IOError:
return False
return code == 200
def find_project_folders(image_base, path, filter_term, only_unknown, depth=1):
""" Finds projects in a folder structure by testing for the presence of an
info/project YAML file.
"""
dirs = []
projects = {}
not_readable = []
for current_file in glob.glob( os.path.join(path, filter_term) ):
if os.path.isdir(current_file):
# Check if the folder has a info file
info_file = os.path.join(current_file, info_file_name)
if os.path.exists(info_file):
short_name = current_file.replace(settings.CATMAID_IMPORT_PATH, "")
# Check if this folder is already known by testing if
# there is a stack with the same image base as a subfolder
# would have.
try:
# Make sure we have slashes in our directery URL part
url_dir = short_name
if os.sep != '/':
url_dir = url_dir.replace("\\", "/")
project_url = urljoin(image_base, url_dir)
project = PreProject( info_file, project_url, short_name, only_unknown )
if only_unknown and project.already_known:
continue
else:
projects[current_file] = project
# Remember this project if it isn't available yet
dirs.append( (current_file, short_name) )
except Exception as e:
not_readable.append( (info_file, e) )
elif depth > 1:
# Recurse in subdir if requested
dirs = dirs + find_files( current_file, depth - 1)
return (dirs, projects, not_readable)
def get_roi_image(request, project_id=None, roi_id=None):
""" Returns the URL to the cropped image, described by the ROI. These
images are cached, and won't get removed automatically. If the image is
already present its URL is used and returned. For performance reasons it
might be a good idea, to add this test to the web-server config.
"""
file_name, file_path = create_roi_path(roi_id)
if not os.path.exists(file_path):
# Start async processing
create_roi_image(request.user, project_id, roi_id, file_path)
# Use waiting image
url = urljoin(settings.STATIC_URL, "images/wait_bgwhite.gif")
else:
# Create real image di
url_base = urljoin(settings.MEDIA_URL, settings.MEDIA_ROI_SUBDIRECTORY)
url = urljoin(url_base, file_name)
return redirect(url)
def get_roi_image(request, project_id=None, roi_id=None):
""" Returns the URL to the cropped image, described by the ROI. These
images are cached, and won't get removed automatically. If the image is
already present its URL is used and returned. For performance reasons it
might be a good idea, to add this test to the web-server config.
"""
file_name, file_path = create_roi_path(roi_id)
if not os.path.exists(file_path):
# Start async processing
create_roi_image(request.user, project_id, roi_id, file_path)
# Use waiting image
url = urljoin(settings.STATIC_URL,
"images/wait_bgwhite.gif")
else:
# Create real image di
url_base = urljoin(settings.MEDIA_URL,
settings.MEDIA_ROI_SUBDIRECTORY)
url = urljoin(url_base, file_name)
return redirect(url)
def export_skeleton_as_nrrd_async(skeleton_id, source_ref, target_ref, user_id,
mirror=True, create_message=True):
result = export_skeleton_as_nrrd(skeleton_id, source_ref, target_ref,
user_id, mirror)
if create_message:
msg = Message()
msg.user = User.objects.get(pk=int(user_id))
msg.read = False
if result['errors']:
msg.title = "No NRRD file could be creaed for skeleton {}".format(skeleton_id)
msg.text = "There was at least one error during the NRRD export: {}".format('\n'.join(result['errors']))
msg.action = ""
else:
url = urljoin(urljoin(settings.MEDIA_URL, settings.MEDIA_EXPORT_SUBDIRECTORY), result['nrrd_name'])
msg.title = "Exported skeleton {} as NRRD file".format(skeleton_id)
msg.text = "The requested skeleton was exported as NRRD file. You " \
"can download it from this location: <a href='{}'>{}</a>".format(url, url)
msg.action = url
msg.save()
return "Errors: {}".format('\n'.join(result['errors'])) if result['errors'] else result['nrrd_path']
def set_image_fields(self, info_object, project_url, data_folder, needs_zoom):
""" Sets the image_base, num_zoom_levels, file_extension fields
of the calling object. Favor a URL field, if there is one. A URL
field, however, also requires the existence of the
'fileextension' and and 'zoomlevels' field.
"""
if 'url' in info_object:
self.image_base = info_object['url']
if needs_zoom:
self.num_zoom_levels = info_object['zoomlevels']
self.file_extension = info_object['fileextension']
else:
# The image base of a stack is the combination of the
# project URL and the stack's folder name.
folder = info_object['folder']
self.image_base = urljoin(project_url, folder)
# Favor 'zoomlevel' and 'fileextension' fields, if
# available, but try to find this information if thosa
# fields are not present.
zoom_available = 'zoomlevels' in info_object
ext_available = 'fileextension' in info_object
if zoom_available and needs_zoom:
self.num_zoom_levels = info_object['zoomlevels']
if ext_available:
self.file_extension = info_object['fileextension']
# Only retrieve file extension and zoom level if one of
# them is not available in the stack definition.
if (not zoom_available and needs_zoom) or not ext_available:
file_ext, zoom_levels = find_zoom_levels_and_file_ext(
data_folder, folder, needs_zoom )
# If there is no zoom level provided, use the found one
if not zoom_available and needs_zoom:
self.num_zoom_levels = zoom_levels
# If there is no file extension level provided, use the
# found one
if not ext_available:
self.file_extension = file_ext
# Make sure the image base has a trailing slash, because this is expected
if self.image_base[-1] != '/':
self.image_base = self.image_base + '/'
# Test if the data is accessible through HTTP
self.accessible = check_http_accessibility( self.image_base, self.file_extension )
def find_project_folders(image_base, path, filter_term, only_unknown, depth=1):
""" Finds projects in a folder structure by testing for the presence of an
info/project YAML file.
"""
dirs = []
projects = {}
not_readable = []
for current_file in glob.glob(os.path.join(path, filter_term)):
if os.path.isdir(current_file):
# Check if the folder has a info file
info_file = os.path.join(current_file, info_file_name)
if os.path.exists(info_file):
short_name = current_file.replace(settings.CATMAID_IMPORT_PATH,
"")
# Check if this folder is already known by testing if
# there is a stack with the same image base as a subfolder
# would have.
try:
# Make sure we have slashes in our directery URL part
url_dir = short_name
if os.sep != '/':
url_dir = url_dir.replace("\\", "/")
project_url = urljoin(image_base, url_dir)
project = PreProject(info_file, project_url, short_name,
only_unknown)
if only_unknown and project.already_known:
continue
else:
projects[current_file] = project
# Remember this project if it isn't available yet
dirs.append((current_file, short_name))
except Exception as e:
not_readable.append((info_file, e))
elif depth > 1:
# Recurse in subdir if requested
dirs = dirs + find_files(current_file, depth - 1)
return (dirs, projects, not_readable)
def test_import_projects(self):
"""Import a set of new projects, stacks and stack groups. This tests
only the actual import. Retrieving the data to import from different
sources is not part of this test.
"""
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
existing_projects = list(Project.objects.all())
existing_project_ids = [p.id for p in existing_projects]
p1_config = {
'project': {
'title': 'test-no-stacks',
}
}
p2_config = {
'project': {
'title':
'test-two-stacks',
'stacks': [
{
# A basic stack, only with required information
'title':
'test-stack-1',
'dimension':
'(7, 17, 23)',
'resolution':
'(2, 3, 5)',
'zoomlevels':
-1,
'mirrors': [{
'title': 'test-mirror-1',
'fileextension': 'jpg'
}]
},
{
# A basic stack with a little more information
'title':
'test-stack-2',
'dimension':
'(7, 17, 23)',
'resolution':
'(2, 3, 5)',
'zoomlevels':
-1,
'mirrors': [{
'title': 'test-mirror-2',
'fileextension': 'jpg',
'url': 'https://this.is.my.stack/'
}]
},
{
# A stack with all optional properties
'title':
'test-stack-3',
'dimension':
'(4, 34, 9)',
'resolution':
'(1, 2, 3)',
'metadata':
'Test meta data',
'zoomlevels':
-1,
'translation':
'(10, 20, 30)',
'mirrors': [{
'title': 'test-mirror-3',
'folder': 'abc/',
'fileextension': 'jpg',
'tile_width': 123,
'tile_height': 456,
'tile_source_type': 2,
}],
'stackgroups': [{
# Add a single stack group with only this stack
# in it.
'title': 'Test group 1',
'relation': 'view',
}],
}
]
}
}
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 5
default_position = 0
cls_graph_ids_to_link = []
remove_unref_stack_data = False
imported, not_imported = importer.import_projects(
self.user, pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertListEqual(pre_projects, imported)
self.assertListEqual([], not_imported)
new_projects = list(
Project.objects.exclude(
id__in=existing_project_ids).order_by('title'))
self.assertEqual(2, len(new_projects))
# Projects should be ordered by name, so the first project will be based
# on p1_config. Test p1 first, it is not expected to have any stacks.
p1 = new_projects[0]
self.assertEqual(p1_config['project']['title'], p1.title)
self.assertEqual(0, p1.stacks.all().count())
# Test p2.
p2 = new_projects[1]
self.assertEqual(p2_config['project']['title'], p2.title)
p2_stacks = p2.stacks.all().order_by('title')
self.assertEqual(3, len(p2_stacks))
p2cfg_stacks = p2_config['project']['stacks']
for n, p2s in enumerate(p2_stacks):
stack = p2cfg_stacks[n]
# Test required fields
self.assertEqual(stack['title'], p2s.title)
six.assertCountEqual(self, literal_eval(stack['dimension']),
literal_eval(str(p2s.dimension)))
six.assertCountEqual(self, literal_eval(stack['resolution']),
literal_eval(str(p2s.resolution)))
self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels)
# Test mirrors
mirrors = p2s.stackmirror_set.all().order_by('title')
self.assertEqual(len(stack['mirrors']), len(mirrors))
for m, omirror in enumerate(mirrors):
mirror = stack['mirrors'][m]
self.assertEqual(mirror['title'], omirror.title)
self.assertEqual(mirror['fileextension'],
omirror.file_extension)
# Test fields with potential default values
self.assertEqual(mirror.get('position', default_position),
omirror.position)
self.assertEqual(mirror.get('tile_width', default_tile_width),
omirror.tile_width)
self.assertEqual(
mirror.get('tile_height', default_tile_height),
omirror.tile_height)
self.assertEqual(
mirror.get('tile_source_type', default_tile_source_type),
omirror.tile_source_type)
if 'url' in mirror:
image_base = mirror['url']
else:
image_base = urljoin(
project_url,
urljoin(mirror.get('path', ''),
mirror.get('folder', '')))
self.assertEqual(image_base, omirror.image_base)
# Test project-stack link
ps = ProjectStack.objects.get(project=p2.id, stack=p2s)
six.assertCountEqual(
self, literal_eval(stack.get('translation', '(0,0,0)')),
literal_eval(str(ps.translation)))
# Test stack groups
ostack_group_links = StackStackGroup.objects.filter(
stack=p2s).order_by('stack__title')
stack_groups = stack.get('stackgroups', [])
self.assertEqual(len(ostack_group_links), len(stack_groups))
for m, sg_cfg in enumerate(stack_groups):
ostack_group_link = ostack_group_links[m]
ostack_group = ostack_group_link.stack_group
self.assertEqual(sg_cfg['title'], ostack_group.title)
self.assertEqual(sg_cfg['relation'],
ostack_group_link.group_relation.name)
self.assertEqual(sg_cfg.get('position', default_position),
ostack_group_link.position)
def set_image_fields(self, info_object, project_url, data_folder,
needs_zoom):
""" Sets the image_base, num_zoom_levels, file_extension fields
of the calling object. Favor a URL field, if there is one. A URL
field, however, also requires the existence of the
'fileextension' and 'zoomlevels' field.
"""
if 'url' in info_object:
# Make sure all required data is available
required_fields = [
'fileextension',
]
if needs_zoom:
required_fields.append('zoomlevels')
for f in required_fields:
if f not in info_object:
raise RuntimeError("Missing required stack/overlay " \
"field '%s'" % f)
# Read out data
self.image_base = info_object['url']
if needs_zoom:
self.num_zoom_levels = info_object['zoomlevels']
self.file_extension = info_object['fileextension']
else:
# The image base of a stack is the combination of the
# project URL and the stack's folder name.
folder = info_object['folder']
self.image_base = urljoin(project_url, folder)
# Favor 'zoomlevel' and 'fileextension' fields, if
# available, but try to find this information if those
# fields are not present.
zoom_available = 'zoomlevels' in info_object
ext_available = 'fileextension' in info_object
if zoom_available and needs_zoom:
self.num_zoom_levels = info_object['zoomlevels']
if ext_available:
self.file_extension = info_object['fileextension']
# Only retrieve file extension and zoom level if one of
# them is not available in the stack definition.
if (not zoom_available and needs_zoom) or not ext_available:
file_ext, zoom_levels = find_zoom_levels_and_file_ext(
data_folder, folder, needs_zoom)
# If there is no zoom level provided, use the found one
if not zoom_available and needs_zoom:
if not zoom_levels:
raise RuntimeError("Missing required stack/overlay " \
"field 'zoomlevels' and couldn't retrieve " \
"this information from image data.")
self.num_zoom_levels = zoom_levels
# If there is no file extension level provided, use the
# found one
if not ext_available:
if not file_ext:
raise RuntimeError("Missing required stack/overlay " \
"field 'fileextension' and couldn't retrieve " \
"this information from image data.")
self.file_extension = file_ext
# Make sure the image base has a trailing slash, because this is expected
if self.image_base[-1] != '/':
self.image_base = self.image_base + '/'
# Test if the data is accessible through HTTP
self.accessible = check_http_accessibility(self.image_base,
self.file_extension)
def test_import_projects(self):
"""Import a set of new projects, stacks and stack groups. This tests
only the actual import. Retrieving the data to import from different
sources is not part of this test.
"""
project_url = 'https://catmaid-test/'
data_folder = '/tmp/catmaid-test/'
existing_projects = list(Project.objects.all())
existing_project_ids = [p.id for p in existing_projects]
p1_config = {
'project': {
'title': 'test-no-stacks',
}
}
p2_config = {
'project': {
'title': 'test-two-stacks',
'stacks': [
{
# A basic stack, only with required information
'title': 'test-stack-1',
'dimension': '(7, 17, 23)',
'resolution': '(2, 3, 5)',
'zoomlevels': -1,
'mirrors': [{
'title': 'test-mirror-1',
'fileextension': 'jpg'
}]
},
{
# A basic stack with a little more information
'title': 'test-stack-2',
'dimension': '(7, 17, 23)',
'resolution': '(2, 3, 5)',
'zoomlevels': -1,
'mirrors': [{
'title': 'test-mirror-2',
'fileextension': 'jpg',
'url': 'https://this.is.my.stack/'
}]
},
{
# A stack with all optional properties
'title': 'test-stack-3',
'dimension': '(4, 34, 9)',
'resolution': '(1, 2, 3)',
'metadata': 'Test meta data',
'zoomlevels': -1,
'translation': '(10, 20, 30)',
'mirrors': [{
'title': 'test-mirror-3',
'folder': 'abc/',
'fileextension': 'jpg',
'tile_width': 123,
'tile_height': 456,
'tile_source_type': 2,
}],
'stackgroups': [{
# Add a single stack group with only this stack
# in it.
'title': 'Test group 1',
'relation': 'view',
}],
}
]
}
}
pre_projects = [
importer.PreProject(p1_config, project_url, data_folder),
importer.PreProject(p2_config, project_url, data_folder),
]
tags = []
permissions = []
default_tile_width = 256
default_tile_height = 512
default_tile_source_type = 5
default_position = 0
cls_graph_ids_to_link = []
remove_unref_stack_data = False
imported, not_imported = importer.import_projects(self.user,
pre_projects, tags, permissions, default_tile_width,
default_tile_height, default_tile_source_type,
cls_graph_ids_to_link, remove_unref_stack_data)
self.assertListEqual(pre_projects, imported)
self.assertListEqual([], not_imported)
new_projects = list(Project.objects.exclude(id__in=existing_project_ids).order_by('title'))
self.assertEqual(2, len(new_projects))
# Projects should be ordered by name, so the first project will be based
# on p1_config. Test p1 first, it is not expected to have any stacks.
p1 = new_projects[0]
self.assertEqual(p1_config['project']['title'], p1.title)
self.assertEqual(0, p1.stacks.all().count())
# Test p2.
p2 = new_projects[1]
self.assertEqual(p2_config['project']['title'], p2.title)
p2_stacks = p2.stacks.all().order_by('title')
self.assertEqual(3, len(p2_stacks))
p2cfg_stacks = p2_config['project']['stacks']
for n, p2s in enumerate(p2_stacks):
stack = p2cfg_stacks[n]
# Test required fields
self.assertEqual(stack['title'], p2s.title)
six.assertCountEqual(self, literal_eval(stack['dimension']),
literal_eval(str(p2s.dimension)))
six.assertCountEqual(self, literal_eval(stack['resolution']),
literal_eval(str(p2s.resolution)))
self.assertEqual(stack['zoomlevels'], p2s.num_zoom_levels)
# Test mirrors
mirrors = p2s.stackmirror_set.all().order_by('title')
self.assertEqual(len(stack['mirrors']), len(mirrors))
for m, omirror in enumerate(mirrors):
mirror = stack['mirrors'][m]
self.assertEqual(mirror['title'], omirror.title)
self.assertEqual(mirror['fileextension'], omirror.file_extension)
# Test fields with potential default values
self.assertEqual(mirror.get('position', default_position),
omirror.position)
self.assertEqual(mirror.get('tile_width', default_tile_width),
omirror.tile_width)
self.assertEqual(mirror.get('tile_height', default_tile_height),
omirror.tile_height)
self.assertEqual(mirror.get('tile_source_type', default_tile_source_type),
omirror.tile_source_type)
if 'url' in mirror:
image_base = mirror['url']
else:
image_base = urljoin(project_url,
urljoin(mirror.get('path', ''), mirror.get('folder', '')))
self.assertEqual(image_base, omirror.image_base)
# Test project-stack link
ps = ProjectStack.objects.get(project=p2.id, stack=p2s)
six.assertCountEqual(self, literal_eval(stack.get('translation', '(0,0,0)')),
literal_eval(str(ps.translation)))
# Test stack groups
ostack_group_links = StackStackGroup.objects.filter(stack=p2s).order_by('stack__title')
stack_groups = stack.get('stackgroups', [])
self.assertEqual(len(ostack_group_links), len(stack_groups))
for m, sg_cfg in enumerate(stack_groups):
ostack_group_link = ostack_group_links[m]
ostack_group = ostack_group_link.stack_group
self.assertEqual(sg_cfg['title'], ostack_group.title)
self.assertEqual(sg_cfg['relation'],
ostack_group_link.group_relation.name)
self.assertEqual(sg_cfg.get('position', default_position),
ostack_group_link.position)
def set_image_fields(self, info_object, project_url, data_folder, needs_zoom):
""" Sets the image_base, num_zoom_levels, file_extension fields
of the calling object. Favor a URL field, if there is one. A URL
field, however, also requires the existence of the
'fileextension' and 'zoomlevels' field.
"""
if "url" in info_object:
# Make sure all required data is available
required_fields = ["fileextension"]
if needs_zoom:
required_fields.append("zoomlevels")
for f in required_fields:
if f not in info_object:
raise RuntimeError("Missing required stack/overlay " "field '%s'" % f)
# Read out data
self.image_base = info_object["url"]
if needs_zoom:
self.num_zoom_levels = info_object["zoomlevels"]
self.file_extension = info_object["fileextension"]
else:
# The image base of a stack is the combination of the
# project URL and the stack's folder name.
folder = info_object["folder"]
self.image_base = urljoin(project_url, folder)
# Favor 'zoomlevel' and 'fileextension' fields, if
# available, but try to find this information if those
# fields are not present.
zoom_available = "zoomlevels" in info_object
ext_available = "fileextension" in info_object
if zoom_available and needs_zoom:
self.num_zoom_levels = info_object["zoomlevels"]
if ext_available:
self.file_extension = info_object["fileextension"]
# Only retrieve file extension and zoom level if one of
# them is not available in the stack definition.
if (not zoom_available and needs_zoom) or not ext_available:
file_ext, zoom_levels = find_zoom_levels_and_file_ext(data_folder, folder, needs_zoom)
# If there is no zoom level provided, use the found one
if not zoom_available and needs_zoom:
if not zoom_levels:
raise RuntimeError(
"Missing required stack/overlay "
"field 'zoomlevels' and couldn't retrieve "
"this information from image data."
)
self.num_zoom_levels = zoom_levels
# If there is no file extension level provided, use the
# found one
if not ext_available:
if not file_ext:
raise RuntimeError(
"Missing required stack/overlay "
"field 'fileextension' and couldn't retrieve "
"this information from image data."
)
self.file_extension = file_ext
# Make sure the image base has a trailing slash, because this is expected
if self.image_base[-1] != "/":
self.image_base = self.image_base + "/"
# Test if the data is accessible through HTTP
self.accessible = check_http_accessibility(self.image_base, self.file_extension)