def __init__(self, filepath=None, jsonNamespace=None, cwd=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
"""
cwd = cwd or os.getcwd()
Location = DatasetInfo.Location
self.location = Location.FileSystem # Whether the data will be found/stored on the filesystem or in the project file
self._filePath = "" # The original path to the data (also used as a fallback if the data isn't in the project yet)
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
self.allowLabels = True # OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.drange = None
self.normalizeDisplay = True
self.fromstack = False
self.nickname = ""
self.axistags = None
self.subvolume_roi = None
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
if '*' in filepath:
file_list = glob.glob(filepath)
if not isUrl(filepath) and os.path.pathsep in filepath:
file_list = filepath.split(os.path.pathsep)
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = map(lambda f: make_absolute(f, cwd), file_list)
filepath = os.path.pathsep.join( file_list )
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len( os.path.splitext(file_list[-1])[1] )
nickname = PathComponents(prefix).filenameBase + ("_"*num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
if jsonNamespace is not None:
self.updateFromJson( jsonNamespace )
def post_process_lane_export(self, lane_index):
# FIXME: This probably only works for the non-blockwise export slot.
# We should assert that the user isn't using the blockwise slot.
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
from lazyflow.utility import PathComponents, make_absolute, format_known_keys
if settings:
self.dataExportTrackingApplet.progressSignal.emit(-1)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportTrackingApplet.progressSignal.emit(100)
def post_process_lane_export(self, lane_index):
# FIXME: This probably only works for the non-blockwise export slot.
# We should assert that the user isn't using the blockwise slot.
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
from lazyflow.utility import PathComponents, make_absolute, format_known_keys
if settings:
self.dataExportTrackingApplet.progressSignal.emit(-1)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportTrackingApplet.progressSignal.emit(100)
def getPartiallyFormattedName(self, lane_index, path_format_string):
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
known_keys['result_type'] = self.dataExportTrackingApplet.topLevelOperator.SelectedPlugin._value
# use partial formatting to fill in non-coordinate name fields
partially_formatted_name = format_known_keys(path_format_string, known_keys)
return partially_formatted_name
def getPartiallyFormattedName(self, lane_index, path_format_string):
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
known_keys['result_type'] = self.dataExportTrackingApplet.topLevelOperator.SelectedPlugin._value
# use partial formatting to fill in non-coordinate name fields
partially_formatted_name = format_known_keys(path_format_string, known_keys)
return partially_formatted_name
def post_process_lane_export(self, lane_index):
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(
lane_index).get_table_export_settings()
if settings:
self.dataExportApplet.progressSignal.emit(0)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[
lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(
raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(
os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys(
name_format, known_keys)
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(
lane_index
).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportApplet.progressSignal.emit(100)
# Restore state of axis ranges
parameters = self.trackingApplet.topLevelOperator.Parameters.value
parameters['time_range'] = self.prev_time_range
parameters['x_range'] = self.prev_x_range
parameters['y_range'] = self.prev_y_range
parameters['z_range'] = self.prev_z_range
def getPartiallyFormattedName(self, lane_index: int, path_format_string: str) -> str:
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.topLevelOperator.RawDatasetInfo[lane_index].value
project_path = self.topLevelOperator.WorkingDirectory.value
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_path)
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys = {
'dataset_dir': abs_dataset_dir,
'nickname': nickname,
'result_type': self.topLevelOperator.SelectedPlugin._value,
}
return format_known_keys(path_format_string, known_keys)
def post_process_lane_export(self, lane_index):
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
if settings:
self.dataExportApplet.progressSignal.emit(0)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportApplet.progressSignal.emit(100)
# Restore state of axis ranges
parameters = self.trackingApplet.topLevelOperator.Parameters.value
parameters['time_range'] = self.prev_time_range
parameters['x_range'] = self.prev_x_range
parameters['y_range'] = self.prev_y_range
parameters['z_range'] = self.prev_z_range
def __init__(self, filepath=None, jsonNamespace=None, cwd=None, preloaded_array=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged vigra.VigraArray.
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
self._filePath = "" # The original path to the data (also used as a fallback if the data isn't in the project yet)
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
self.allowLabels = True # OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.drange = None
self.normalizeDisplay = True
self.fromstack = False
self.nickname = ""
self.axistags = None
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.fromstack = False
self.nickname = "preloaded-{}-array".format( self.preloaded_array.dtype.name )
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
if '*' in filepath:
file_list = glob.glob(filepath)
file_list = sorted(file_list)
if not isUrl(filepath) and os.path.pathsep in filepath:
file_list = filepath.split(os.path.pathsep)
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = map(lambda f: make_absolute(f, cwd), file_list)
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join( file_list )
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len( os.path.splitext(file_list[-1])[1] )
nickname = PathComponents(prefix).filenameBase + ("_"*num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
if jsonNamespace is not None:
self.updateFromJson( jsonNamespace )
def post_process_lane_export(self, lane_index, checkOverwriteFiles=False):
# FIXME: This probably only works for the non-blockwise export slot.
# We should assert that the user isn't using the blockwise slot.
# Plugin export if selected
logger.info("Export source is: " + self.dataExportTrackingApplet.
topLevelOperator.SelectedExportSource.value)
print "in post_process_lane_export"
if self.dataExportTrackingApplet.topLevelOperator.SelectedExportSource.value == OpTrackingBaseDataExport.PluginOnlyName:
logger.info("Export source plugin selected!")
selectedPlugin = self.dataExportTrackingApplet.topLevelOperator.SelectedPlugin.value
exportPluginInfo = pluginManager.getPluginByName(
selectedPlugin, category="TrackingExportFormats")
if exportPluginInfo is None:
logger.error("Could not find selected plugin %s" %
exportPluginInfo)
else:
exportPlugin = exportPluginInfo.plugin_object
logger.info("Exporting tracking result using %s" %
selectedPlugin)
name_format = self.dataExportTrackingApplet.topLevelOperator.getLane(
lane_index).OutputFilenameFormat.value
partially_formatted_name = self.getPartiallyFormattedName(
lane_index, name_format)
if exportPlugin.exportsToFile:
filename = partially_formatted_name
if os.path.basename(filename) == '':
filename = os.path.join(filename, 'pluginExport.txt')
else:
filename = os.path.dirname(partially_formatted_name)
if filename is None or len(str(filename)) == 0:
logger.error(
"Cannot export from plugin with empty output filename")
return
exportStatus = self.trackingApplet.topLevelOperator.getLane(
lane_index).exportPlugin(filename, exportPlugin,
checkOverwriteFiles)
if not exportStatus:
return False
logger.info("Export done")
return
# CSV Table export (only if plugin was not selected)
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(
lane_index).get_table_export_settings()
from lazyflow.utility import PathComponents, make_absolute, format_known_keys
if settings:
self.dataExportTrackingApplet.progressSignal.emit(-1)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[
lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(
raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(
os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys(
name_format, known_keys)
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(
lane_index
).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportTrackingApplet.progressSignal.emit(100)
def __init__(self,
filepath=None,
jsonNamespace=None,
cwd=None,
preloaded_array=None,
sequence_axis=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged
vigra.VigraArray.
sequence_axis: Axis along which to stack (only applicable for stacks).
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
# The original path to the data (also used as a fallback if the data isn't in the project yet)
self._filePath = ""
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
# OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.allowLabels = True
self.drange = None
self.normalizeDisplay = True
self.sequenceAxis = None
self.fromstack = False
self.nickname = ""
self.axistags = None
self.original_axistags = None
# Necessary in headless mode in order to recover the shape of the raw data
self.laneShape = None
self.laneDtype = None
# A flag indicating whether the dataset is backed by a real source (e.g. file)
# or by the fake provided (e.g. in headless mode when raw data are not necessary)
self.realDataSource = True
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.nickname = "preloaded-{}-array".format(
self.preloaded_array.dtype.name)
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
# To support h5 sequences, filepath may contain external and
# internal path components
if not isUrl(filepath):
file_list = filepath.split(os.path.pathsep)
pathComponents = [PathComponents(x) for x in file_list]
externalPaths = [pc.externalPath for pc in pathComponents]
internalPaths = [pc.internalPath for pc in pathComponents]
if len(file_list) > 0:
if len(externalPaths) == 1:
if '*' in externalPaths[0]:
if internalPaths[0] is not None:
assert ('*' not in internalPaths[0]), (
"Only internal OR external glob placeholder supported"
)
file_list = sorted(glob.glob(filepath))
else:
file_list = [externalPaths[0]]
if internalPaths[0] is not None:
if '*' in internalPaths[0]:
# overwrite internalPaths, will be assembled further down
glob_string = "{}{}".format(
externalPaths[0], internalPaths[0])
internalPaths = \
OpStreamingH5N5SequenceReaderS.expandGlobStrings(
externalPaths[0], glob_string)
if internalPaths:
file_list = [externalPaths[0]
] * len(internalPaths)
else:
file_list = None
else:
assert (not any('*' in ep for ep in externalPaths)), (
"Multiple glob paths shouldn't be happening")
file_list = [ex for ex in externalPaths]
assert all(
pc.extension == pathComponents[0].extension
for pc in pathComponents[1::]), (
"Supplied multiple files with multiple extensions")
# The following is necessary for h5 as well as npz-files
internalPathExts = (OpInputDataReader.h5_n5_Exts +
OpInputDataReader.npzExts)
internalPathExts = [
".{}".format(ipx) for ipx in internalPathExts
]
if pathComponents[
0].extension in internalPathExts and internalPaths:
if len(file_list) == len(internalPaths):
# assuming a matching internal paths to external paths
file_list_with_internal = []
for external, internal in zip(
file_list, internalPaths):
if internal:
file_list_with_internal.append(
'{}/{}'.format(external, internal))
else:
file_list_with_internal.append(external)
file_list = file_list_with_internal
else:
# sort of fallback, in case of a mismatch in lengths
for i in range(len(file_list)):
file_list[i] += '/' + internalPaths[0]
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = [make_absolute(f, cwd) for f in file_list]
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join(file_list)
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len(
os.path.splitext(file_list[-1])[1])
nickname = PathComponents(prefix).filenameBase + (
"_" * num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
self.sequenceAxis = sequence_axis
if jsonNamespace is not None:
self.updateFromJson(jsonNamespace)
def __init__(self, filepath=None, jsonNamespace=None, cwd=None, preloaded_array=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged vigra.VigraArray.
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
self._filePath = "" # The original path to the data (also used as a fallback if the data isn't in the project yet)
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
self.allowLabels = True # OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.drange = None
self.normalizeDisplay = True
self.fromstack = False
self.nickname = ""
self.axistags = None
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.fromstack = False
self.nickname = "preloaded-{}-array".format( self.preloaded_array.dtype.name )
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
if '*' in filepath:
file_list = glob.glob(filepath)
file_list = sorted(file_list)
if not isUrl(filepath) and os.path.pathsep in filepath:
file_list = filepath.split(os.path.pathsep)
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = map(lambda f: make_absolute(f, cwd), file_list)
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join( file_list )
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len( os.path.splitext(file_list[-1])[1] )
nickname = PathComponents(prefix).filenameBase + ("_"*num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
if jsonNamespace is not None:
self.updateFromJson( jsonNamespace )
def __init__(self, filepath=None, jsonNamespace=None, cwd=None, preloaded_array=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged vigra.VigraArray.
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
self._filePath = "" # The original path to the data (also used as a fallback if the data isn't in the project yet)
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
self.allowLabels = True # OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.drange = None
self.normalizeDisplay = True
self.fromstack = False
self.nickname = ""
self.axistags = None
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.fromstack = False
self.nickname = "preloaded-{}-array".format( self.preloaded_array.dtype.name )
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
# To support h5 sequences, filepath may contain external and
# internal path components
if not isUrl(filepath):
file_list = filepath.split(os.path.pathsep)
pathComponents = [PathComponents(x) for x in file_list]
externalPaths = [pc.externalPath for pc in pathComponents]
internalPaths = [pc.internalPath for pc in pathComponents]
if len(file_list) > 0:
if len(externalPaths) == 1:
if '*' in externalPaths[0]:
if internalPaths[0] is not None:
assert ('*' not in internalPaths[0]), (
"Only internal OR external glob placeholder supported"
)
file_list = sorted(glob.glob(filepath))
else:
file_list = [externalPaths[0]]
if internalPaths[0] is not None:
if '*' in internalPaths[0]:
# TODO single hdf5 file stacks
raise NotImplementedError(
'Single file h5Stack import is not implemented in the GUI yet.')
else:
assert (not any('*' in ep for ep in externalPaths)), (
"Multiple glob paths shouldn't be happening"
)
file_list = [ex for ex in externalPaths]
assert all(pc.extension == pathComponents[0].extension
for pc in pathComponents[1::]), (
"Supplied multiple files with multiple extensions"
)
# The following is necessary for h5 as well as npz-files
internalPathExts = (
OpInputDataReader.h5Exts +
OpInputDataReader.npzExts
)
internalPathExts = [".{}".format(ipx) for ipx in internalPathExts]
if pathComponents[0].extension in internalPathExts and internalPaths[0]:
for i in xrange(len(file_list)):
file_list[i] += '/' + internalPaths[0]
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = map(lambda f: make_absolute(f, cwd), file_list)
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join( file_list )
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len( os.path.splitext(file_list[-1])[1] )
nickname = PathComponents(prefix).filenameBase + ("_"*num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
if jsonNamespace is not None:
self.updateFromJson( jsonNamespace )
def __init__(self, filepath=None, jsonNamespace=None, cwd=None,
preloaded_array=None, sequence_axis=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged
vigra.VigraArray.
sequence_axis: Axis along which to stack (only applicable for stacks).
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
# The original path to the data (also used as a fallback if the data isn't in the project yet)
self._filePath = ""
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
# OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.allowLabels = True
self.drange = None
self.normalizeDisplay = True
self.sequenceAxis = None
self.fromstack = False
self.nickname = ""
self.axistags = None
self.original_axistags = None
# Necessary in headless mode in order to recover the shape of the raw data
self.laneShape = None
self.laneDtype = None
# A flag indicating whether the dataset is backed by a real source (e.g. file)
# or by the fake provided (e.g. in headless mode when raw data are not necessary)
self.realDataSource = True
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.nickname = "preloaded-{}-array".format(self.preloaded_array.dtype.name)
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
# To support h5 sequences, filepath may contain external and
# internal path components
if not isUrl(filepath):
file_list = filepath.split(os.path.pathsep)
pathComponents = [PathComponents(x) for x in file_list]
externalPaths = [pc.externalPath for pc in pathComponents]
internalPaths = [pc.internalPath for pc in pathComponents]
if len(file_list) > 0:
if len(externalPaths) == 1:
if '*' in externalPaths[0]:
if internalPaths[0] is not None:
assert ('*' not in internalPaths[0]), (
"Only internal OR external glob placeholder supported"
)
file_list = sorted(glob.glob(filepath))
else:
file_list = [externalPaths[0]]
if internalPaths[0] is not None:
if '*' in internalPaths[0]:
# overwrite internalPaths, will be assembled further down
glob_string = "{}{}".format(externalPaths[0], internalPaths[0])
internalPaths = \
OpStreamingH5N5SequenceReaderS.expandGlobStrings(
externalPaths[0], glob_string)
if internalPaths:
file_list = [externalPaths[0]] * len(internalPaths)
else:
file_list = None
else:
assert (not any('*' in ep for ep in externalPaths)), (
"Multiple glob paths shouldn't be happening"
)
file_list = [ex for ex in externalPaths]
assert all(pc.extension == pathComponents[0].extension
for pc in pathComponents[1::]), (
"Supplied multiple files with multiple extensions"
)
# The following is necessary for h5 as well as npz-files
internalPathExts = (
OpInputDataReader.h5_n5_Exts +
OpInputDataReader.npzExts
)
internalPathExts = [".{}".format(ipx) for ipx in internalPathExts]
if pathComponents[0].extension in internalPathExts and internalPaths:
if len(file_list) == len(internalPaths):
# assuming a matching internal paths to external paths
file_list_with_internal = []
for external, internal in zip(file_list, internalPaths):
if internal:
file_list_with_internal.append('{}/{}'.format(external, internal))
else:
file_list_with_internal.append(external)
file_list = file_list_with_internal
else:
# sort of fallback, in case of a mismatch in lengths
for i in range(len(file_list)):
file_list[i] += '/' + internalPaths[0]
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = [make_absolute(f, cwd) for f in file_list]
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join(file_list)
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len(os.path.splitext(file_list[-1])[1])
nickname = PathComponents(prefix).filenameBase + ("_" * num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
self.sequenceAxis = sequence_axis
if jsonNamespace is not None:
self.updateFromJson(jsonNamespace)