class InputSchema(ArgSchema):
log_level = Str(required=False, default="INFO")
depths_tif = InputFile(required=True,
description="Full path to depth 2p tiff file.")
surface_tif = InputFile(required=True,
description="Full path to surface 2p tiff file.")
timeseries_tif = InputFile(
required=True, description="Full path to timeseries tiff file.")
storage_directory = OutputDir(
required=True, description="Folder for column stack outputs.")
plane_groups = Nested(PlaneGroup, many=True)
test_mode = Int(
default=0,
description=("Flag to run without actually splitting data. For testing"
" runner mechanism and metadata. Testing of splitting "
"is handled in testing for the mesoscope_2p package."))
class Image(DefaultSchema):
input_path = InputFile(description="Read the image from here",
required=True)
output_path = OutputFile(
description="Write outputs to (siblings of) this path", required=True)
downsample = Int(
description=("Downsample the image by this amount on each dimension "
"(currently this is just a decimation, hence Int)."),
required=True,
default=8)
overlay_types = List(
String,
description=("produce these types of overlays for this image. "
"See ImageOutputter for options"),
required=True,
default=["before", "after"])
class QualityMetricsParams(DefaultSchema):
isi_threshold = Float(required=False, default=0.0015, help='Maximum time (in seconds) for ISI violation')
min_isi = Float(required=False, default=0.00, help='Minimum time (in seconds) for ISI violation')
num_channels_to_compare = Int(required=False, default=13, help='Number of channels to use for computing PC metrics; must be odd')
max_spikes_for_unit = Int(required=False, default=500, help='Number of spikes to subsample for computing PC metrics')
max_spikes_for_nn = Int(required=False, default=10000, help='Further subsampling for NearestNeighbor calculation')
n_neighbors = Int(required=False, default=4, help='Number of neighbors to use for NearestNeighbor calculation')
n_silhouette = Int(required=False, default=10000, help='Number of spikes to use for calculating silhouette score')
drift_metrics_min_spikes_per_interval = Int(required=False, default=10, help='Minimum number of spikes for computing depth')
drift_metrics_interval_s = Float(required=False, default=100, help='Interval length is seconds for computing spike depth')
class PointMatchCollectionParameters(DefaultSchema):
owner = Str(
required=False,
default=None,
missing=None,
description="Point match collection owner (defaults to render owner")
match_collection = Str(required=True,
description="Point match collection name")
server = Str(required=False,
default=None,
missing=None,
description="baseURL of the Render service holding the "
"point match collection")
verbose = Int(required=False,
default=0,
missing=0,
description="Verbose output flag")
class CreateFastStacksParameters(RenderParameters):
statetableFile = InputFile(required=True, description='state table file')
projectDirectory = InputDir(required=True,
description='path to project root')
outputStackPrefix = Str(
required=False,
default="ACQ",
description=
'prefix to include in front of channel name for render stack')
pool_size = Int(require=False,
default=20,
description='number of parallel threads to use')
delete_stack = Boolean(
require=False,
default=True,
description=
'flag to decide whether stack should be deleted before new upload')
class PointMatchCollectionParameters(DefaultSchema):
server = Str(required=False,
default=None,
missing=None,
description="Render server's base URL")
owner = Str(
required=False,
default=None,
missing=None,
description=
"Owner of the point match collection (default - render service owner)")
match_collection = Str(required=True,
description="Montage point match collection")
verbose = Int(required=False,
default=0,
missing=0,
description="Verbose output from point match loader needed?")
class ReceptiveFieldMapping(DefaultSchema):
stimulus_key = List(String,
default=ReceptiveFieldMapping.known_stimulus_keys(),
help='Key for the receptive field mapping stimulus')
trial_duration = Float(
default=0.25,
help='typical length of a epoch for given stimulus in seconds')
minimum_spike_count = Int(
default=10,
help='Minimum number of spikes for computing receptive field parameters'
)
mask_threshold = Float(
default=1.0,
help=
'Threshold (as fraction of peak) for computing receptive field mask')
stimulus_step_size = Float(
default=10.0, help='Distance between stimulus locations in degrees')
class FeatureStorageParameters(argschema.schemas.DefaultSchema):
# TODO is this inputdir or outputdir?
rootFeatureDirectory = Str(
required=False,
description=("Root directory for saved feature lists. "
"Features extracted from dynamically rendered canvases "
"if excluded or None."))
requireStoredFeatures = Boolean(
required=False,
description=(
"Whether to throw an exception in case features stored in "
"rootFeatureDirectory cannot be found. "
"Missing features are extracted from dynamically rendered canvases "
"if excluded or None"))
maxFeatureCacheGb = Int(
required=False,
description=(
"Maximum size of feature cache, in GB. 2GB if excluded or None"))
class MaterializedBoxParameters(argschema.schemas.DefaultSchema):
stack = Str(required=True,
description=("stack fromw which boxes will be materialized"))
rootDirectory = OutputDir(
required=True,
description=
("directory in which materialization directory structure will be "
"created (structure is "
"<rootDirectory>/<project>/<stack>/<width>x<height>/<mipMapLevel>/<z>/<row>/<col>.<fmt>)"
))
width = Int(required=True,
description=("width of flat rectangular tiles to generate"))
height = Int(required=True,
description=("height of flat rectangular tiles to generate"))
maxLevel = Int(required=False,
default=0,
description=("maximum mipMapLevel to generate."))
fmt = Str(required=False,
validator=validate.OneOf(['PNG', 'TIF', 'JPG']),
description=("image format to generate mipmaps -- "
"PNG if not specified"))
maxOverviewWidthAndHeight = Int(
required=False,
description=(
"maximum pixel size for width or height of overview image. "
"If excluded or 0, no overview generated."))
skipInterpolation = Boolean(
required=False,
description=("whether to skip interpolation (e.g. DMG data)"))
binaryMask = Boolean(
required=False,
description=("whether to use binary mask (e.g. DMG data)"))
label = Boolean(
required=False,
description=("whether to generate single color tile labels rather "
"than actual images"))
createIGrid = Boolean(required=False,
description=("whther to create an IGrid file"))
forceGeneration = Boolean(
required=False, description=("whether to regenerate existing tiles"))
renderGroup = Int(
required=False,
description=(
"index (1-n) identifying coarse portion of layer to render"))
numberOfRenderGroups = Int(
required=False,
description=(
"used in conjunction with renderGroup, total number of groups "
"being used"))
filterListName = Str(
required=False,
description=("Apply specified filter list to all renderings"))
class PostprocessingParams(DefaultSchema):
within_unit_overlap_window = Float(
required=False,
default=0.000166,
help='Time window for removing overlapping spikes for one unit.')
between_unit_overlap_window = Float(
required=False,
default=0.000166,
help='Time window for removing overlapping spikes between two units.')
between_unit_dist_um = Int(
required=False,
default=5,
help=
'Number of channels (above and below peak channel) to search for overlapping spikes'
)
deletion_mode = String(required=False,
default='lowAmpCluster',
help='lowAmpCluster or deleteFirst')
class ViewMatchesSchema(ArgSchema):
collection_path = InputFile(
required=False,
description="if specified, will read collection from here")
collection_basename = Str(
required=True,
missing="collection.json",
default="collection.json",
description=("basename for collection file if collection_path"
" not specified. will also check for .json.gz"))
data_dir = InputDir(
required=True,
description=("directory containing image files. Will also be dir"
" dir for collection path, if not otherwise specified"))
resolved_tiles = List(
Str,
required=True,
missing=["resolvedtiles.json.gz", "resolvedtiles_input.json.gz"],
description=("will take the transform from the first file"
" matching this list, if possible"))
transform_file = InputFile(
required=False,
description=("if provided, will get lens correction transform "
" from here"))
view_all = Boolean(
required=True,
missing=False,
default=False,
description=("will plot all the pair matches. can be useful "
"for lens correction to file. probably not desirable "
"for montage"))
show = Boolean(required=True,
missing=True,
default=True,
description=("show on screen?"))
match_index = Int(required=True,
missing=0,
default=0,
description=("which index of self.matches to plot"))
pdf_out = OutputFile(required=True,
missing='./view_matches_output.pdf',
default='./view_matches_output.pdf',
description="where to write the pdf output")
class PointMatchTransferParameters(RenderTransferParameters):
collection_source = Str(
required=True,
metadata={
'description': 'point match collection to move from source_render'
})
collection_target = Str(
required=False,
metadata={
'description':
'point match colleciton to move to target_render (default to the same)'
})
pool_size = Int(
required=False,
default=20,
metadata={
'description':
'point match colleciton to move to target_render (default to the same)'
})
class SwapPointMatches(RenderParameters):
match_owner = Str(
required=True,
description="Match collection owner name")
source_collection = Str(
required=True,
description="Source point match collection")
target_collection = Str(
required=True,
description="Target point match collection")
zValues = List(
Int,
required=True,
description="List of integer group ids")
pool_size = Int(
required=False,
default=5,
missing=5,
description="Pool size")
class NonFileParameters(DefaultSchema):
use_rsync = Boolean(default=True,
description='copy files using rsync rather than '
'shutil (this is not likely to work if '
'you are running windows!)')
hasher_key = String(default='sha256',
validate=lambda st: st in available_hashers,
allow_none=True,
description='select a hash function to compute over '
'base64-encoded pre- and post-copy files')
raise_if_comparison_fails = Boolean(default=True,
description='if a hash comparison '
'fails, throw an error ('
'vs. a warning)')
make_parent_dirs = Boolean(default=True,
description='build missing parent directories '
'for destination')
chmod = Int(default=775,
description="destination files (and any created parents will "
"have these permissions")
class ConsolidateTransformsParameters(RenderParameters):
stack = Str(required=True, description='stack to consolidate')
postfix = Str(
required=False,
default="_CONS",
description=
'postfix to add to stack name on saving if no output defined (default _CONS)'
)
output_stack = Str(
required=False,
description='name of output stack (default to adding postfix to input)'
)
output_directory = Str(
required=True,
description='location to store tilespecs before uploading')
pool_size = Int(
required=False,
default=20,
description='name of output stack (default to adding postfix to input)'
)
class DepthField(DefaultSchema):
gradient_field_path = InputFile(description=(
"The path to an xarray file describing the gradient of cortical "
"depth on some domain. This file should contain one dataarray "
"called 'gradient' which has dimensions drawn from "
"{'x', 'y', 'z', 'component'}. The coords of x, y, z define the "
"domain over which the gradient was computed. The component "
"dimension describes the dimension associated with each component "
"of the gradient and should have coords drawn from {'x', 'y', 'z'}."),
required=True)
depth_field_path = InputFile(
description=("As gradient field, but gives depth values"),
required=True)
soma_origin = Bool(
description="If true, the field is centered at the soma",
required=True,
default=True)
pia_sign = Int(description="which direction is the pia",
required=True,
default=1,
validate=lambda val: val in {1, -1})
class InputParameters(ArgSchema):
swc_path = InputFile(description="path to input swc (csv) file",
required=True)
depth = Nested(DepthField,
description=("A transform which can be evaluated at the "
"location of each node in the input swc"),
required=True,
many=False)
layers = Nested(Layer,
description="specification of layer bounds",
many=True,
required=True)
step_size = Float(description=(
"size of each step, in the same units as the depth field and swc"),
required=True,
default=1.0)
output_path = OutputFile(description="write (csv) outputs here",
required=True)
max_iter = Int(description="how many steps to take before giving up",
required=True,
default=1000)
class QualityMetricsParams(DefaultSchema):
isi_threshold = Float(required=False, default=0.0015, help='Maximum time (in seconds) for ISI violation')
min_isi = Float(required=False, default=0.00, help='Minimum time (in seconds) for ISI violation')
num_channels_to_compare = Int(required=False, default=13, help='Number of channels to use for computing PC metrics; must be odd')
max_spikes_for_unit = Int(required=False, default=500, help='Number of spikes to subsample for computing PC metrics')
max_spikes_for_nn = Int(required=False, default=10000, help='Further subsampling for NearestNeighbor calculation')
n_neighbors = Int(required=False, default=4, help='Number of neighbors to use for NearestNeighbor calculation')
n_silhouette = Int(required=False, default=10000, help='Number of spikes to use for calculating silhouette score')
drift_metrics_min_spikes_per_interval = Int(required=False, default=10, help='Minimum number of spikes for computing depth')
drift_metrics_interval_s = Float(required=False, default=100, help='Interval length is seconds for computing spike depth')
quality_metrics_output_file = String(required=True, help='CSV file where metrics will be saved')
include_pc_metrics = Bool(required=False, default=True, help='Compute features that require principal components')
class ConsolidateTransformsParameters(RenderParameters):
stack = Str(required=True, description='stack to consolidate')
postfix = Str(
required=False,
default="_CONS",
description=
'postfix to add to stack name on saving if no output defined (default _CONS)'
)
transforms_slice = Slice(
required=True,
description="a string representing a slice describing \
the set of transforms to be consolidated (i.e. 1:)"
)
output_stack = Str(
required=False,
description='name of output stack (default to adding postfix to input)'
)
pool_size = Int(
required=False,
default=10,
description='name of output stack (default to adding postfix to input)'
)
minZ = Float(
required=False,
description=
"""minimum z to consolidate in read in from stack and write to output_stack\
default to minimum z in stack""")
maxZ = Float(
required=False,
description=
"""minimaximummum z to consolidate in read in from stack and write to output_stack\
default to maximum z in stack""")
overwrite_zlayer = Boolean(
required=False,
default=False,
description=("whether to remove the existing layer from the "
"target stack before uploading."))
close_stack = Boolean(required=False, default=False)
class PointMatchFilteringOptions(DefaultSchema):
NumRandomSamplingsMethod = Str(
required=False,
default="Desired confidence",
missing="Desired confidence",
description="Numerical random sampling method")
MaximumRandomSamples = Int(required=False,
default=5000,
missing=5000,
description="Max number of random samples")
DesiredConfidence = Float(
required=False,
default=99.9,
missing=99.9,
description="Desired confidence value for point match filtering")
Transform = Str(required=False,
default="Affine",
missing="Affine",
description="RANSAC model to fit")
PixelDistanceThreshold = Float(required=False,
default=0.1,
missing=0.1,
description="Pixel distance threshold")
class PointMatchOptimizationParameters(RenderParameters):
stack = Str(required=True,
description='Name of the stack containing the tile pair')
tile_stack = Str(
required=False,
default=None,
missing=None,
description='Name of the stack that will hold these two tiles')
tileId1 = Str(required=True,
description='tileId of the first tile in the tile pair')
tileId2 = Str(required=True,
description='tileId of the second tile in the tile pair')
pool_size = Int(required=False,
default=10,
missing=10,
description='Pool size for parallel processing')
SIFT_options = Nested(SIFT_options, required=True)
outputDirectory = OutputDir(
required=True,
description=(
'Parent directory in which subdirectories will be '
'created to store images and point-match results from SIFT'))
url_options = Nested(url_options, required=True)
class PostgresInputConfigSchema(mm.Schema):
host = String(
description="",
required=True
)
database = String(
description="",
required=True
)
user = String(
description="",
required=True
)
password = String(
description="",
required=False,
default=os.environ.get("POSTGRES_SOURCE_PASSWORD")
)
port = Int(
description="",
required=False, # seems not to get hydrated from the default
default=5432
)
class ReplaceFileNameParameters(RenderParameters):
input_stack = Str(required=True,
metadata={'description': 'stack to apply affine to'})
output_stack = Str(
required=False,
metadata={
'description':
'stack to save answer into (defaults to overwriting input_stack)'
})
regex_find = Str(required=True,
metadata={
'description':
'regular expression to search for in imageUrl'
})
regex_replace = Str(
required=True,
metadata={'description': 'regular expression replacement string'})
pool_size = Int(required=False,
default=20,
metadata={
'description':
'size of pool for parallel processing (default=20)'
})
class ConsolidateTransformsParameters(RenderParameters):
stack = Str(required=True,
metadata={'description': 'stack to consolidate'})
postfix = Str(
required=False,
default="_CONS",
metadata={
'description':
'postfix to add to stack name on saving if no output defined (default _CONS)'
})
output_stack = Str(
required=False,
metadata={
'description':
'name of output stack (default to adding postfix to input)'
})
pool_size = Int(
required=False,
default=20,
metadata={
'description':
'name of output stack (default to adding postfix to input)'
})
class RenderSchema(DefaultSchema):
protocol = Str(
default="http",
help="Protocol to connect to render with (http or https)",
required=False)
port = Int(default=80, required=False)
encoding = Str(
default="jpg",
help="Encoding option for the neuroglancer render datasource (jpg or raw16)",
required=False)
all_channels = Boolean(default=False,
help="Use Render API to query for and load all channels",
required=False)
alt_render = Str(
default="",
help="Alternate render host to use for vizrelay API calls [to work in Docker]",
required=False)
enable_one_channel = Boolean(default=False,
help="Enable only one of the channels",
required=False)
channel_name_shader_sub = Dict(default={},
help="Dictionary of CHANNEL_NAME : { SUB_NAME : SUB_VALUE }",
required=False)
class OtherParameters(DefaultSchema):
rod = Float(required=True, metadata={'description': ''})
maxEpsilon = Float(required=True, metadata={'description': ''})
minInlierRatio = Float(required=True, metadata={'description': ''})
minNumInliers = Int(required=True, metadata={'description': ''})
expectedModelIndex = Int(required=True, metadata={'description': ''})
multipleHypotheses = Bool(required=True, metadata={'description': ''})
rejectIdentity = Bool(required=True, metadata={'description': ''})
identityTolerance = Float(required=True, metadata={'description': ''})
tilesAreInPlace = Bool(required=True, metadata={'description': ''})
desiredModelIndex = Int(required=True, metadata={'description': ''})
regularize = Bool(required=True, metadata={'description': ''})
maxIterationsOptimize = Int(required=True, metadata={'description': ''})
maxPlateauWidthOptimize = Int(required=True, metadata={'description': ''})
dimension = Int(required=True, metadata={'description': ''})
lambdaVal = Float(required=True, metadata={'description': ''})
clearTransform = Bool(required=True, metadata={'description': ''})
visualize = Bool(required=True, metadata={'description': ''})
class MeshLensCorrectionSchema(ArgSchema):
nvertex = Int(required=False,
default=1000,
missinf=1000,
description="maximum number of vertices to attempt")
tilespec_file = InputFile(required=False,
description="path to json of tilespecs")
tilespecs = List(Dict,
required=False,
description="list of dict of tilespecs")
match_file = InputFile(required=False,
description="path to json of matches")
matches = List(Dict, required=False, description="list of dict of matches")
regularization = Nested(regularization, missing={})
good_solve = Nested(good_solve_criteria, missing={})
output_dir = OutputDir(required=False,
description="directory for output files")
outfile = Str(required=False,
description=("Basename to which resolved json output of "
"lens correction is written"))
compress_output = Boolean(
required=False,
missing=True,
default=True,
description=("tilespecs will be .json or .json.gz"))
timestamp = Boolean(required=False,
missing=False,
default=False,
description="add a timestamp to basename output")
@mm.post_load
def one_of_two(self, data):
for a, b in [['tilespecs', 'tilespec_file'], ['matches',
'match_file']]:
if (a in data) == (b in data): # xor
raise mm.ValidationError(
'must specify one and only one of %s or %s' % (a, b))
class PointMatchParameters(DefaultSchema):
NumRandomSamplingsMethod = Str(
required=False,
default="Desired confidence",
missing="Desired confidence",
description='Numerical Random sampling method')
MaximumRandomSamples = Int(required=False,
default=5000,
missing=5000,
description='Maximum number of random samples')
DesiredConfidence = Float(required=False,
default=99.9,
missing=99.9,
description='Desired confidence level')
PixelDistanceThreshold = Float(
required=False,
default=0.1,
missing=0.1,
description='Pixel distance threshold for filtering')
Transform = Str(required=False,
default="AFFINE",
missing="AFFINE",
description="Transformation type parameter for point "
"match filtering (default AFFINE)")
class LensCorrectionSchema(ArgSchema):
data_dir = InputDir(
required=True,
description="directory containing metafile, images, and matches")
output_dir = OutputDir(required=False,
description="directory for output files")
mask_file = InputFile(required=False,
default=None,
missing=None,
description="mask to apply to each tile")
nvertex = Int(required=False,
default=1000,
missinf=1000,
description="maximum number of vertices to attempt")
ransac_thresh = Float(required=False,
default=5.0,
missing=5.0,
description="ransac outlier threshold")
regularization = Nested(regularization, missing={})
good_solve = Nested(good_solve_criteria, missing={})
ignore_match_indices = List(
Int,
required=False,
default=None,
missing=None,
description=("debug feature for ignoring certain indices"
" of the match collection"))
compress_output = Boolean(
required=False,
missing=True,
default=True,
description=("tilespecs will be .json or .json.gz"))
timestamp = Boolean(required=False,
missing=False,
default=False,
description="add a timestamp to basename output")
class QualityMetricsParams(DefaultSchema):
isi_threshold = Float(required=False,
default=0.0015,
help='Maximum time (in seconds) for ISI violation')
min_isi = Float(required=False,
default=0.00,
help='Minimum time (in seconds) for ISI violation')
max_radius_um = Int(required=False,
default=68,
help='Maximum radius for computing PC metrics, in um')
max_spikes_for_unit = Int(
required=False,
default=500,
help='Number of spikes to subsample for computing PC metrics')
max_spikes_for_nn = Int(
required=False,
default=10000,
help='Further subsampling for NearestNeighbor calculation')
n_neighbors = Int(
required=False,
default=4,
help='Number of neighbors to use for NearestNeighbor calculation')
n_silhouette = Int(
required=False,
default=10000,
help='Number of spikes to use for calculating silhouette score')
drift_metrics_min_spikes_per_interval = Int(
required=False,
default=10,
help='Minimum number of spikes for computing depth')
drift_metrics_interval_s = Float(
required=False,
default=100,
help='Interval length is seconds for computing spike depth')
include_pcs = Boolean(
required=False,
default=True,
help='Set to false if features were not saved with Phy output')
