class MontagePlotsSchema(ArgSchema):
collection_path = InputFile(required=True,
description="point matches from here")
resolved_path = InputFile(required=True,
description="resolved tiles from here")
save_json_path = OutputFile(
required=True,
missing=None,
default=None,
description=("save residuals to this path if not None"))
save_plot_path = OutputFile(
required=True,
missing=None,
default=None,
description=("save plot to this path if not None"))
make_plot = Boolean(required=True,
missing=True,
default=True,
description=("make a plot?"))
show = Boolean(required=True,
missing=True,
default=True,
description=("show on screen?"))
pdf_out = OutputFile(required=True,
missing=None,
default=None,
description="where to write the pdf output")
class LensCorrectionParameters(ArgSchema):
manifest_path = InputFile(required=True,
description='path to manifest file')
project_path = Str(required=True, description='path to project directory')
fiji_path = InputFile(required=True, description='path to FIJI')
grid_size = Int(
required=True,
description=('maximum row and column to form square '
'subset of tiles starting from zero (based on filenames '
'which end in "\{row\}_\{column\}.tif")'))
heap_size = Int(required=False,
default=20,
description="memory in GB to allocate to Java heap")
outfile = Str(required=False,
description=("File to which json output of lens correction "
"(leaf TransformSpec) is written"))
processing_directory = Str(
required=False,
allow_none=True,
description=("directory to which trakem2 processing "
"directory will be written "
"(will place in project_path directory if "
"unspecified or create temporary directory if None)"))
SIFT_params = Nested(SIFTParameters)
align_params = Nested(AlignmentParameters)
max_threads_SIFT = Int(required=False,
default=3,
description=("Threads specified for SIFT"))
class InputParameters(ArgSchema):
swc_path = InputFile(description='path to swc file for soma location',
required=True)
marker_path = InputFile(description='path to reconstruction marker file',
required=True)
slice_image_flip = Boolean(description=(
'indicates whether the image was flipped relative '
'to the slice (avg_group_label.name = \'Flip Slice Indicator\''),
required=True)
ccf_soma_location = List(
Float,
description='Soma location (x,y,z) coordinates in CCF',
required=True)
slice_transform_list = List(
Float,
required=False,
cli_as_single_argument=True,
description='List defining the transform defining slice cut angle')
slice_transform_dict = Nested(
AffineDictSchema,
description='Dict defining the transform defining the slice cut angle',
required=False)
ccf_path = InputFile(
description='path to common cortical framework streamline file',
required=True)
@mm.validates_schema
def validate_schema_input(self, data):
validate_input_affine(data)
class PairwiseExperimentSchema(OnPremExperimentSchema):
nice_mask_path = InputFile(
required=True,
description="path to mask tiff with unique labels")
nice_dict_path = InputFile(
required=True,
description="path to dict for mask labels to LIMS ids")
class PlaneGroup(DefaultSchema):
local_z_stack_tif = InputFile(
required=True,
description="Full path to local z stack tiff file for this group.")
column_z_stack_tif = InputFile(
description="Full path to column z stack tiff file.")
ophys_experiments = Nested(ExperimentPlane, many=True)
class GenerateEMTileSpecsParameters(ArgSchema):
metafile = InputFile(
required=True,
description="metadata file containing TEMCA acquisition data")
maskUrl = InputFile(required=False,
default=None,
missing=None,
description="absolute path to image mask to apply")
image_directory = InputDir(
required=False,
description=("directory used in determining absolute paths to images. "
"Defaults to parent directory containing metafile "
"if omitted."))
maximum_intensity = Int(
required=False,
default=255,
description=("intensity value to interpret as white"))
minimum_intensity = Int(
required=False,
default=0,
description=("intensity value to interpret as black"))
z = Float(required=False, default=0, description=("z value"))
sectionId = Str(
required=False,
description=("sectionId to apply to tiles during ingest. "
"If unspecified will default to a string "
"representation of the float value of z_index."))
output_path = OutputFile(required=False,
description="directory for output files")
compress_output = Boolean(
required=False,
missing=True,
default=True,
escription=("tilespecs will be .json or .json.gz"))
class SweepExtractionParameters(ArgSchema):
input_nwb_file = InputFile(description="input nwb file", required=True)
input_h5_file = InputFile(description="input h5 file", required=False)
stimulus_ontology_file = OutputFile(description="stimulus ontology JSON",
required=False)
manual_seal_gohm = Float(description="blah")
manual_initial_access_resistance_mohm = Float(description="blah")
manual_initial_input_mohm = Float(description="blah")
class PipelineParameters(ArgSchema):
input_nwb_file = InputFile(description="input nwb file", required=True)
stimulus_ontology_file = OutputFile(description="blash", required=False)
input_h5_file = InputFile(desription="input h5 file", required=False)
output_nwb_file = OutputFile(description="output nwb file", required=True)
qc_fig_dir = OutputFile(description="output qc figure directory",
required=False)
qc_criteria = Nested(QcCriteria, required=True)
manual_sweep_states = Nested(ManualSweepState, required=False, many=True)
class FeatureExtractionParameters(ArgSchema):
input_nwb_file = InputFile(description="input nwb file", required=True)
stimulus_ontology_file = InputFile(description="stimulus ontology JSON",
required=False)
output_nwb_file = OutputFile(description="output nwb file", required=True)
qc_fig_dir = OutputFile(description="output qc figure directory",
required=False)
sweep_features = Nested(FxSweepFeatures, many=True)
cell_features = Nested(CellFeatures, required=True)
class MontageSolverSchema(ArgSchema):
data_dir = InputDir(
required=False,
description="directory containing metafile, images, and matches")
metafile = InputFile(
required=False,
description=("fullpath to metafile. Helps in the case of multiple"
" metafiles in one directory. data_dir will take "
" os.path.dirname(metafile)"))
output_dir = OutputDir(required=False,
missing=None,
default=None,
description="directory for output files")
read_transform_from = Str(
required=False,
missing='metafile',
default='metafile',
validator=mm.validate.OneOf(['metafile', 'reffile', 'dict']),
description="3 possible ways to read in the reference transform")
ref_transform = InputFile(required=False,
missing=None,
default=None,
description="transform json")
ref_transform_dict = Dict(require=False,
missing=None,
description="transform in from memory")
ransacReprojThreshold = Float(
required=False,
missing=10.0,
default=10.0,
description=("passed into cv2.estimateAffinePartial2D()"
"for RANSAC filtering of montage template matches"))
compress_output = Boolean(
required=False,
missing=True,
default=True,
description=("tilespecs will be .json or .json.gz"))
solver_templates = List(
Str,
required=True,
description="input json basenames for the solver args")
solver_template_dir = InputDir(
required=True, description="location of the templates for the solver")
@mm.post_load
def check_solver_inputs(self, data):
for args in data['solver_templates']:
argpath = os.path.join(data['solver_template_dir'], args)
if not os.path.isfile(argpath):
raise mm.ValidationError("solver arg file doesn't exist: %s" %
argpath)
@mm.post_load
def check_metafile(self, data):
if ('data_dir' not in data) & ('metafile' not in data):
raise mm.ValidationError(" must specify either data_dir"
" or metafile")
class GenerateEMTileSpecsParameters(OutputStackParameters):
metafile = InputFile(
required=False,
description="metadata file containing TEMCA acquisition data")
metafile_uri = Str(
required=True,
description=("uri of metadata containing TEMCA acquisition data"))
# FIXME maskUrl and image_directory are not required -- posix_to_uri should support this
maskUrl = InputFile(required=False,
default=None,
missing=None,
description="absolute path to image mask to apply")
maskUrl_uri = Str(required=False,
default=None,
missing=None,
description=("uri of image mask to apply"))
image_directory = InputDir(
required=False,
description=("directory used in determining absolute paths to images. "
"Defaults to parent directory containing metafile "
"if omitted."))
image_prefix = Str(
required=False,
description=(
"prefix used in determining full uris of images in metadata. "
"Defaults to using the / delimited prefix to "
"the metadata_uri if omitted"))
maximum_intensity = Int(
required=False,
default=255,
description=("intensity value to interpret as white"))
minimum_intensity = Int(
required=False,
default=0,
description=("intensity value to interpret as black"))
sectionId = Str(
required=False,
description=("sectionId to apply to tiles during ingest. "
"If unspecified will default to a string "
"representation of the float value of z_index."))
@pre_load
def metafile_to_uri(self, data):
rendermodules.utilities.schema_utils.posix_to_uri(
data, "metafile", "metafile_uri")
# FIXME not required -- does this work
@pre_load
def maskUrl_to_uri(self, data):
rendermodules.utilities.schema_utils.posix_to_uri(
data, "storage_directory", "storage_prefix")
@pre_load
def image_directory_to_prefix(self, data):
rendermodules.utilities.schema_utils.posix_to_uri(
data, "image_directory", "image_prefix")
class InputParameters(ArgSchema):
swc_path = InputFile(description='path to swc file', required=True)
marker_path = InputFile(description='path to marker file', required=True)
soma_depth = Float(
description='Recorded Depth of soma in slice',
required=True,
)
cut_thickness = Float(
description='Thickness of the slice when cut',
required=False,
default=350.,
)
class QcParameters(ArgSchema):
stimulus_ontology_file = InputFile(description="blash", required=False)
qc_criteria = Nested(QcCriteria, required=True)
sweep_features = Nested(QcSweepFeatures, many=True, required=True)
cell_features = Nested(CellFeatures)
output_json = OutputFile(description="output feature json file",
required=True)
class PairwiseRigidSchema(StackTransitionParameters):
match_collection = Str(required=True,
description="Point match collection name")
gap_file = InputFile(
required=False,
default=None,
missing=None,
description="json file {k: v} where int(k) is a z value to skip"
"entries in here that are not already missing will"
"be omitted from the output stack"
"i.e. this is a place one can skip sections")
translate_to_positive = Bool(
required=False,
default=True,
missing=True,
description="translate output stack to positive space")
translation_buffer = List(Float,
required=False,
default=[0, 0],
missing=[0, 0],
description=("minimum (x, y) of output stack if "
"translate_to_positive=True"))
anchor_stack = Str(
require=False,
default=None,
missing=None,
description=("fix transforms using tiles in this stack"))
class InputSchema(ArgSchema):
log_level = Str(required=False, default="INFO")
depths_tif = InputFile(required=True,
description="Full path to depth 2p tiff file.")
surface_tif = InputFile(required=True,
description="Full path to surface 2p tiff file.")
timeseries_tif = InputFile(
required=True, description="Full path to timeseries tiff file.")
storage_directory = OutputDir(
required=True, description="Folder for column stack outputs.")
plane_groups = Nested(PlaneGroup, many=True)
test_mode = Int(
default=0,
description=("Flag to run without actually splitting data. For testing"
" runner mechanism and metadata. Testing of splitting "
"is handled in testing for the mesoscope_2p package."))
class CopiedFile(DefaultSchema):
source = InputFile(required=True, description='copied from here')
destination = FileExists(required=True, description='copied to here')
key = String(required=False, description='passed from inputs')
source_hash = List(Int,
required=False) # int array vs bytes for JSONability
destination_hash = List(Int, required=False)
class ApplyLensCorrectionParameters(StackTransitionParameters):
transform = Nested(TransformParameters)
refId = Str(allow_none=True,
required=True,
description=('Reference ID to use when uploading transform to '
'render database (Not Implemented)'))
labels = List(Str,
required=False,
missing=['lens'],
default=['lens'],
description="labels for the lens correction transform")
maskUrl = InputFile(
required=False,
default=None,
missing=None,
description='path to level 0 maskUrl to apply to stack')
maskUrl_uri = Str(required=False,
default=None,
missing=None,
description="uri for level 0 mask image to apply")
@marshmallow.pre_load
def maskUrl_to_uri(self, data):
rendermodules.utilities.schema_utils.posix_to_uri(
data, "maskUrl", "maskUrl_uri")
class TransformLocalAnnotationParameters(RenderParameters):
stack = Str(required=True,
description='stack to look for transform annotations into')
input_annotation_file = InputFile(required=True,
description='path to annotation file')
output_annotation_file = Str(
required=True, description='path to save transformed annotation')
class DataLoaderSchema(ArgSchema):
landmark_file = InputFile(required=True,
description=("csv file, one line per landmark"))
actions = List(Str,
required=False,
missing=[],
default=[],
cli_as_single_argument=True,
description=("actions to perform on data"))
header = List(Str,
required=False,
default=None,
missing=None,
cli_as_single_argument=True,
description=("passed as names=header to pandas.read_csv()"))
sd_set = Nested(src_dst)
all_flags = Bool(required=False,
missing=False,
default=False,
description="if False, returns only flag=True data")
exclude_labels = List(Int,
required=True,
missing=[100000, 200000],
default=[100000, 200000],
description="ignore Pt labels in this range")
class PointMatchClientParametersQsub(
RenderParameters, SIFTPointMatchParameters, SparkOptions):
sparkhome = InputDir(
required=False,
default="/allen/aibs/pipeline/image_processing/"
"volume_assembly/utils/spark",
missing="/allen/aibs/pipeline/image_processing/"
"volume_assembly/utils/spark",
description="Path to the spark home directory")
pbs_template = InputFile(
required=True,
description="pbs template to wrap spark job")
no_nodes = Int(
required=False,
default=30,
missing=10,
description='Number of nodes to run the pbs job')
ppn = Int(
required=False,
default=30,
missing=30,
description='Number of processors per node (default = 30)')
queue_name = Str(
required=False,
default='connectome',
missing='connectome',
description='Name of the queue to submit the job')
logdir = OutputDir(
required=True,
description="location to set logging for qsub command"
)
class QCPointMatchResultsParameters(RenderParameters):
matchcollections = List(
Str,
required=True,
metadata={'description': 'list of match collections to analyze'})
input_tilepairfile = InputFile(
required=True,
metadata={'description': 'file path of tile pair file to qc'})
output_tilepairfile = Str(
required=True,
metadata={
'description':
'file path of where to save the tile pair file to qc'
})
figdir = Str(required=True,
metadata={'description': 'directory to save images'})
min_matches = Int(
required=False,
default=5,
metadata={
'description':
'number of matches between tiles to be considered a valid match'
})
pool_size = Int(
required=False,
default=20,
metadata={'description': 'number of parallel threads to use'})
class PostProcessROIsInputSchema(ArgSchema):
suite2p_stat_path = Str(
required=True,
validate=lambda x: Path(x).exists(),
description=("Path to s2p output stat file containing ROIs generated "
"during source extraction"))
motion_corrected_video = Str(
required=True,
validate=lambda x: Path(x).exists(),
description=("Path to motion corrected video file *.h5"))
motion_correction_values = InputFile(
required=True,
description=("Path to motion correction values for each frame "
"stored in .csv format. This .csv file is expected to"
"have a header row of either:\n"
"['framenumber','x','y','correlation','kalman_x',"
"'kalman_y']\n['framenumber','x','y','correlation',"
"'input_x','input_y','kalman_x',"
"'kalman_y','algorithm','type']"))
output_json = OutputFile(
required=True, description=("Path to a file to write output data."))
maximum_motion_shift = Float(
missing=30.0,
required=False,
allow_none=False,
description=("The maximum allowable motion shift for a frame in pixels"
" before it is considered an anomaly and thrown out of "
"processing"))
abs_threshold = Float(
missing=None,
required=False,
allow_none=True,
description=("The absolute threshold to binarize ROI masks against. "
"If not provided will use quantile to generate "
"threshold."))
binary_quantile = Float(
missing=0.1,
validate=Range(min=0, max=1),
description=("The quantile against which an ROI is binarized. If not "
"provided will use default function value of 0.1."))
npixel_threshold = Int(
default=50,
required=False,
description=("ROIs with fewer pixels than this will be labeled as "
"invalid and small size."))
aspect_ratio_threshold = Float(
default=0.2,
required=False,
description=("ROIs whose aspect ratio is <= this value are "
"not recorded. This captures a large majority of "
"Suite2P-created artifacts from motion border"))
morphological_ops = Bool(
default=True,
required=False,
description=("whether to perform morphological operations after "
"binarization. ROIs that are washed away to empty "
"after this operation are eliminated from the record. "
"This can apply to ROIs that were previously labeled "
"as small size, for example."))
class SIFTPointMatchParameters(
argschema.ArgSchema,
FeatureExtractionParameters, FeatureRenderParameters,
FeatureRenderClipParameters,
FeatureStorageParameters, MatchDerivationParameters,
RenderParametersMatchWebServiceParameters):
pairJson = InputFile(required=True, description=(
"JSON file where tile pairs are stored (.json, .gz, .zip)"))
class FileToCopy(DefaultSchema):
source = InputFile(required=True, description='copy from here')
destination = String(
required=True,
description='copy to here (full path, not just directory!)')
key = String(required=True,
description='will be passed through to outputs, allowing a '
'name or kind to be associated with this file')
class ImportTrakEM2AnnotationParameters(RenderTrakEM2Parameters):
EMstack = Str(required=True,
description='stack to look for trakem2 patches in')
trakem2project = InputFile(required=True,
description='trakem2 file to read in')
output_annotation_file = OutputFile(
required=True, description="place to save annotation output")
output_bounding_box_file = OutputFile(
required=True, description="place to save bounding box output")
class GenerateSerialEMTileSpecsParameters(OutputStackParameters):
image_file = InputFile(
required=True,
description="metadata file containing SerialEM acquisition data (idoc)"
)
z_spacing = Float(required=False,
default=100.0,
description="spacing between slices/ section thickness")
class Nwb2SinkConfig(DefaultSchema):
"""Configure an Nwb2Sink (by arguing the input nwb path)
"""
nwb_path = InputFile(
description=(
"Path to input NWB. This will serve as the basis for the output "
"file."
),
required=True
)
class ViewMatchesSchema(ArgSchema):
collection_path = InputFile(
required=False,
description="if specified, will read collection from here")
collection_basename = Str(
required=True,
missing="collection.json",
default="collection.json",
description=("basename for collection file if collection_path"
" not specified. will also check for .json.gz"))
data_dir = InputDir(
required=True,
description=("directory containing image files. Will also be dir"
" dir for collection path, if not otherwise specified"))
resolved_tiles = List(
Str,
required=True,
missing=["resolvedtiles.json.gz", "resolvedtiles_input.json.gz"],
description=("will take the transform from the first file"
" matching this list, if possible"))
transform_file = InputFile(
required=False,
description=("if provided, will get lens correction transform "
" from here"))
view_all = Boolean(
required=True,
missing=False,
default=False,
description=("will plot all the pair matches. can be useful "
"for lens correction to file. probably not desirable "
"for montage"))
show = Boolean(required=True,
missing=True,
default=True,
description=("show on screen?"))
match_index = Int(required=True,
missing=0,
default=0,
description=("which index of self.matches to plot"))
pdf_out = OutputFile(required=True,
missing='./view_matches_output.pdf',
default='./view_matches_output.pdf',
description="where to write the pdf output")
class InputParameters(ArgSchema):
gradient_path = InputFile(
description=(
"File at this location is a netcdf-formatted 2D gradient field."
"Dimensions are x, y, dim, where dim defines the component (dx "
"or dy) of the gradient"
),
required=True
)
node = List(Float,
description='[x,y,z] location in gradient field to get angle',
cli_as_single_argument=True,
default=[0, 0, 0],
required=False,
)
step = Int(
description=(
"The input gradient field will be decimated isometrically by "
"this factor"
),
default=1,
required=False,
)
neighbors = Int(
description=('number of x and y neighbor idxs to use for interpolation, '
'must even and greater > 4'
),
required=False,
default=12,
# min data points must be >= (k+1)**2, k=1 for linear
)
swc_path = InputFile(
description='path to swc file for soma location',
required=True
)
@mm.validates_schema
def validate_schema_input(self, data):
validate_neighbors(data.get('neighbors'))
class DepthField(DefaultSchema):
gradient_field_path = InputFile(description=(
"The path to an xarray file describing the gradient of cortical "
"depth on some domain. This file should contain one dataarray "
"called 'gradient' which has dimensions drawn from "
"{'x', 'y', 'z', 'component'}. The coords of x, y, z define the "
"domain over which the gradient was computed. The component "
"dimension describes the dimension associated with each component "
"of the gradient and should have coords drawn from {'x', 'y', 'z'}."),
required=True)
depth_field_path = InputFile(
description=("As gradient field, but gives depth values"),
required=True)
soma_origin = Bool(
description="If true, the field is centered at the soma",
required=True,
default=True)
pia_sign = Int(description="which direction is the pia",
required=True,
default=1,
validate=lambda val: val in {1, -1})