class tPrimeParams(DefaultSchema): tPrime_path = InputDir(help='directory containing the TPrime executable.') sync_period = Float(default=1.0, help='Period of sync waveform (sec).') toStream_sync_params = String( required=False, default='SY=0,384,6,500', help= 'string of CatGT params used to extract to stream sync edges, e.g. SY=0,384,6,500' ) ni_sync_params = String( required=False, default='XA=0,1,3,500', help= 'string of CatGT params used to extract NI sync edges, e.g. XA=0,1,3,500' ) ni_ex_list = String( required=False, default='', help= 'string of CatGT params used to extract edges from ni, e.g. XA=0,1,3,500' ) im_ex_list = String( required=False, default='', help= 'string of CatGT params used to extract edges from im streams, e.g. SY=0,384,6,500' ) tPrime_3A = Boolean(required=False, default=False, help='is this 3A data?') toStream_path_3A = String(required=False, help='full path to toStream edges file') fromStream_list_3A = List( String, required=False, help='list of full paths to fromStream edges files')
class SessionMetadata(RaisingSchema): specimen_name = String(required=True) age_in_days = Float(required=True) full_genotype = String(required=True) strain = String(required=True) sex = String(required=True) stimulus_name = String(required=True)
class Channel(RaisingSchema): @mm.pre_load def set_field_defaults(self, data, **kwargs): if data.get("filtering") is None: data["filtering"] = ("AP band: 500 Hz high-pass; " "LFP band: 1000 Hz low-pass") if data.get("manual_structure_acronym") is None: data["manual_structure_acronym"] = "" return data id = Int(required=True) probe_id = Int(required=True) valid_data = Boolean(required=True) local_index = Int(required=True) probe_vertical_position = Int(required=True) probe_horizontal_position = Int(required=True) manual_structure_id = Int(required=True, allow_none=True) manual_structure_acronym = String(required=True) anterior_posterior_ccf_coordinate = Float(allow_none=True) dorsal_ventral_ccf_coordinate = Float(allow_none=True) left_right_ccf_coordinate = Float(allow_none=True) impedence = Float(required=False, allow_none=True, default=None) filtering = String(required=False) @mm.post_load def set_impedence_default(self, data, **kwargs): # This must be a post_load operation as np.nan is not a valid # JSON format 'float' type for the Marshmallow `Float` field # (so validation fails if this is set at pre_load) if data.get("impedence") is None: data["impedence"] = np.nan return data
class KilosortParameters(DefaultSchema): Nfilt = Int(required=False, default=1024) Threshold = String(required=False, default="[4, 10, 10]") lam = String(required=False, default="[5, 20, 20]") IntitalizeTh = Int(required=False, default=-4) InitializeNfilt = Int(required=False, default=10000)
class ProbeInputParameters(DefaultSchema): name = String(required=True, help="Identifier for this probe") sampling_rate = Float( required=True, help= "The sampling rate of the probe, in Hz, assessed on the probe clock.", ) lfp_sampling_rate = Float( required=True, help="The sampling rate of the LFP collected on this probe.") start_index = Int( default=0, help="Sample index of probe recording start time. Defaults to 0.") barcode_channel_states_path = String( required=True, help= "Path to the channel states file. This file contains a 1-dimensional array whose axis is events and whose " "values indicate the state of the channel line (rising or falling) at that event.", ) barcode_timestamps_path = String( required=True, help= "Path to the timestamps file. This file contains a 1-dimensional array whose axis is events and whose " "values indicate the sample on which each event was detected.", ) mappable_timestamp_files = Nested( ProbeMappable, many=True, help= "Timestamps files for this probe. Describe the times (in probe samples) when e.g. lfp samples were taken or spike events occured", )
class ProbeOutputParameters(DefaultSchema): name = String(required=True, help='Identifier for this probe.') csd_path = String(required=True, help='Path to current source density file.') csd_channels = List(Int, required=True, help='LFP channels from which CSD was calculated.')
class ProbeInputParameters(DefaultSchema): name = String(required=True, help='Identifier for this probe.') lfp_data_path = String(required=True, help='Path to lfp data for this probe') lfp_timestamps_path = String( required=True, help="Path to aligned lfp timestamps for this probe.") surface_channel = Int( required=True, help='Estimate of surface (pia boundary) channel index') reference_channels = List( Int, many=True, help='Indices of reference channels for this probe') csd_output_path = String(required=True, help='CSD output will be written here.') sampling_rate = Float(required=True, help='sampling rate assessed on master clock') total_channels = Int(default=384, help='Total channel count for this probe.') surface_channel_adjustment = Int( default=40, help= 'Erring up in the surface channel estimate is less dangerous for the CSD calculation than erring down, so an adjustment is provided.' ) spacing = Float( default=0.04, help= 'distance (in millimiters) between lengthwise-adjacent rows of recording sites on this probe.' ) phase = String( required=True, help= 'The probe type (3a or PXI) which determines if channels need to be reordered' )
class ProbeOutputParameters(DefaultSchema): name = String(required=True, help='Identifier for this probe.') lfp_data_path = String(required=True, help='Output subsampled data file.') lfp_timestamps_path = String(required=True, help='Timestamps for subsampled data.') lfp_channel_info_path = String( required=True, help='LFP channels from that was subsampled.')
class CopiedFile(RaisingSchema): source = String(required=True, description='copied from here') destination = String(required=True, description='copied to here') key = String(required=False, description='passed from inputs') source_hash = List(Int, required=False) # int array vs bytes for JSONability destination_hash = List(Int, required=False)
class InputParameters(ArgSchema): reconstructions = Nested( Reconstruction, description="The morphological reconstructions to be processed", required=True, many=True ) heavy_output_path = OutputFile( description=( "features whose results are heavyweight data (e.g. the numpy " "arrays returned by layer histograms features) are stored here." ), required=True ) feature_set = String( description="select the basic set of features to calculate", required=False, default="aibs_default" ) only_marks = List( String, cli_as_single_argument=True, description=( "restrict calculated features to those with this set of marks" ), required=False ) required_marks = String( description=( "Error (vs. skip) if any of these marks fail validation" ), required=False, many=True ) output_table_path = OutputFile( description=( "this module writes outputs to a json specified as --output_json. " "If you want to store outputs in a different format " "(.csv is supported currently), specify this parameter" ), required=False ) num_processes = Int( description=( "Run a multiprocessing pool with this many processes. " "Default is min(number of cpus, number of swcs). " "Setting num_processes to 1 will avoid a pool." ), required=False, default=None, allow_none=True ) global_parameters = Nested( GlobalParameters, description=( "provide additional configuration to this feature extraction run. " "This configuration will be applied to all morphologies processed." ), required=False )
class PiaWmStreamlineSchema(ArgSchema): """Arg Schema for run_pia_wm_streamlines""" pia_path_str = String( required=True, description='string alternating x, y coordinates outlining the pia') wm_path_str = String( required=True, description='string alternating x, y coordinates outlining the wm') soma_path_str = String( required=False, description='string alternating x, y coordinates outlining the soma. ' 'If provided, streamlines will be translated so that ' 'the origin is at the soma') resolution = Float(required=False, default=1, description='Resolution of pixels in microns') pia_fixed_value = Float(required=False, default=1, description='Fixed value pia boundary condition') wm_fixed_value = Float(required=False, default=0, description='Fixed value wm boundary condition') mesh_res = Int(required=False, default=20, description='Resolution for mesh for laplace solver') output_dir = OutputDir(required=True, description='Directory to write xarray results')
class FileToCopy(DefaultSchema): source = InputFile(required=True, description='copy from here') destination = String( required=True, description='copy to here (full path, not just directory!)') key = String(required=True, description='will be passed through to outputs, allowing a ' 'name or kind to be associated with this file')
class OutputParameters(OutputSchema): execution_time = Float() kilosort_commit_hash = String() kilosort_commit_date = String() mask_channels = NumpyArray() nTemplate = Int() nTot = Int()
class OutputSchema(DefaultSchema): input_parameters = Nested( InputParameters, description=("Input parameters the module " "was run with"), required=True, ) output_path = String(help="Path to output csv file") output_frame_times_path = String(help="output all frame times here")
class StimulusInputParameters(DefaultSchema): stimulus_table_path = String(required=True, help='Path to stimulus table') key = String(required=True, help='CSD is calculated from a specific stimulus, defined (' 'in part) by this key.') index = Int(default=None, allow_none=True, help='CSD is calculated from a specific stimulus, defined (' 'in part) by this index.')
class ExtractFromNpxParams(DefaultSchema): npx_directory = String(help='Path to NPX file(s) saved by Open Ephys') settings_xml = String(help='Path to settings.xml file saved by Open Ephys') npx_extractor_executable = String( help='Path to .exe file for NPX extraction (Windows only)') npx_extractor_repo = String( required=False, default='None', help='Path to local repository for NPX extractor')
class S3LandingBucket(DefaultSchema): name = String(description="s3 landing bucket name or access point arn", required=True) region = String(description="s3 landing bucket's region", required=True) credentials_file = String( description= "the INI config file path to the s3 landing bucket's credentials", required=False)
class InputParameters(ArgSchema): opto_pickle_path = String( required=True, help='path to file containing optotagging information') sync_h5_path = String( required=True, help='path to h5 file containing syncronization information') output_opto_table_path = String( required=True, help='the optotagging stimulation table will be written here') conditions = Dict(String, Nested(Condition), default=known_conditions)
class EphysParams(DefaultSchema): sample_rate = Float(required=True, default=30000.0, help='Sample rate of Neuropixels AP band continuous data') lfp_sample_rate = Float(require=True, default=2500.0, help='Sample rate of Neuropixels LFP band continuous data') bit_volts = Float(required=True, default=0.195, help='Scalar required to convert int16 values into microvolts') num_channels = Int(required=True, default=384, help='Total number of channels in binary data files') reference_channels = NumpyArray(required=False, default=[36, 75, 112, 151, 188, 227, 264, 303, 340, 379], help='Reference channels on Neuropixels probe (numbering starts at 0)') template_zero_padding = Int(required=True, default=21, help='Zero-padding on templates output by Kilosort') vertical_site_spacing = Float(required=False, default=20e-6, help='Vertical site spacing in meters') probe_type = String(required=False, default='3A', help='3A, 3B1, or 3B2') lfp_band_file = String(required=False, help='Location of LFP band binary file') ap_band_file = String(required=False, help='Location of AP band binary file')
class OutputSchema(RaisingSchema): neuropil_trace_file = String( required=True, description='path to output h5 file containing neuropil traces' ) # TODO rename these to _path roi_trace_file = String( required=True, description='path to output h5 file containing roi traces') exclusion_labels = Nested( ExclusionLabel, many=True, description='a report of roi-wise problems detected during extraction')
class InputParameters(ArgSchema): class Meta: unknown = RAISE log_level = LogLevel(default='INFO', description="set the logging level of the module") case = String(required=True, validate=lambda s: s in VALID_CASES, help='select a use case to run') sub_images = Nested(SubImage, required=True, many=True, help='Sub images composing this image series') affine_params = List( Float, help='Parameters of affine image stack to reference space transform.') deformation_field_path = String( required=True, help= 'Path to parameters of the deformable local transform from affine-transformed image stack to reference space transform.' ) image_series_slice_spacing = Float( required=True, help='Distance (microns) between successive images in this series.') target_spacings = List( Float, required=True, help='For each volume produced, downsample to this isometric resolution' ) reference_spacing = Nested( ReferenceSpacing, required=True, help='Native spacing of reference space (microns).') reference_dimensions = Nested(ReferenceDimensions, required=True, help='Native dimensions of reference space.') sub_image_count = Int(required=True, help='Expected number of sub images') grid_prefix = String(required=True, help='Write output grid files here') accumulator_prefix = String( required=True, help='If this run produces accumulators, write them here.') storage_directory = String( required=False, help='Storage directory for this image series. Not used') filter_bit = Int( default=None, allow_none=True, help= 'if provided, signals that pixels with this bit high have passed the optional post-filter stage' ) nprocesses = Int(default=8, help='spawn this many worker subprocesses') reduce_level = Int( default=0, help='power of two by which to downsample each input axis')
class SweepFeatures(DefaultSchema): stimulus_code = String(description="stimulus code", required=True) stimulus_name = String( description="index of sweep in order of presentation", required=True) stimulus_amplitude = Float(description="amplitude of stimulus", required=True, allow_none=True) sweep_number = Integer( description="index of sweep in order of presentation", required=True) stimulus_units = String(desription="stimulus units", required=True) bridge_balance_mohm = Float(description="bridge balance", allow_none=True) pre_vm_mv = Float(allow_none=True) leak_pa = Float(allow_none=True)
class FileToCopy(RaisingSchema): source = String(required=True, validate=check_read_access, description='copy from here') destination = String( required=True, validate=check_write_access, description='copy to here (full path, not just directory!)') key = String( required=True, description= 'will be passed through to outputs, allowing a name or kind to be associated with this file' )
class InputParameters(ArgSchema): drifting_gratings = Nested(DriftingGratings) static_gratings = Nested(StaticGratings) natural_scenes = Nested(NaturalScenes) # natural_movies = Nested(NaturalMovies) dot_motion = Nested(DotMotion) # contrast_tuning = Nested(ContrastTuning) flashes = Nested(Flashes) receptive_field_mapping = Nested(ReceptiveFieldMapping) input_session_nwb = String(required=True, help='Ecephys spiking nwb file for session') output_file = String(required=True, help='Location for saving output file')
class db_params(DefaultSchema): owner = String(default='', required=False, description='render or mongo owner') project = String(default='', required=False, description='render or mongo project') name = List(String, cli_as_single_argument=True, required=False, many=True, description='render or mongo collection name') host = String(required=False, description='render host') port = Int(default=8080, required=False, description='render port') mongo_host = String(default='em-131fs', required=False, description='mongodb host') mongo_port = Int(default=27017, required=False, description='mongodb port') mongo_userName = String(default='', required=False, description='mongo user name') mongo_authenticationDatabase = String(default='', required=False, description='mongo admin db') mongo_password = String(default='', required=False, description='mongo pwd') db_interface = String(default='mongo', validator=mm.validate.OneOf( ['render', 'mongo', 'file']), description=("render: read or write via render\n" "mongo: read or write via pymongo\n" "file: read or write to file")) client_scripts = String( default=("/allen/aibs/pipeline/image_processing/" "volume_assembly/render-jars/production/scripts"), required=False, description='see renderapi.render.RenderClient') memGB = String(required=False, default='5G', description='see renderapi.render.RenderClient') validate_client = Boolean(required=False, default=False, description='see renderapi.render.RenderClient') @mm.pre_load def tolist(self, data): if 'name' in data: if not isinstance(data['name'], list): data['name'] = [data['name']]
class db_params(ArgSchema): owner = String( default='', required=False, description='owner') project = String( default='', required=False, description='project') name = List( String, cli_as_single_argument=True, required=True, many=True, description='stack name') host = String( default=None, required=False, description='render host') port = Int( default=8080, required=False, description='render port') mongo_host = String( default='em-131fs', required=False, description='mongodb host') mongo_port = Int( default=27017, required=False, description='mongodb port') mongo_userName = String( default='', required=False, description='mongo user name') mongo_authenticationDatabase = String( default='', required=False, description='mongo admin db') mongo_password = String( default='', required=False, description='mongo pwd') db_interface = String( default='mongo') client_scripts = String( default=("/allen/aibs/pipeline/image_processing/" "volume_assembly/render-jars/production/scripts"), required=False, description='render bin path') @pre_load def tolist(self, data): if not isinstance(data['name'], list): data['name'] = [data['name']]
class InputParameters(ArgSchema): destination_bucket = Nested( S3LandingBucket, description= "s3 landing bucket info (bucket_name/access_point_arn and region)", required=True) neuron_reconstruction_id = Int( description="neuron reconstruction id", required=True, ) specimen_id = Int(description="specimen id", required=True) primary_boundaries = Nested(PrimaryBoundaries, description="primary boundaries", required=True) swc_file = String(description="path to input swc (csv) file", required=True) cell_depth = Float(description="cell depth", required=True, allow_none=True) cut_thickness = Float(description="cut thickness", required=True, allow_none=True) marker_file = String(description="path to input marker (csv) file", required=True) ccf_soma_xyz = List(Float, cli_as_single_argument=True, description="soma location (x,y,z) coordinates in CCF", required=True) slice_transform = List( Float, cli_as_single_argument=True, description='List defining the transform defining slice cut angle', required=True, allow_none=True, ) slice_image_flip = Boolean(description=( 'indicates whether the image was flipped relative ' 'to the slice (avg_group_label.name = \'Flip Slice Indicator\''), required=True)
class CatGTParams(DefaultSchema): run_name = String(required=True, help='undecorated run name (no g or t indices') gate_string = String(required=True, default='0', help='gate string') trigger_string = String( required=True, default='0,0', help='string specifying trials to concatenate, e.g. 0,200') probe_string = String(required=True, default='0', help='string specifying probes, e.g. 0:3') stream_string = String(required=True, default='-ap', help='string specifying which streams to process') car_mode = String( required=False, default='None', help='Comaon average reference mode. Must = None, gbldmx, or loccar ') loccar_inner = Int(required=False, default=2, help='Inner radius for loccar in sites') loccar_outer = Int(required=False, default=8, help='Outer radius for loccar in sites') cmdStr = String( required=True, default='-prbfld -aphipass=300 -gbldmx -gfix=0.40,0.10,0.02', help='input stream filter, error correct and extract settings for CatGT' ) extract_string = String(required=True, default='', help='extract edges from datastreams') catGTPath = InputDir(help='directory containing the CatGT executable.')
class ProbeMappable(DefaultSchema): name = String( required=True, help='What kind of mappable data is this? e.g. "spike_timestamps"', ) input_path = String( required=True, help= "Input path for this file. Should point to a file containing a 1D timestamps array with values in probe samples.", ) output_path = String( required=True, help= "Output path for the mapped version of this file. Will write a 1D timestamps array with values in seconds on the master clock.", )
class output_stack(db_params): output_file = OutputFile( required=False, missing=None, default=None, description=("json or json.gz serialization of input stack" "ResolvedTiles.")) compress_output = Boolean( required=False, default=True, missing=True, description=("if writing file, compress with gzip.")) collection_type = String(default='stack', description="'stack' or 'pointmatch'") use_rest = Boolean( default=False, description=("passed as kwarg to " "renderapi.client.import_tilespecs_parallel")) @mm.post_load def validate_file(self, data): if data['db_interface'] == 'file': if data['output_file'] is None: raise mm.ValidationError("with db_interface 'file', " "'output_file' must be a file") @mm.post_load def validate_data(self, data): if 'name' in data: if len(data['name']) != 1: raise mm.ValidationError("only one input or output " "stack name is allowed")