예제 #1
0
    def run(self):
        self.logger.debug('Mipmap generation module')
        """
        # get the list of z indices
        zvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
                                  self.args['input_stack'])
        zvalues = list(set(self.zValues).intersection(set(zvalues)))
        """
        zvalues = self.get_overlapping_inputstack_zvalues()
        if not zvalues:
            raise RenderModuleException(
                "No sections found for stack {} for specified zs".format(
                    self.args['input_stack']))

        self.logger.debug("Creating mipmaps...")

        mipmap_args = make_tilespecs_and_cmds(
            self.render, self.args['input_stack'], self.args['output_prefix'],
            zvalues, self.args['levels'], self.args['imgformat'],
            self.args['convert_to_8bit'], self.args['force_redo'],
            self.args['pool_size'], self.args['method'])

        self.output({
            "levels": self.args["levels"],
            "output_prefix": self.args["output_prefix"]
        })
    def run(self):
        if self.args['transform_json']:
            with open(self.args['transform_json'], 'r') as f:
                j = json.load(f)
            tfj = RegexDict({
                k: renderapi.transform.AffineModel(json=v)
                for (k, v) in j.items()
            })
        else:
            tfj = RegexDict(
                get_patch_transforms_from_xml(self.args['transform_xml']))
        transform_zs = [int(k.split('_')[0]) for k in tfj]
        z_overlap = self.get_overlapping_inputstack_zvalues(
            zValues=transform_zs)

        new_specs = []
        for z in z_overlap:
            tilespec = renderapi.tilespec.get_tile_specs_from_z(
                self.args['input_stack'], z, render=self.render)
            if len(tilespec) != 1:
                raise RenderModuleException(
                    "expected 1 tilespec for z = %d in stack %s, "
                    "found %d" % (z, self.args['input_stack'], len(tilespec)))
            tilespec[0].tforms = list(tfj.get_matching('%d_*' % z))
            new_specs.append(tilespec[0])

        self.output_tilespecs_to_stack(new_specs)
    def run(self):
        if "MCRROOT" not in os.environ:
            raise ValidationError("MCRROOT not set")

        if self.args['first_section'] >= self.args['last_section']:
            raise RenderModuleException(
                "First section z cannot be greater or equal to last section z")

        zvalues = renderapi.stack.get_z_values_for_stack(
            self.args['source_collection']['stack'], render=self.render)
        self.args['first_section'] = min(
            zvalues) if self.args['first_section'] > min(
                zvalues) else self.args['first_section']
        self.args['last_section'] = max(
            zvalues) if self.args['last_section'] > max(
                zvalues) else self.args['last_section']

        # generate a temporary json to feed in to the solver
        tempjson = tempfile.NamedTemporaryFile(suffix=".json",
                                               mode="w",
                                               delete=False)
        tempjson.close()

        with open(tempjson.name, 'w') as f:
            json.dump(self.args, f, indent=4)
            f.close()

        #assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable
        cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'],
                            tempjson.name)
        ret = os.system(cmd)

        # on successful completion remove the input json file
        if ret == 0:
            os.remove(tempjson.name)
        else:
            raise RenderModuleException("solve failed with input_json {}",
                                        self.args)

        d = {
            'minz': self.args['first_section'],
            'maxz': self.args['last_section']
        }
        self.output(d)
    def run(self):
        # get match collections
        collecs = renderapi.pointmatch.get_matchcollections(
            owner=self.args['match_owner'], render=self.render)

        collections = [c['collectionId']['name'] for c in collecs]

        if (self.args['source_collection']
                not in collections) or (self.args['target_collection']
                                        not in collections):
            raise RenderModuleException(
                "One of source or target collections does not exist")

        # get all groupIds from source and target collections
        source_ids = renderapi.pointmatch.get_match_groupIds(
            self.args['source_collection'],
            owner=self.args['match_owner'],
            render=self.render)

        target_ids = renderapi.pointmatch.get_match_groupIds(
            self.args['target_collection'],
            owner=self.args['match_owner'],
            render=self.render)

        # check existence of zvalues
        ids = [
            s for s in self.args['zValues']
            if "{}.0".format(str(s)) in source_ids
            and "{}.0".format(str(s)) in target_ids
        ]

        # create a temp collection to copy the data
        zvalues = []
        mypartial = partial(swap_pt_matches,
                            self.render,
                            self.args['source_collection'],
                            self.args['target_collection'],
                            match_owner=self.args['match_owner'])

        with renderapi.client.WithPool(self.args['pool_size']) as pool:
            output_bool = pool.map(mypartial, ids)

        zvalues = [z for z, n in zip(ids, output_bool) if n]

        self.output({
            "source_collection":
            self.args['source_collection'],
            "target_collection":
            self.args['target_collection'],
            "swapped_zs":
            zvalues,
            "nonswapped_zs":
            set(self.args['zValues']).difference(zvalues)
        })
    def remove_mask_from_tilespec(self, z):
        tspec = self.get_tilespec(z)
        if not tspec:
            return tspec

        if tspec.ip[0].maskUrl is None:
            raise RenderModuleException(
                "Tilespec does not have a mask for z = %d" % (z))

        tspec.ip[0].maskUrl = None

        return tspec
    def get_tilespec(self, z):
        try:
            tspecs = renderapi.tilespec.get_tile_specs_from_z(
                self.args['stack'], z, render=self.render)
        except RenderError:
            return None

        if len(tspecs) != 1:
            raise RenderModuleException(
                "Expected 1 tilespec for z=%d, found %d" % (z, len(tspecs)))

        return tspecs[0]
예제 #7
0
    def run(self):
        # with open(self.args['metafile'], 'r') as f:
        #     meta = json.load(f)
        meta = json.loads(uri_utils.uri_readbytes(self.args['metafile_uri']))
        roidata = meta[0]['metadata']
        imgdata = meta[1]['data']
        img_coords = {
            img['img_path']: self.image_coords_from_stage(
                img['img_meta']['stage_pos'],
                img['img_meta']['pixel_size_x_move'],
                img['img_meta']['pixel_size_y_move'],
                numpy.radians(img['img_meta']['angle']))
            for img in imgdata
        }

        if not imgdata:
            raise RenderModuleException(
                "No relevant image metadata found for metadata at {}".format(
                    self.args['metafile_uri']))

        minX, minY = numpy.min(numpy.array(list(img_coords.values())), axis=0)
        # assume isotropic pixels
        pixelsize = roidata['calibration']['highmag']['x_nm_per_pix']

        imgdir = self.args.get('image_prefix',
                               uri_utils.uri_prefix(self.args['metafile_uri']))

        tspecs = [
            self.ts_from_imgdata(img,
                                 imgdir,
                                 img_coords[img['img_path']][0] - minX,
                                 img_coords[img['img_path']][1] - minY,
                                 minint=self.args['minimum_intensity'],
                                 maxint=self.args['maximum_intensity'],
                                 width=roidata['camera_info']['width'],
                                 height=roidata['camera_info']['height'],
                                 z=self.zValues[0],
                                 sectionId=self.args.get('sectionId'),
                                 scopeId=roidata['temca_id'],
                                 cameraId=roidata['camera_info']['camera_id'],
                                 pixelsize=pixelsize,
                                 maskUrl=self.args['maskUrl_uri'])
            for img in imgdata
        ]

        self.output_tilespecs_to_stack(tspecs)

        try:
            self.output({'stack': self.output_stack})
        except AttributeError as e:
            self.logger.error(e)
    def add_mask_to_tilespec(self, z, exts):
        tspec = self.get_tilespec(z)
        if not tspec:
            return tspec

        if tspec.ip[0].maskUrl is not None:
            raise RenderModuleException(
                "Tilespec already has a mask for z = %d" % (z))

        fnames = []
        for ext in exts:
            fnames += glob.glob(
                os.path.join(self.args['mask_dir'],
                             '%d_*' % z + os.extsep + ext))

        if len(fnames) != 1:
            raise RenderModuleException(
                "Expected 1 mask file for z = %d, found %d in %s" %
                (z, len(fnames), self.args['mask_dir']))

        tspec.ip['0'].maskUrl = pathlib.Path(fnames[0]).as_uri()

        return tspec
예제 #9
0
def dereference_tforms(tforms, ref_tforms):
    deref_tforms = []
    for tf in tforms:
        if isinstance(tf, ReferenceTransform):
            try:
                mtf = next(mt for mt in ref_tforms
                           if mt.transformId == tf.refId)
                deref_tforms.append(mtf)
            except StopIteration as e:
                raise RenderModuleException((
                    "reference transform: {} not found in provided refererence transforms {}"
                    .format(tf.refId, ref_tforms)))
        else:
            deref_tforms.append(tf)
    return deref_tforms
def tspecjob(collection, render, tilespecs, estimate=True):
    result = {}
    result['tilespecs'] = tilespecs

    matches = renderapi.pointmatch.get_matches_from_tile_to_tile(
        collection,
        tilespecs[0].layout.sectionId,
        tilespecs[0].tileId,
        tilespecs[1].layout.sectionId,
        tilespecs[1].tileId,
        render=render)

    if len(matches) != 1:
        estr = "\n  expected 1 matching tile pair, found %d for" % (
            len(matches))
        estr += '\n  %s\n  group: %s\n  %s\n  group: %s' % (
            tilespecs[0].tileId, tilespecs[0].layout.sectionId,
            tilespecs[1].tileId, tilespecs[1].layout.sectionId)
        raise RenderModuleException(estr)

    match = matches[0]

    if estimate:
        tf = renderapi.transform.RigidModel()
        src = np.array(match['matches']['p']).transpose()
        dst = np.array(match['matches']['q']).transpose()
        if match['pGroupId'] == tilespecs[0].layout.sectionId:
            src = np.array(match['matches']['q']).transpose()
            dst = np.array(match['matches']['p']).transpose()

        ind = np.argwhere(np.isclose(np.array(match['matches']['w']),
                                     1.0)).flatten()
        tf.estimate(src[ind, :], dst[ind, :])
        result['transform'] = tf

    result['avg_residual'] = {
        'z0': tilespecs[0].z,
        'z1': tilespecs[1].z,
        'avg_residual': avg_residual(match, tilespecs[0], tilespecs[1])
    }

    return result
예제 #11
0
 def run(self):
     if len(self.zValues) != len(self.args['new_zValues']):
         raise RenderModuleException(
             "zValues with length {} cannot be mapped to "
             "zValues with length {}".format(len(self.zValues),
                                             len(self.args['new_zValues'])))
     for z, newz in zip(self.zValues, self.args['new_zValues']):
         resolvedtiles = renderapi.resolvedtiles.get_resolved_tiles_from_z(
             self.input_stack, z, render=self.render)
         for ts in resolvedtiles.tilespecs:
             ts.z = newz
             if self.args.get('remap_sectionId'):
                 ts.layout.sectionId = self.sectionId_from_z(newz)
         self.output_tilespecs_to_stack(
             resolvedtiles.tilespecs,
             sharedTransforms=resolvedtiles.transforms,
             zValues=[z])
     self.output({
         "zValues": self.zValues,
         "output_stack": self.output_stack
     })
예제 #12
0
    def run(self, run_lc_bsh=None):
        run_lc_bsh = self.default_bsh if run_lc_bsh is None else run_lc_bsh
        outfn = self.args.get(
            'outfile',
            os.path.abspath(
                os.path.join(self.args['project_path'],
                             'lens_correction.json')))

        delete_procdir = False
        procdir = self.args.get('processing_directory',
                                self.args['project_path'])
        if procdir is None:
            delete_procdir = True
            procdir = tempfile.mkdtemp()

        # command line argument for beanshell script
        bsh_call = [
            "xvfb-run", "-a", self.args['fiji_path'], "-Xms{}g".format(
                self.args['heap_size']), "-Xmx{}g".format(
                    self.args['heap_size']), "-Xincgc", "-Dman=" +
            self.args['manifest_path'], "-Ddir=" + self.args['project_path'],
            "-Dgrid=" + str(self.args['grid_size']),
            "-Disig=" + str(self.args['SIFT_params']['initialSigma']),
            "-Dsteps=" + str(self.args['SIFT_params']['steps']),
            "-Dminos=" + str(self.args['SIFT_params']['minOctaveSize']),
            "-Dmaxos=" + str(self.args['SIFT_params']['maxOctaveSize']),
            "-Dfdsize=" + str(self.args['SIFT_params']['fdSize']),
            "-Dfdbins=" + str(self.args['SIFT_params']['fdBins']),
            "-Drod=" + str(self.args['align_params']['rod']),
            "-Dmaxe=" + str(self.args['align_params']['maxEpsilon']),
            "-Dmir=" + str(self.args['align_params']['minInlierRatio']),
            "-Dmni=" + str(self.args['align_params']['minNumInliers']),
            "-Demi=" + str(self.args['align_params']['expectedModelIndex']),
            "-Dmh=" + str(self.args['align_params']['multipleHypotheses']),
            "-Dri=" + str(self.args['align_params']['rejectIdentity']),
            "-Dit=" + str(self.args['align_params']['identityTolerance']),
            "-Dtaip=" + str(self.args['align_params']['tilesAreInPlace']),
            "-Ddmi=" + str(self.args['align_params']['desiredModelIndex']),
            "-Dreg=" + str(self.args['align_params']['regularize']),
            "-Dmio={}".format(
                self.args['align_params']['maxIterationsOptimize']),
            "-Dmpwo={}".format(
                self.args['align_params']['maxPlateauWidthOptimize']),
            "-Ddim=" + str(self.args['align_params']['dimension']),
            "-Dlam=" + str(self.args['align_params']['lambdaVal']),
            "-Dprocdir={}".format(procdir), "-Doutfn={}".format(outfn),
            "-Dctrans=" + str(self.args['align_params']['clearTransform']),
            "-Dvis=" + str(self.args['align_params']['visualize']),
            "-DthreadsSIFT={}".format(
                self.args['max_threads_SIFT']), "--", "--no-splash", run_lc_bsh
        ]

        self.logger.debug('bsh_cmd: {}'.format(bsh_call))
        try:
            subprocess.check_call(bsh_call)
        except subprocess.CalledProcessError as e:
            raise (RenderModuleException("{}".format(e)))

        # self.logger.debug('ret_code: {}'.format(ret))
        if delete_procdir:
            shutil.rmtree(procdir)

        try:
            self.output({'output_json': outfn})
        except AttributeError as e:
            # output validation will need to wait for argschema PR
            self.logger.error(e)
    def run(self):
        if "MCRROOT" not in os.environ:
            raise ValidationError("MCRROOT not set")
        if self.clone_section_stack:
            tmp_stack = "{}_zs{}_ze{}_t{}".format(
                self.args['source_collection']['stack'],
                self.args['first_section'], self.args['last_section'],
                time.strftime("%m%d%y_%H%M%S"))
            zvalues = renderapi.stack.get_z_values_for_stack(
                self.args['source_collection']['stack'], render=self.render)
            zs = [z for z in zvalues if \
                (z>=self.args['first_section']) \
                and (z<=self.args['last_section'])]
            renderapi.stack.clone_stack(
                self.args['source_collection']['stack'],
                tmp_stack,
                zs=zs,
                close_stack=True,
                render=self.render)
            self.args['source_collection']['stack'] = tmp_stack

            # the clone stack option should also set the first and last section z appropriately in self.args for solver
            self.args['first_section'] = min(zs)
            self.args['last_section'] = max(zs)

        # Check if the target stack exists and whether we need to overwrite the z
        target_host = self.args['target_collection']['service_host']
        num = target_host[-4:]
        port = None
        if num.isdigit():
            target_host = target_host[:-5]
            if target_host.find('http://') < 0:
                target_host = 'http://%s' % (target_host)
            port = int(num)

        list_of_stacks = renderapi.render.get_stacks_by_owner_project(
            owner=self.args['target_collection']['owner'],
            project=self.args['target_collection']['project'],
            host=target_host,
            port=port,
            render=self.render)

        if self.args['target_collection']['stack'] in list_of_stacks:
            for z in xrange(int(self.args['first_section']),
                            int(self.args['last_section']) + 1):
                renderapi.stack.delete_section(
                    self.args['target_collection']['stack'],
                    z,
                    host=target_host,
                    port=port,
                    owner=self.args['target_collection']['owner'],
                    project=self.args['target_collection']['project'],
                    render=self.render)

        # generate a temporary json to feed in to the solver
        tempjson = tempfile.NamedTemporaryFile(suffix=".json",
                                               mode='w',
                                               delete=False)
        tempjson.close()

        with open(tempjson.name, 'w') as f:
            json.dump(self.args, f, indent=4)
            f.close()

        #assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable
        cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'],
                            tempjson.name)
        ret = os.system(cmd)

        # one successful completion remove the input json file
        if ret == 0:
            os.remove(tempjson.name)
        else:
            raise RenderModuleException("solve failed with input_json {}",
                                        self.args)

        #try to return the z's solved in the output json
        try:
            d = {'zs': zs}
            self.output(d)
        except:
            raise RenderModuleException("unable to output json {}", d)

        #if you made a tmp stack destroy it
        if self.clone_section_stack:
            renderapi.stack.delete_stack(tmp_stack, render=self.render)
예제 #14
0
    def run(self):
        if len(self.args['source_stack']) != len(self.args['target_stack']):
            raise RenderModuleException(
                "Count of source and target stacks do not match")

        list_of_stacks = renderapi.render.get_stacks_by_owner_project(
            owner=self.render.DEFAULT_OWNER,
            project=self.render.DEFAULT_PROJECT,
            host=self.render.DEFAULT_HOST,
            port=self.render.DEFAULT_PORT,
            render=self.render)

        source_stacks = []
        target_stacks = []
        zvalues = []
        for source_stack, target_stack, zVals in zip(self.args['source_stack'],
                                                     self.args['target_stack'],
                                                     self.args['zValues']):
            if source_stack not in list_of_stacks or target_stack not in list_of_stacks:  # pragma: no cover
                continue
            #if source_stack not in list_of_stacks:
            #    raise RenderModuleException("Source stack {} not in render".format(source_stack))

            #if target_stack not in list_of_stacks:
            #    raise RenderModuleException("Target stack {} not in render".format(target_stack))

            szvalues = renderapi.stack.get_z_values_for_stack(
                source_stack, render=self.render)
            tzvalues = renderapi.stack.get_z_values_for_stack(
                target_stack, render=self.render)
            avail_zs = set(szvalues).intersection(tzvalues)
            final_zs = list(set(zVals).intersection(avail_zs))

            #source_stacks.append(source_stack)
            #target_stacks.append(target_stack)
            #zvalues.append(final_zs)
            print(final_zs)
            #mypartial = partial(swap_section, self.render, source_stack, target_stack)

            #with renderapi.client.WithPool(self.args['pool_size']) as pool:
            #    output_bool = pool.map(mypartial, final_zs)

            output_bool = []
            for z in final_zs:
                outbool = swap_section(self.render, source_stack, target_stack,
                                       z)
                output_bool.append(outbool)

            for z, n in zip(final_zs, output_bool):
                if n:
                    zvalues.append(z)
                    source_stacks.append(source_stack)
                    target_stacks.append(target_stack)

            #zs = [z for z, n in zip(final_zs, output_bool) if n]
            #zvalues.append(zs)

            if self.args['complete_source_stack']:
                renderapi.stack.set_stack_state(source_stack,
                                                state='COMPLETE',
                                                render=self.render)

            if self.args['complete_target_stack']:
                renderapi.stack.set_stack_state(target_stack,
                                                state='COMPLETE',
                                                render=self.render)

            # delete source stack if specified in input
            if self.args['delete_source_stack']:  # pragma: no cover
                renderapi.stack.delete_stack(source_stack, render=self.render)

            #self.swap_zs_from_stacks(source_stack, target_stack, final_zs)

        self.output({
            "source_stacks": source_stacks,
            "target_stacks": target_stacks,
            "swapped_zvalues": zvalues
        })
    def run(self):
        zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack,
                                self.args['poststitched_stack'])
        zrange = range(self.args['minZ'], self.args['maxZ']+1)
        zvalues = list(set(zvalues1).intersection(set(zrange)))
        if len(zvalues) == 0:
            raise RenderModuleException('No valid zvalues found in stack for given range {} - {}'.format(self.args['minZ'], self.args['maxZ']))

        # check if pre or post stitched stack is in LOADING state.
        # if so, clone them to new stacks
        status1, new_prestitched = check_status_of_stack(self.render,
                                                self.args['prestitched_stack'],
                                                zvalues)

        status2, new_poststitched = check_status_of_stack(self.render,
                                                 self.args['poststitched_stack'],
                                                 zvalues)
        disconnected_tiles, gap_tiles, seam_centroids, post_tspecs, matches, stats = detect_stitching_mistakes(
                                                                                    self.render,
                                                                                    new_prestitched,
                                                                                    new_poststitched,
                                                                                    self.args['match_collection'],
                                                                                    self.args['match_collection_owner'],
                                                                                    self.args['residual_threshold'],
                                                                                    self.args['neighbors_distance'],
                                                                                    self.args['min_cluster_size'],
                                                                                    zvalues,
                                                                                    pool_size=self.args['pool_size'])
        
        # find the indices of sections having holes
        hole_indices = [i for i, dt in enumerate(disconnected_tiles) if len(dt) > 0]
        gaps_indices = [i for i, gt in enumerate(gap_tiles) if len(gt) > 0]
        seams_indices = [i for i, sm in enumerate(seam_centroids) if len(sm) > 0]
        holes = [zvalues[i] for i in hole_indices]
        gaps = [zvalues[i] for i in gaps_indices]
        seams = [zvalues[i] for i in seams_indices]
        combinedz = list(set(holes + gaps + seams))
        qc_passed_sections = set(zvalues) - set(combinedz)
        centroids = [seam_centroids[i] for i in seams_indices]

        self.args['output_html'] = []
        if self.args['plot_sections']:
            self.args['output_html'] = plot_section_maps(self.render, 
                                                         self.args['poststitched_stack'], 
                                                         post_tspecs, 
                                                         matches,
                                                         disconnected_tiles, 
                                                         gap_tiles, 
                                                         seam_centroids,
                                                         stats, 
                                                         zvalues, 
                                                         out_html_dir=self.args['out_html_dir'])

        self.output({'output_html':self.args['output_html'],
                     'qc_passed_sections': qc_passed_sections,
                     'hole_sections': holes,
                     'gap_sections':gaps,
                     'seam_sections':seams,
                     'seam_centroids':np.array(centroids)})

        # delete the stacks that were cloned
        if status1 == 'LOADING':
            self.render.run(renderapi.stack.delete_stack, new_prestitched)

        if status2 == 'LOADING':
            self.render.run(renderapi.stack.delete_stack, new_poststitched)