def save_posn(self):
        Save data for the current position.


        # Update the edited file

        # Update the file of saved traces, keeping only relevant variables
        df =[['Saved'] == True]
        if len(df) > 0:
            d = {k : list(df[k]) for k in ('Trace', 'Divns', 'Mother')}

            # Restructure the labels into a DataFrame commensurate with data
            num_saved = len(df)
            num_frames = self.num_frames
            frames = range(num_frames) * num_saved
            traces = []
            for v in xrange(num_saved):
                traces.extend([v] * num_frames)
            index = MultiIndex.from_arrays([traces, frames],
                                           names=('Trace', 'Frame'))
            data = np.hstack(df['Label'].values)
            s = DataFrame(data, index=index, columns=('Label', ))['Label']

            # Remove frames that one should not keep
            d['Label'] = s[np.hstack(df['Keep'].values) & (data > 0)]

            # Save the times directly as arrays
            d['TimeP'] = self.time_phase
            if hasattr(self, 'time_fluor'):
                d['TimeF'] = self.time_fluor

            # Make generations data from divisons
            d['Divns'] = [np.asarray(v) for v in d['Divns']]
            d['Gens'] = [[] for _ in d['Divns']]
            d['Taus'] = [[] for _ in d['Divns']]
            for i, v in enumerate(d['Divns']):
                v += 1
                for j1, j2 in zip(v[:-1], v[1:]):
                    d['Gens'][i].append(slice(j1, j2))
                    t1 = np.mean(d['TimeP'][j1-1:j1+1])
                    t2 = np.mean(d['TimeP'][j2-1:j2+1])
                    d['Taus'][i].append(t2 - t1)
            d['Taus'] = [np.asarray(v) for v in d['Taus']]

            with open(os.path.join(self.posn_dir, 'saved.pickle'), 'wb') as f:
                pickle.dump(d, f)

        # Delete unzipped directories
        b = os.path.join(self.posn_dir, 'blocks')
        for v in read.listdirs(b, read.PATTERN['blockdir']):
            shutil.rmtree(os.path.join(b, v, 'PhaseSegment'))

        # Update the log file
        read.updatelog(self.expt_name, self.posns[self.posn_idx],
                       'edit', self.analyses_dir)
def posteditmovie(expt_raw_data_dir, expt_analyses_dir, positions, params):
    Automated, post-editing analysis.

    expt = os.path.basename(expt_analyses_dir)

    # Execute each position in succession
    for p in positions:
        # Update the terminal display
        read.updatelog(expt, p, 'postedit')

        # Names of blocks directories containing each frame range
        posn_analyses_dir = os.path.join(expt_analyses_dir, p)
        block_dirs = [os.path.join(posn_analyses_dir, 'blocks', v) for v in
                      os.listdir(os.path.join(posn_analyses_dir, 'blocks'))
                      if 'frame' in v]

        # Track the blocks in parallel
        args = [(v, params['general']) for v in block_dirs]
        parallel.main(posteditblock, args, params['general']['num_procs'])

        # Update the experiment log file
        read.updatelog(expt, p, 'postedit', expt_analyses_dir)
def preeditmovie(expt_raw_data_dir, expt_analyses_dir, positions, params):
    Automated steps to perform prior to editing.

    expt = os.path.basename(expt_analyses_dir)
    g = params['general']

    # First load or create log files for each position
    log.main(expt_raw_data_dir, expt_analyses_dir, positions, g['write_mode'])

    # Execute each position in succession
    for p in positions:
        # Update the terminal display
        read.updatelog(expt, p, 'preedit')
        print 'start position ' + p + ': ' + time.asctime()

        posn_raw_data_dir = os.path.join(expt_raw_data_dir, p)
        posn_analyses_dir = os.path.join(expt_analyses_dir, p)

        # Segmented files will be saved to a temporary directory
        temp_dir = os.path.join(posn_analyses_dir, 'temp')
        if g['write_mode'] == 0:

        # Pad with default parameters, and find frames to process
        frame_start, frame_stop = float('inf'), 0.
        for mode in MODES:
            print '---mode', mode
            d = params[mode]

            # Pad with default parameters as necessary
            d = eval('%s.workflow.fillparams(d)' % mode)

            # Find all .tif images of specified type in the given directory
            d['segment']['file_list'] = []
            for f in read.listfiles(posn_raw_data_dir, d['segment']['pattern']):
                j = read.getframenum(f, d['segment']['pattern'])
                if g['frame_range'][0] <= j < g['frame_range'][1]:
                    frame_start = min(frame_start, j)
                    frame_stop = max(frame_stop, j)
            frame_stop += 1

        # Create arguments for parallel processing
        args = [(posn_raw_data_dir, temp_dir,
                 MODES, copy.deepcopy(params)) for _ in range(g['num_procs'])]
        file_list = sorted(args[0][3]['phase']['segment']['file_list'])

        # # debug: select only a few files -BK
        # print 'initial frame stop', frame_stop
        # frame_stop = 500
        # file_list = file_list[:frame_stop]
        # # debug: select only a few files -BK

        inds = partition_indices(file_list, g['num_procs'])
        for (sta_ind, end_ind), arg in zip(inds, args):
            arg[3]['phase']['segment']['file_list'] = file_list[sta_ind:end_ind]

        # Process each block of frames in parallel
        parallel.main(preeditblock, args, g['num_procs'])
        print 'extract: ' + time.asctime()

        # Archive the output files into .zip files, then delete each .tif
        num_tifs = frame_stop - frame_start
        num_digits = int(np.ceil(np.log10(num_tifs + 1)))

        # Create new set of directories with pre-specified block size
        frames = range(frame_start, frame_stop-1, g['block_size'])
        block_frames = zip(frames[:-1], frames[1:])

        # Make directories to hold files, named according to frames
        read.cmkdir(os.path.join(posn_analyses_dir, 'blocks'))
        block_dirs = []
        for j1, j2 in block_frames:
            strs = [str(v).zfill(num_digits) for v in (j1, j2)]
            v = os.path.join(posn_analyses_dir, 'blocks',

        for m in MODES:
            # The segmented .tif files will be stored in a .zip file
            zip_name = m.capitalize() + 'Segment'
            [read.cmkdir(os.path.join(v, zip_name)) for v in block_dirs]

            # Find all segmented .tif images and transfer to the new directories
            d = params[m]
            for f in read.listfiles(temp_dir, d['segment']['pattern']):
                j = read.getframenum(f, d['segment']['pattern'])
                for i, (j1, j2) in enumerate(block_frames):
                    if j1 <= j < j2:
                        old_name = os.path.join(temp_dir, f)
                        zip_dir = os.path.join(block_dirs[i], zip_name)
                        shutil.move(old_name, zip_dir)

            # Zip each directory of segmented .tif files
            old_dir = os.path.abspath(os.curdir)
            for v in block_dirs:
                archive_util.make_zipfile(zip_name, zip_name)

            # Make temporary directories for data outputs
            dat_name = m.capitalize() + 'Data'
            [read.cmkdir(os.path.join(v, dat_name)) for v in block_dirs]

            # Find all analyzed .pickle files and transfer to the new directories
            f, e = os.path.splitext(d['segment']['pattern'])
            dat_pattern = (f + '.pickle' + e[4:])
            for f in read.listfiles(temp_dir, dat_pattern):
                j = read.getframenum(f, dat_pattern)
                for i, (j1, j2) in enumerate(block_frames):
                    if j1 <= j < j2:
                        # Transfer each frame to the correct block
                        old_name = os.path.join(temp_dir, f)
                        dat_dir = os.path.join(block_dirs[i], dat_name)
                        shutil.move(old_name, dat_dir)

            # Concatenate each set of files into a DataFrame for each parameter
            for block_dir in block_dirs:
                dat_dir = os.path.join(block_dir, dat_name)
                data = []
                for u in os.listdir(dat_dir):
                    dat_file = os.path.join(dat_dir, u)
                        d = read_pickle(dat_file)
                df = concat(data)
                df = df.reindex(sorted(df.index))
                for c in df.columns:
                    df[c].to_pickle(os.path.join(block_dir, c + '.pickle'))
        print 'shuffle: ' + time.asctime()

        # Delete all temporary files
        block_dirs = [os.path.join(posn_analyses_dir, 'blocks', v) for v in
                      os.listdir(os.path.join(posn_analyses_dir, 'blocks'))
                      if 'frame' in v]
        # Track the blocks in parallel
        args = []
        for v in block_dirs:
            output_file = os.path.join(v, 'Trace.pickle')
            if os.path.isfile(output_file):
            args.append((v, output_file, params['phase']['track']))
        parallel.main(trackblock, args, g['num_procs'])
        print 'track: ' + time.asctime()

        # Stitch independently-tracked trajectories together
        stitchblocks(block_dirs, params['phase']['track'])
        print 'stitch: ' + time.asctime()

        # Collate the data for manual editing
        output_file = os.path.join(posn_analyses_dir, 'edits.pickle')
        collateblocks(block_dirs, output_file, params['phase']['collate'])
        print 'collate: ' + time.asctime()

        # Update the experiment log file
        read.updatelog(expt, p, 'preedit', expt_analyses_dir)
        print 'final: ' + time.asctime()