Example #1
0
    def plot_spectrum(self, colors = None):
        """
        params:
            `colors`: any interable of symbols of color `supported` by matplotlib
        """

        if colors is None:
            colors = _cycle(["#FF6600", "#0033CC"])
        else:
            colors = _cycle(colors)

        N = len(self.audio_data.data) if self.audio_data.CHANNELS == 1 else len(self.audio_data.data[0])

        if self.__cached_fft is None:
            self.fft()

        coefs = self.__cached_fft

        fig, axs = _plt.subplots(len(coefs) + 1, 1, figsize = (20, 7))
        zoom_ax = axs[-1]
        zoom_ax.set_axis_bgcolor("#E6E6B8")
        zoom_ax.title.set_text("Selected Segment")
        zoom_line, = zoom_ax.plot([], [], "#008A2E")

        for i in range(len(coefs)):
            coef = coefs[i]
            y = _np.abs(coef[1:N/2])
            x = _np.arange(len(y))
            color = colors.next()
            ax = axs[i]

            ax.set_axis_bgcolor("#E6E6B8")
            ax.title.set_text("Channel {}".format(i+1))
            ax.plot(x, y, color)

            def onselect_callback(xmin, xmax):

                indmin, indmax = _np.searchsorted(x, (xmin, xmax))
                indmax = min(len(y) - 1, indmax)

                x_segment = x[indmin:indmax]
                y_segment = y[indmin:indmax]
                zoom_line.set_data(x_segment, y_segment)
                zoom_ax.set_xlim(x_segment[0], x_segment[-1])
                zoom_ax.set_ylim(y_segment.min(), y_segment.max())
                zoom_ax.title.set_text("Selected Segment: {} to {}".format(indmin, indmax))
                
                _plt.draw()

            span_selector = _SpanSelector(ax, onselect_callback, 'horizontal', span_stays = True, 
                                          rectprops = dict(alpha = 0.5, facecolor = 'cyan'))
            self.__current_widgets["span_selectors"].append(span_selector)

        self.__current_figs.append(fig)
        fig.subplots_adjust(top = 0.9, hspace = 0.3)
Example #2
0
def _dot(A, B):
    """Dot product."""
    M, N = A.msize[0], B.msize[1]
    MN = M * N
    na = _marray(_typegreater(A.dtype, B.dtype), (M, N))
    cols = (_islice(B._a, i * M, (i + 1) * M) for i in _cycle(xrange(N)))
    rows = _cycle(_islice(A._a, i, MN + i, M) for i in xrange(M))
    # fill in the result in the FORTRAN order
    for i in xrange(MN):
        col = cols.next()
        for j in xrange(N):
            s = sum(a * b for a, b in _izip(rows.next(), col))
            na._a[i] = s
    return na
Example #3
0
def _dot(A, B):
    """Dot product."""
    M, N = A.msize[0], B.msize[1]
    MN = M*N
    na = _marray(_typegreater(A.dtype, B.dtype), (M, N))
    cols = (_islice(B._a, i*M, (i+1)*M) for i in _cycle(xrange(N)))
    rows = _cycle( _islice(A._a, i, MN+i, M) for i in xrange(M) )
    # fill in the result in the FORTRAN order
    for i in xrange(MN):
        col = cols.next()
        for j in xrange(N):
            s = sum( a*b for a, b in _izip(rows.next(), col) )
            na._a[i] = s
    return na
Example #4
0
def range_step(start, stop=None, step=None):
    """Like range but step is an iterator. If iterator is exhausted, the last
    value will be use as the step."""
    if stop is None:
        stop = start
        start = 0
    if step is None:
        step = _cycle([1])

    current = start
    while current < stop:
        yield current
        try:
            current_step = next(step)
        except StopIteration:
            step = _cycle(current_step)
        current += current_step
Example #5
0
def plot_surface_area(surface_area, fig_fn, timeunit=None, fig_close=False):
    """Plot surface area as a function of time.

    Parameters
    ----------
    surface_area : pandas.DataFrame
    save_dir : str
    timeunit : str or None, optional, default=None

    See also
    ---------
    pylipid.func.calculate_surface_area
        The function that generates surface_area data.
    pylipid.plot.plot_surface_area_stats
        The function that plot surface_area in a matplotlib violin plot.

    """
    from itertools import cycle as _cycle

    color_set = _cycle(plt.get_cmap("tab10").colors)
    plt.rcParams["font.size"] = 10
    plt.rcParams["font.weight"] = "normal"

    if timeunit is None:
        timeunit = ""
    elif timeunit == "ns":
        timeunit = " (ns)"
    elif timeunit == "us":
        timeunit = r" ($\mu$s)"

    row_set = list(set([ind[:2] for ind in surface_area.index]))
    row_set.sort()
    col_set = [col for col in surface_area.columns if col != "Time"]
    colors = [next(color_set) for dummy in col_set]
    fig, axes = plt.subplots(len(row_set), len(col_set), figsize=(len(col_set)*2.4, len(row_set)*1.6),
                             sharex=True, sharey=True)
    plt.subplots_adjust(wspace=0.2, hspace=0.16)
    if len(col_set) == 1:
        axes = axes[:, np.newaxis]
    for row_idx, row in enumerate(row_set):
        df = surface_area.loc[row]
        for col_idx, bs_name in enumerate(col_set):
            axes[row_idx, col_idx].plot(df["Time"], df[bs_name], color=colors[col_idx],
                                        label="traj {} prot {}".format(row[0], row[1]))
            if row_idx == len(row_set)-1:
                axes[row_idx, col_idx].set_xlabel("Time{}".format(timeunit), fontsize=10)
            if col_idx == 0:
                axes[row_idx, col_idx].set_ylabel(r"Area (nm$^2$)", fontsize=10)
            if row_idx == 0:
                axes[row_idx, col_idx].set_title(bs_name, fontsize=10)
            axes[row_idx, col_idx].legend(loc="best", frameon=False)
    fig.tight_layout()
    fig.savefig(fig_fn, dpi=200)
    if fig_close:
        plt.close()

    return
Example #6
0
 def __ge__(self, other):
     from array import array
     if _isscalar(other):
         if hasattr(other, '__len__'): other = _cycle(other)
         else: other = _repeat(other)
     na = _marray('bool', self.msize, 
                  array(_dtype2array['bool'],
                        (x >= y for x, y in _izip(self, other))))
     return na
Example #7
0
 def __elmul__(self, him):
     from array import array
     if _isscalar(him):
         if hasattr(him, '__len__'): him = _cycle(him)
         else: him = _repeat(him)
     na = _marray(self.dtype, self.msize, 
                  array(_dtype2array[self.dtype], 
                        (x*y for x, y in _izip(self._a, him))))
     return na
Example #8
0
 def __ge__(self, other):
     from array import array
     if _isscalar(other):
         if hasattr(other, '__len__'): other = _cycle(other)
         else: other = _repeat(other)
     na = _marray(
         'bool', self.msize,
         array(_dtype2array['bool'],
               (x >= y for x, y in _izip(self, other))))
     return na
Example #9
0
 def __elmul__(self, him):
     from array import array
     if _isscalar(him):
         if hasattr(him, '__len__'): him = _cycle(him)
         else: him = _repeat(him)
     na = _marray(
         self.dtype, self.msize,
         array(_dtype2array[self.dtype],
               (x * y for x, y in _izip(self._a, him))))
     return na
Example #10
0
 def __setitem1__(self, i, val):
     # determine the size of the new array
     nshp = _ndshape1(self.msize, *i)
     i = ( isinstance(x, _marray) and iter(x._a) or x for x in i )
     ins = list(_ndilin1(self.msize, *i))
     if _isscalar(val):
         if hasattr(val, '__len__'): val = _cycle(val)
         else: val = _repeat(val)
     for j, v in _izip(ins, val):
         self._a[j] = v
Example #11
0
 def __setitem1__(self, i, val):
     # determine the size of the new array
     nshp = _ndshape1(self.msize, *i)
     i = (isinstance(x, _marray) and iter(x._a) or x for x in i)
     ins = list(_ndilin1(self.msize, *i))
     if _isscalar(val):
         if hasattr(val, '__len__'): val = _cycle(val)
         else: val = _repeat(val)
     for j, v in _izip(ins, val):
         self._a[j] = v
Example #12
0
def sieve_of_eratosthenes(upper):
    """Returns a generator of primes < upper through the use of the sieve of
    Eratosthenes."""
    composites = set()
    yield 2
    yield 3
    i = 3
    for j in range(i ** 2, upper + 1, 2 * i):
        if j % 5 != 0:
            composites.add(j)
    yield 5

    for i in range_step(7, upper + 1, _cycle([2, 2, 2, 4])):
        if i not in composites:
            for j in range(i ** 2, upper + 1, 2 * i):
                if j % 5 != 0:
                    composites.add(j)
            yield i
        else:
            composites.remove(i)
Example #13
0
def cycle_thru_list(list_obj, i):
    """
    Cycles through a list
    
    Parameters
    ----------
    list_obj : list
        List to be cycled through
    i : int
        Index of list to be put in the first position
    
    Returns
    -------
    cycled_list : list
        List with index `i` now in the first position
    """
    cycle_obj = _cycle(list_obj)
    L = len(list_obj)
    slice_obj = _islice(cycle_obj, i, i + L)
    cycled_list = list(slice_obj)
    return(cycled_list)   
    def plot(self, by_sec=True, colors=None):
        """
        Plot the audio data.

        params:
            `by_sec`: if True, it will plot the audio in second or it will plot
                      the audio data in the number of frames.
            `colors`: Any iterable of symbols of color supported by matplotlib.
        """
        if colors is None:
            colors = _cycle(["#FF6600", "#0033CC"])
        else:
            colors = _cycle(colors)

        num_subplots = self.audio_data.CHANNELS + 1
        fig, axs = _plt.subplots(num_subplots, 1, figsize=(20, 7))
        fig.suptitle(repr(self.audio_data), fontsize=14, y=1)
        frames = [
            self.audio_data.data[i] for i in range(2)
        ] if self.audio_data.CHANNELS == 2 else [self.audio_data.data]

        if by_sec:
            x = _np.linspace(0,
                             len(frames[0]) / self.audio_data.SAMPLE_RATE,
                             num=len(frames[0]))
            x_label = "Time (Seconds)"
        else:
            x = _np.linspace(0, len(frames[0]), num=len(frames[0]))
            x_label = "Frame"

        zoom_ax = axs[-1]
        zoom_ax.set_axis_bgcolor("#E6E6B8")
        zoom_ax.title.set_text("Selected Segment")
        zoom_ax.xaxis.set_label_text(x_label)
        zoom_ax.yaxis.set_label_text("Altitude")
        zoom_line, = zoom_ax.plot([], [], "#008A2E")

        temp = []
        for i, (ax, frame) in enumerate(zip(axs[:-1], frames)):
            color = colors.next()

            ax.title.set_text("Channel {}".format(i + 1))
            ax.set_axis_bgcolor("#E6E6B8")
            ax.plot(x, frame, color)
            ax.set_xlim(x[0], x[-1])
            ax.xaxis.set_label_text(x_label)
            ax.yaxis.set_label_text("Altitude")

            def onselect_callback(xmin, xmax):

                indmin, indmax = _np.searchsorted(x, (xmin, xmax))
                indmax = min(len(frames[0]) - 1, indmax)

                x_segment = x[indmin:indmax]
                y_segment = frame[indmin:indmax]
                zoom_line.set_data(x_segment, y_segment)
                zoom_ax.set_xlim(x_segment[0], x_segment[-1])
                zoom_ax.set_ylim(y_segment.min(), y_segment.max())

                if by_sec:
                    x_start = int(self.audio_data.duration * float(indmin) /
                                  len(x))
                    x_stop = int(self.audio_data.duration * float(indmax) /
                                 len(x))
                else:
                    x_start = indmin
                    x_stop = indmax

                start_time = self.audio_data.duration * float(indmin) / len(x)
                stop_time = self.audio_data.duration * float(indmax) / len(x)
                zoom_ax.title.set_text("Selected Segment: {} to {}".format(
                    x_start, x_stop))
                _plt.draw()

                self.audio_data.play(start_time, stop_time)

            span_selector = _SpanSelector(ax,
                                          onselect_callback,
                                          'horizontal',
                                          span_stays=True,
                                          rectprops=dict(alpha=0.5,
                                                         facecolor='cyan'))
            self.__current_widgets["span_selectors"].append(span_selector)
        fig.subplots_adjust(top=0.9, hspace=0.7)
        self.__current_figs.append(fig)
Example #15
0
def plot_binding_site_data(data,
                           fig_fn,
                           ylabel=None,
                           title=None,
                           fig_close=False):
    """Plot binding site data in a matplotlib violin plot.

    The provided ``data`` needs to be a pandas.DataFrame object which has "Binding Site {idx}" as its column names and
    records binding site information by column.

    Parameters
    ----------
    data : padnas.DataFrame
        Data to plot. It needs to have "Binding Site {idx}" as its column names and records binding site information by
        column.
    fig_fn : str
        Figure name.
    ylabel : str, optional, default=None
        Y label.
    title : str, optional, default=None
        Figure title.
    fig_close : bool, optional, default=False
        Use plt.close() to close the figure. Can be used to save memory if many figures are opened.

    """
    from itertools import cycle as _cycle

    def adjacent_values(vals, q1, q3):
        upper_adjacent_value = q3 + (q3 - q1) * 1.5
        upper_adjacent_value = np.clip(upper_adjacent_value, q3, vals[-1])
        lower_adjacent_value = q1 - (q3 - q1) * 1.5
        lower_adjacent_value = np.clip(lower_adjacent_value, vals[0], q1)
        return lower_adjacent_value, upper_adjacent_value

    if ylabel is None:
        ylabel = ""
    if title is None:
        title = ""

    color_set = _cycle(plt.get_cmap("tab10").colors)
    plt.rcParams["font.size"] = 10
    plt.rcParams["font.weight"] = "bold"

    BS_names = [col for col in data.columns]
    BS_id_set = [int(name.split()[-1]) for name in BS_names]
    BS_id_set.sort()
    data_processed = [
        np.sort(data["Binding Site {}".format(bs_id)].dropna().tolist())
        for bs_id in BS_id_set
    ]
    colors = [next(color_set) for dummy in BS_id_set]
    fig, ax = plt.subplots(1, 1, figsize=(len(BS_id_set) * 0.6, 2.8))
    plt.subplots_adjust(bottom=0.20, top=0.83)
    ax.set_title(title, fontsize=10, weight="bold")
    parts = ax.violinplot(data_processed,
                          showmeans=False,
                          showmedians=False,
                          showextrema=False)
    for pc_idx, pc in enumerate(parts['bodies']):
        pc.set_facecolors(colors[pc_idx])
        pc.set_edgecolor('black')
        pc.set_alpha(1)

    # deal with the situation in which the columns in data have different lengths.
    quartile1, medians, quartile3 = np.array(
        [np.percentile(d, [25, 50, 75]) for d in data_processed]).T
    whiskers = np.array([
        adjacent_values(sorted_array, q1, q3)
        for sorted_array, q1, q3 in zip(data_processed, quartile1, quartile3)
    ])
    whiskers_min, whiskers_max = whiskers[:, 0], whiskers[:, 1]

    inds = np.arange(1, len(medians) + 1)
    ax.scatter(inds, medians, marker='o', color='white', s=3, zorder=3)
    ax.vlines(inds, quartile1, quartile3, color='k', linestyle='-', lw=5)
    ax.vlines(inds, whiskers_min, whiskers_max, color='k', linestyle='-', lw=1)

    ax.get_xaxis().set_tick_params(direction='out')
    ax.xaxis.set_ticks_position('bottom')
    ax.set_xticks(np.arange(1, len(BS_id_set) + 1))
    ax.set_xticklabels(BS_id_set, fontsize=10, weight="bold")
    ax.set_xlim(0.25, len(BS_id_set) + 0.75)
    ax.set_xlabel('Binding Site', fontsize=10, weight="bold")
    ax.set_ylabel(ylabel, fontsize=10, weight="bold")
    plt.tight_layout()
    fig.savefig(fig_fn, dpi=200)
    if fig_close:
        plt.close()
    return
Example #16
0
 def get_next(self) -> Iterator:
     return _cycle(self.dicts)
    def plot_spectrum(self, colors=None):
        """
        params:
            `colors`: any interable of symbols of color `supported` by matplotlib
        """

        if colors is None:
            colors = _cycle(["#FF6600", "#0033CC"])
        else:
            colors = _cycle(colors)

        N = len(
            self.audio_data.data) if self.audio_data.CHANNELS == 1 else len(
                self.audio_data.data[0])

        if self.__cached_fft is None:
            self.fft()

        coefs = self.__cached_fft

        fig, axs = _plt.subplots(len(coefs) + 1, 1, figsize=(20, 7))
        zoom_ax = axs[-1]
        zoom_ax.set_axis_bgcolor("#E6E6B8")
        zoom_ax.title.set_text("Selected Segment")
        zoom_line, = zoom_ax.plot([], [], "#008A2E")

        for i in range(len(coefs)):
            coef = coefs[i]
            y = _np.abs(coef[1:N / 2])
            x = _np.arange(len(y))
            color = colors.next()
            ax = axs[i]

            ax.set_axis_bgcolor("#E6E6B8")
            ax.title.set_text("Channel {}".format(i + 1))
            ax.plot(x, y, color)

            def onselect_callback(xmin, xmax):

                indmin, indmax = _np.searchsorted(x, (xmin, xmax))
                indmax = min(len(y) - 1, indmax)

                x_segment = x[indmin:indmax]
                y_segment = y[indmin:indmax]
                zoom_line.set_data(x_segment, y_segment)
                zoom_ax.set_xlim(x_segment[0], x_segment[-1])
                zoom_ax.set_ylim(y_segment.min(), y_segment.max())
                zoom_ax.title.set_text("Selected Segment: {} to {}".format(
                    indmin, indmax))

                _plt.draw()

            span_selector = _SpanSelector(ax,
                                          onselect_callback,
                                          'horizontal',
                                          span_stays=True,
                                          rectprops=dict(alpha=0.5,
                                                         facecolor='cyan'))
            self.__current_widgets["span_selectors"].append(span_selector)

        self.__current_figs.append(fig)
        fig.subplots_adjust(top=0.9, hspace=0.3)
Example #18
0
    def plot(self, by_sec = True, colors = None):
        """
        Plot the audio data.

        params:
            `by_sec`: if True, it will plot the audio in second or it will plot
                      the audio data in the number of frames.
            `colors`: Any iterable of symbols of color supported by matplotlib.
        """
        if colors is None:
            colors = _cycle(["#FF6600", "#0033CC"])
        else:
            colors = _cycle(colors)
        
        num_subplots = self.audio_data.CHANNELS + 1
        fig, axs = _plt.subplots(num_subplots, 1, figsize = (20, 7))
        fig.suptitle(repr(self.audio_data), fontsize = 14, y = 1)
        frames = [self.audio_data.data[i] for i in range(2)] if self.audio_data.CHANNELS == 2 else [self.audio_data.data]

        if by_sec:
            x = _np.linspace(0, len(frames[0])/self.audio_data.SAMPLE_RATE, num = len(frames[0]))
            x_label = "Time (Seconds)"
        else:
            x = _np.linspace(0, len(frames[0]), num = len(frames[0]))
            x_label = "Frame"

        zoom_ax = axs[-1]
        zoom_ax.set_axis_bgcolor("#E6E6B8")
        zoom_ax.title.set_text("Selected Segment")
        zoom_ax.xaxis.set_label_text(x_label)
        zoom_ax.yaxis.set_label_text("Altitude")
        zoom_line, = zoom_ax.plot([], [], "#008A2E")

        temp = []
        for i, (ax, frame) in enumerate(zip(axs[:-1], frames)):
            color = colors.next()

            ax.title.set_text("Channel {}".format(i+1))
            ax.set_axis_bgcolor("#E6E6B8")
            ax.plot(x, frame, color)
            ax.set_xlim(x[0], x[-1])
            ax.xaxis.set_label_text(x_label)
            ax.yaxis.set_label_text("Altitude")

            def onselect_callback(xmin, xmax):

                indmin, indmax = _np.searchsorted(x, (xmin, xmax))
                indmax = min(len(frames[0]) - 1, indmax)

                x_segment = x[indmin:indmax]
                y_segment = frame[indmin:indmax]
                zoom_line.set_data(x_segment, y_segment)
                zoom_ax.set_xlim(x_segment[0], x_segment[-1])
                zoom_ax.set_ylim(y_segment.min(), y_segment.max())

                if by_sec:
                    x_start = int(self.audio_data.duration * float(indmin) / len(x))
                    x_stop = int(self.audio_data.duration * float(indmax) / len(x))
                else:
                    x_start = indmin
                    x_stop = indmax
                
                start_time = self.audio_data.duration * float(indmin) / len(x)
                stop_time = self.audio_data.duration * float(indmax) / len(x)
                zoom_ax.title.set_text("Selected Segment: {} to {}".format(x_start, x_stop))
                _plt.draw()

                self.audio_data.play(start_time, stop_time)
            
            span_selector = _SpanSelector(ax, onselect_callback, 'horizontal', span_stays = True, 
                                          rectprops = dict(alpha = 0.5, facecolor = 'cyan'))
            self.__current_widgets["span_selectors"].append(span_selector)
        fig.subplots_adjust(top = 0.9, hspace = 0.7)
        self.__current_figs.append(fig)
Example #19
0
def plot_surface_area(surface_area, fig_fn, timeunit=None, fig_close=False):
    """Plot binding site surface area as a function of time.

    The provided ``surface_area`` needs to be a pandas.DataFrame object, which has columns named as "Binding Site {idx}"
    to record binding site surface areas and a column named "Time" to record the timesteps at which the surface area data
    are taken.

    Parameters
    ----------
    surface_area : pandas.DataFrame
        A pandas.DataFrame object which has columns named as "Binding Site {idx}"
        to record binding site surface areas and a column named "Time" to record the timesteps at which the surface area data
    fig_fn : str
        Figure filename.
    timeunit : {"ns", "us"} or None, optional, default=None
        Time unit shown in x label.
    fig_close : bool, optional, default=False
        Use plt.close() to close the figure. Can be used to save memory if many figures are opened.

    See also
    ---------
    pylipid.func.calculate_surface_area
        The function that generates surface_area data.

    """
    from itertools import cycle as _cycle

    color_set = _cycle(plt.get_cmap("tab10").colors)
    plt.rcParams["font.size"] = 10
    plt.rcParams["font.weight"] = "normal"

    if timeunit is None:
        timeunit = ""
    elif timeunit == "ns":
        timeunit = " (ns)"
    elif timeunit == "us":
        timeunit = r" ($\mu$s)"

    row_set = list(set([ind[:2] for ind in surface_area.index]))
    row_set.sort()
    col_set = [col for col in surface_area.columns if col != "Time"]
    colors = [next(color_set) for dummy in col_set]
    fig, axes = plt.subplots(len(row_set),
                             len(col_set),
                             figsize=(len(col_set) * 2.4, len(row_set) * 1.6),
                             sharex=True,
                             sharey=True)
    plt.subplots_adjust(wspace=0.2, hspace=0.16)
    if len(col_set) == 1:
        axes = np.atleast_1d(axes)[:, np.newaxis]
    else:
        axes = np.atleast_2d(axes)
    for row_idx, row in enumerate(row_set):
        df = surface_area.loc[row]
        for col_idx, bs_name in enumerate(col_set):
            axes[row_idx,
                 col_idx].plot(df["Time"],
                               df[bs_name],
                               color=colors[col_idx],
                               label="traj {} prot {}".format(row[0], row[1]))
            if row_idx == len(row_set) - 1:
                axes[row_idx, col_idx].set_xlabel("Time{}".format(timeunit),
                                                  fontsize=10)
            if col_idx == 0:
                axes[row_idx, col_idx].set_ylabel(r"Area (nm$^2$)",
                                                  fontsize=10)
            if row_idx == 0:
                axes[row_idx, col_idx].set_title(bs_name, fontsize=10)
            axes[row_idx, col_idx].legend(loc="best", frameon=False)
    fig.tight_layout()
    fig.savefig(fig_fn, dpi=200)
    if fig_close:
        plt.close()

    return
Example #20
0
    def create_summary_data(self, predictions={}, verbosity=2, auxtypes=[]):
        """
        todo
        """
        assert (self.multids
                is not None), "Cannot generate summary data without a DataSet!"
        assert ('standard' in self.multids.keys()
                ), "Currently only works for standard dataset!"
        useds = 'standard'
        # We can't use the success-fail dataset if there's any simultaneous benchmarking. Not in
        # it's current format anyway.

        summarydata = {}
        aux = {}
        globalsummarydata = {}
        predsummarydata = {}
        predds = None
        preddskey = None
        for pkey in predictions.keys():
            predsummarydata[pkey] = {}
            if isinstance(predictions[pkey], _stdds.DataSet):
                assert (predds is None), "Can't have two DataSet predictions!"
                predds = predictions[pkey]
                preddskey = pkey
            else:
                assert (isinstance(predictions[pkey],
                                   _oplessmodel.SuccessFailModel)
                        ), "If not a DataSet must be an ErrorRatesModel!"

        datatypes = [
            'success_counts', 'total_counts', 'hamming_distance_counts',
            'success_probabilities'
        ]
        if self.dscomparator is not None:
            stabdatatypes = ['tvds', 'pvals', 'jsds', 'llrs', 'sstvds']
        else:
            stabdatatypes = []

        #preddtypes = ('success_probabilities', )
        auxtypes = [
            'twoQgate_count', 'depth', 'target', 'width', 'circuit_index'
        ] + auxtypes

        def _get_datatype(datatype, dsrow, circ, target, qubits):

            if datatype == 'success_counts':
                return _analysis.marginalized_success_counts(
                    dsrow, circ, target, qubits)
            elif datatype == 'total_counts':
                return dsrow.total
            elif datatype == 'hamming_distance_counts':
                return _analysis.marginalized_hamming_distance_counts(
                    dsrow, circ, target, qubits)
            elif datatype == 'success_probabilities':
                sc = _analysis.marginalized_success_counts(
                    dsrow, circ, target, qubits)
                tc = dsrow.total
                if tc == 0:
                    return _np.nan
                else:
                    return sc / tc
            else:
                raise ValueError("Unknown data type!")

        numpasses = len(self.multids[useds].keys())

        for ds_ind in self.multids[useds].keys():

            if verbosity > 0:
                print(
                    " - Processing data from pass {} of {}. Percent complete:".
                    format(ds_ind + 1, len(self.multids[useds])))

            #circuits = {}
            numcircuits = len(self.multids[useds][ds_ind].keys())
            percent = 0

            if preddskey is None or ds_ind > 0:
                iterator = zip(
                    self.multids[useds][ds_ind].items(
                        strip_occurrence_tags=True),
                    self.multids[useds].auxInfo.values(),
                    _cycle(zip([
                        None,
                    ], [
                        None,
                    ])))
            else:
                iterator = zip(
                    self.multids[useds][ds_ind].items(
                        strip_occurrence_tags=True),
                    self.multids[useds].auxInfo.values(),
                    predds.items(strip_occurrence_tags=True))

            for i, ((circ, dsrow), auxdict, (pcirc,
                                             pdsrow)) in enumerate(iterator):

                if pcirc is not None:
                    if not circ == pcirc:
                        print('-{}-'.format(i))
                        pdsrow = predds[circ]
                        _warnings.warn(
                            "Predicted DataSet is ordered differently to the main DataSet!"
                            +
                            "Reverting to potentially slow dictionary hashing!"
                        )

                if verbosity > 0:
                    if _np.floor(100 * i / numcircuits) >= percent:
                        percent += 1
                        if percent in (1, 26, 51, 76):
                            print("\n    {},".format(percent), end='')
                        else:
                            print("{},".format(percent), end='')
                        if percent == 100:
                            print('')

                speckeys = auxdict['spec']
                try:
                    depth = auxdict['depth']
                except:
                    depth = auxdict['length']
                target = auxdict['target']

                if isinstance(speckeys, str):
                    speckeys = [speckeys]

                for speckey in speckeys:
                    specind = self._speckeys.index(speckey)
                    spec = self._specs[specind]
                    structure = spec.get_structure()

                    # If we've not yet encountered this specind, we create the required dictionaries to store the
                    # summary data from the circuits associated with that spec.
                    if specind not in summarydata.keys():

                        assert (ds_ind == 0)
                        summarydata[specind] = {
                            qubits: {datatype: {}
                                     for datatype in datatypes}
                            for qubits in structure
                        }
                        aux[specind] = {
                            qubits: {auxtype: {}
                                     for auxtype in auxtypes}
                            for qubits in structure
                        }

                        # Only do predictions on the first pass dataset.
                        for pkey in predictions.keys():
                            predsummarydata[pkey][specind] = {}
                            for pkey in predictions.keys():
                                if pkey == preddskey:
                                    predsummarydata[pkey][specind] = {
                                        qubits: {
                                            datatype: {}
                                            for datatype in datatypes
                                        }
                                        for qubits in structure
                                    }
                                else:
                                    predsummarydata[pkey][specind] = {
                                        qubits: {
                                            'success_probabilities': {}
                                        }
                                        for qubits in structure
                                    }

                        globalsummarydata[specind] = {
                            qubits:
                            {datatype: {}
                             for datatype in stabdatatypes}
                            for qubits in structure
                        }

                    # If we've not yet encountered this depth, we create the list where the data for that depth
                    # is stored.
                    for qubits in structure:
                        if depth not in summarydata[specind][qubits][
                                datatypes[0]].keys():

                            assert (ds_ind == 0)
                            for datatype in datatypes:
                                summarydata[specind][qubits][datatype][
                                    depth] = [[] for i in range(numpasses)]
                            for auxtype in auxtypes:
                                aux[specind][qubits][auxtype][depth] = []

                            for pkey in predictions.keys():
                                if pkey == preddskey:
                                    for datatype in datatypes:
                                        predsummarydata[pkey][specind][qubits][
                                            datatype][depth] = []
                                else:
                                    predsummarydata[pkey][specind][qubits][
                                        'success_probabilities'][depth] = []

                            for datatype in stabdatatypes:
                                globalsummarydata[specind][qubits][datatype][
                                    depth] = []

                    #print('---', i)
                    for qubits_ind, qubits in enumerate(structure):
                        for datatype in datatypes:
                            x = _get_datatype(datatype, dsrow, circ, target,
                                              qubits)
                            summarydata[specind][qubits][datatype][depth][
                                ds_ind].append(x)
                            # Only do predictions on the first pass dataset.
                            if preddskey is not None and ds_ind == 0:
                                x = _get_datatype(datatype, pdsrow, circ,
                                                  target, qubits)
                                predsummarydata[preddskey][specind][qubits][
                                    datatype][depth].append(x)

                        # Only do predictions and aux on the first pass dataset.
                        if ds_ind == 0:
                            for auxtype in auxtypes:
                                if auxtype == 'twoQgate_count':
                                    auxdata = circ.two_q_gate_count()
                                elif auxtype == 'depth':
                                    auxdata = circ.depth
                                elif auxtype == 'target':
                                    auxdata = target
                                elif auxtype == 'circuit_index':
                                    auxdata = i
                                elif auxtype == 'width':
                                    auxdata = len(qubits)
                                else:
                                    auxdata = auxdict.get(auxtype, None)

                                aux[specind][qubits][auxtype][depth].append(
                                    auxdata)

                            for pkey, predmodel in predictions.items():
                                if pkey != preddskey:
                                    if set(circ.line_labels) != set(qubits):
                                        trimmedcirc = circ.copy(editable=True)
                                        for q in circ.line_labels:
                                            if q not in qubits:
                                                trimmedcirc.delete_lines(q)
                                    else:
                                        trimmedcirc = circ

                                    predsp = predmodel.probabilities(
                                        trimmedcirc)[('success', )]
                                    predsummarydata[pkey][specind][qubits][
                                        'success_probabilities'][depth].append(
                                            predsp)

                            for datatype in stabdatatypes:
                                if datatype == 'tvds':
                                    x = self.dscomparator.tvds.get(
                                        circ, _np.nan)
                                elif datatype == 'pvals':
                                    x = self.dscomparator.pVals.get(
                                        circ, _np.nan)
                                elif datatype == 'jsds':
                                    x = self.dscomparator.jsds.get(
                                        circ, _np.nan)
                                elif datatype == 'llrs':
                                    x = self.dscomparator.llrs.get(
                                        circ, _np.nan)
                                globalsummarydata[specind][qubits][datatype][
                                    depth].append(x)

            if verbosity > 0:
                print('')

        #  Record the data in the object at the end.
        self.predicted_summary_data = predsummarydata
        self.pass_summary_data = summarydata
        self.global_summary_data = globalsummarydata
        self.aux = aux