Example #1
0
def follow_trajectory_with_grabs(pr2, bodypart2traj, ignore_failure=False):
    """
    bodypart2traj is a dictionary with zero or more of the following fields: {l/r}_grab, {l/r}_gripper, {l/r_arm}
    We'll follow all of these bodypart trajectories simultaneously
    Also, if the trajectory involves grabbing with the gripper, and the grab fails, the trajectory will abort
    """
    T = len(bodypart2traj.values()[0])
    l_grab = bodypart2traj["l_grab"] if "l_grab" in bodypart2traj else np.zeros(T,bool)
    r_grab = bodypart2traj["r_grab"] if "r_grab" in bodypart2traj else np.zeros(T,bool)
    l_after_grab_inds = np.flatnonzero(l_grab[1:] > l_grab[:-1])+1
    r_after_grab_inds = np.flatnonzero(r_grab[1:] > r_grab[:-1])+1
    inds_sides = sorted([(i,'l') for i in l_after_grab_inds] + [(i,'r') for i in r_after_grab_inds])
    inds_sides = merge_nearby_grabs(inds_sides)
    
    num_grabs = len(inds_sides)
    print "breaking trajectory into %i segments"%(num_grabs+1)
    i_begin = 0
    for (i_grab, side) in inds_sides:
        t_start = rospy.Time.now().to_sec()
        success = follow_trajectory(pr2, slice_traj(bodypart2traj, i_begin, i_grab))
        rospy.loginfo("follow traj result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start)
        if not success and not ignore_failure: return False
        t_start = rospy.Time.now().to_sec()
        success = close_gripper(pr2, side)
        rospy.loginfo("close gripper result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start)        
        if not success and not ignore_failure: return False        
        i_begin = i_grab
    t_start = rospy.Time.now().to_sec()
    success = follow_trajectory(pr2, slice_traj(bodypart2traj, i_begin, -1))
    rospy.loginfo("follow traj result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start)
    if not success and not ignore_failure: return False
    return True
Example #2
0
def coloring(m, inv_row_p, inv_col_p):
    # The permutation must give a matrix in Hessenberg form
    row_major, col_major = get_permuted_sp_matrices(m, inv_row_p, inv_col_p)
    ncols = col_major.shape[1]
    # Initially all column colors are set to the invalid "color" -1
    column_colors = np.full(ncols, -1, np.int32)
    # If color i is available: color_available[i] is 1, otherwise 0
    # Initially, "color" 0 is available 
    color_available = np.ones(1, np.int8)
    # Iterate through the columns backwards, from right to left
    for c in reversed(irange(ncols)):
        # All rows containing c
        for r in rows_in_col(col_major, c):
            # All other columns in those rows, c's "neighbors"
            cols = cols_in_row(row_major, r)
            idx = np.flatnonzero(cols==c)
            # cols in r: [ left | c | right]
            # only the right index set has been processed so far
            right = cols[idx+1:]
            used_colors = column_colors[right]
            color_available[used_colors] = 0
        #print('Colors available:\n%s' % color_available)
        index = np.flatnonzero(color_available)[0]
        column_colors[c] = index
        # introduce new "color" if index == color_available.size-1 
        color_available = np.ones(max(index+2, color_available.size), np.int8)
    print('Colors:\n%s' % column_colors)
    # chromatic number >= max nonzeros in a row
    lb_chromatic_number = np.max(np.diff(row_major.indptr))
    color_count = color_available.size-1
    print('Color count: %d (>=%d)' % (color_count, lb_chromatic_number))
    return  column_colors, color_count
def dump_pre_post_stim_spike_count(ffname, outprefix, celltype, window=10e-3):
    """Dump the standard deviation in population spike before and after stimulus.
    """
    with open('{}_prepost_spikes_{}_{}ms_window.csv'.format(outprefix, celltype, window*1e3), 'wb') as fd:
        writer = csv.writer(fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        norm_files = get_dbcnt_dict(ffname)
        writer.writerow(['dbcount', 'filename', 'premean', 'premedian', 'prestd', 'postmean', 'postmedian', 'poststd'])
        for dbcnt, flist in norm_files.items():
            for fname in flist:
                data = TraubData(makepath(fname))
                pop_train_list = []
                bgtimes, probetimes = get_stim_times(data, correct_tcr=False)
                times = np.concatenate((bgtimes, probetimes))
                times.sort()
                for cell, train in data.spikes.items():
                    if cell.startswith(celltype):
                        pop_train_list.append(train)
                pop_train = np.concatenate(pop_train_list)
                pop_train.sort()
                pre = []
                post = []
                for t in times:
                    pre.append(np.flatnonzero((pop_train < t) & (pop_train > t - window/2)).shape[0]*1.0 / data.cellcounts._asdict()[celltype])
                    post.append(np.flatnonzero((pop_train > t) & (pop_train < t + window/2)).shape[0]*1.0 / data.cellcounts._asdict()[celltype])
                writer.writerow([dbcnt, fname] + [np.mean(pre), np.median(pre), np.std(pre, ddof=1), np.mean(post), np.median(post), np.std(post, ddof=1)])
Example #4
0
def sort_on_centroids(fn_mwk, fn_nev,k_cent_all,pc_sd_all,T_all,sel=SEL_ELEC_A, nmax=N_SNIPPET_MAX, npts=N_SNIPPET_PTS,c_v=VOLT_CONV):
	#this spikes sorts based on how close a spike is to the farthest centroid from center of mass of all the centroids
	sorted_data = {}
	for arg in getspk(fn_mwk, fn_nev, override_elecs=sel):
			# -- preps
			w = np.array(arg['wav']['unsorted'])
			w *= c_v
			ch = arg['ch']
			if ch not in sorted_data.keys():
				sorted_data[ch] = {'good_sp':[],'bad_sp':[],'g_sp_t_abs':[],'g_sp_t_rel':[],'g_sp_t_imgonset':[],'g_sp_im_id':[],'g_sp_im_ind':[],'b_sp_t_abs':[],'b_sp_t_rel':[],'b_sp_t_imgonset':[],'b_sp_im_id':[],'b_sp_im_ind':[]}
			ch_ind = sel.index(ch)
			#pick the centroid and the spikes sd for that channel
			pc_sd = pc_sd_all[ch_ind]
			k_cent = k_cent_all[ch_ind]
			k_dist = np.array([fastnorm(k-k_cent.mean(axis=0)) for k in k_cent])
			T = T_all[ch_ind]
			b_c = np.flatnonzero(k_dist == k_dist.max())
			#compute the distance of the waveform from the the centroids and determine whether it is a good spike or a bad spike.
			w_dist = np.array([fastnorm((np.dot(w,T)/pc_sd)-k_cent[k]) for k in range(len(k_cent))])
			if  np.flatnonzero(w_dist == w_dist.min())==b_c and k_dist.max() > 1.75:
				sorted_data[ch]['good_sp'].append(w)
				sorted_data[ch]['g_sp_t_abs'].append(arg['t_abs'])
				sorted_data[ch]['g_sp_t_rel'].append(arg['t_rel'])
				sorted_data[ch]['g_sp_t_imgonset'].append(arg['t_imgonset'])
				sorted_data[ch]['g_sp_im_id'].append(arg['imgid'])
				sorted_data[ch]['g_sp_im_ind'].append(arg['iimg'])
			else:
				sorted_data[ch]['b_sp_t_abs'].append(arg['t_abs'])
				sorted_data[ch]['b_sp_t_rel'].append(arg['t_rel'])
				sorted_data[ch]['b_sp_t_imgonset'].append(arg['t_imgonset'])
				sorted_data[ch]['b_sp_im_id'].append(arg['imgid'])
				sorted_data[ch]['b_sp_im_ind'].append(arg['iimg'])
				
	return sorted_data
Example #5
0
def align(s_x, s_y, mode="front"):
    """Function to align the input series

    :param s_x: First Series
    :type s_x: :mod:`pandas.Series`

    :param s_y: Second Series
    :type s_y: :mod:`pandas.Series`

    :param mode: Align Front/Back
    :type mode: str
    """

    p_x = np.flatnonzero(s_x)
    p_y = np.flatnonzero(s_y)

    if not len(p_x) or not len(p_y):
        return s_x, s_y, 0

    if mode == "front":
        p_x = p_x[0]
        p_y = p_y[0]

    if mode == "back":
        p_x = p_x[-1]
        p_y = p_y[-1]

    shift = p_x - p_y

    s_x, s_y = shift_series(s_x, s_y, shift)
    return s_x, s_y, shift
Example #6
0
def _mmd_ab(a, b, one_ring_neighbour):
    """
    Compute median minimal distance between a,b
    
    part computational completion of median_minimal_distance

    Parameters:
    -----------
    a, b: array with 1 label
    one_ring_neighbour: one ring neighbour matrix

    Return:
    -------
    h: minimal distance
    """
    a = np.array(a)
    b = np.array(b)
    h = []
    for i in np.flatnonzero(a):
        hd = np.inf
        for j in np.flatnonzero(b):
            d = surf_dist(i, j, one_ring_neighbour)
            if d<hd:
                hd = d
        h.append(hd)
    return h
def get_cap_check_indices(i):
    # Assumes that there is a test pulse followed by the stimulus pulses (downward first)
    di = np.diff(i)
    up_idx = np.flatnonzero(di > 0)
    down_idx = np.flatnonzero(di < 0)
    
    return up_idx[2::2], down_idx[1::2]
Example #8
0
    def update_nogrid(self, params):

        endog = self.model.endog_li
        cached_means = self.model.cached_means
        varfunc = self.model.family.variance

        dep_params = np.zeros(self.max_lag + 1)
        dn = np.zeros(self.max_lag + 1)
        for i in range(self.model.num_group):

            expval, _ = cached_means[i]
            stdev = np.sqrt(varfunc(expval))
            resid = (endog[i] - expval) / stdev

            j1, j2 = np.tril_indices(len(expval))
            dx = np.abs(self.time[i][j1] - self.time[i][j2])
            ii = np.flatnonzero(dx <= self.max_lag)
            j1 = j1[ii]
            j2 = j2[ii]
            dx = dx[ii]

            vs = np.bincount(dx, weights=resid[
                             j1] * resid[j2], minlength=self.max_lag + 1)
            vd = np.bincount(dx, minlength=self.max_lag + 1)

            ii = np.flatnonzero(vd > 0)
            dn[ii] += 1
            if len(ii) > 0:
                dep_params[ii] += vs[ii] / vd[ii]

        dep_params /= dn
        self.dep_params = dep_params[1:] / dep_params[0]
Example #9
0
def _hausdoff_ab(a, b, one_ring_neighbour):
    """
    Compute hausdoff distance of h(a,b)
    part unit of function hausdoff_distance
    
    Parameters:
    -----------
    a: array with 1 label
    b: array with 1 label
    one_ring_neighbour: one ring neighbour matrix

    Return:
    -------
    h: hausdoff(a,b)

    """
    a = np.array(a)
    b = np.array(b)
    h = 0
    for i in np.flatnonzero(a):
        hd = np.inf
        for j in np.flatnonzero(b):
            d = surf_dist(i,j, one_ring_neighbour)    
            if d<hd:
                hd = copy.deepcopy(d)
        if hd>h:
            h = hd
    return h
  def _calculateSegmentActivity(connections, activeInput, connectedPermanence,
                                activationThreshold, minThreshold):
    """
    Calculate the active and matching segments for this timestep.

    @param connections (SparseMatrixConnections)
    @param activeInput (numpy array)

    @return (tuple)
    - activeSegments (numpy array)
      Dendrite segments with enough active connected synapses to cause a
      dendritic spike

    - matchingSegments (numpy array)
      Dendrite segments with enough active potential synapses to be selected for
      learning in a bursting column

    - potentialOverlaps (numpy array)
      The number of active potential synapses for each segment.
      Includes counts for active, matching, and nonmatching segments.
    """

    # Active
    overlaps = connections.computeActivity(activeInput, connectedPermanence)
    activeSegments = np.flatnonzero(overlaps >= activationThreshold)

    # Matching
    potentialOverlaps = connections.computeActivity(activeInput)
    matchingSegments = np.flatnonzero(potentialOverlaps >= minThreshold)

    return (activeSegments,
            matchingSegments,
            potentialOverlaps)
Example #11
0
    def connection_field_plot_continuous(self, index, afferent=True, density=30):
        weights = numpy.array(self.proj.get('weight', format='list', gather=True))
        if afferent:
            idx = numpy.array(numpy.flatnonzero(weights[:,1].flatten()==index))
            x = self.proj.pre.positions[0][weights[idx,0].astype(int)]
            y = self.proj.pre.positions[1][weights[idx,0].astype(int)]
            w = weights[idx,2]
        else:
            idx = numpy.flatnonzero(weights[:,0]==index)
            x = self.proj.post.positions[0][weights[idx,1].astype(int)]
            y = self.proj.post.positions[1][weights[idx,1].astype(int)]
            w = weights[idx,2]

        xi = numpy.linspace(min(x), max(x), 100)
        yi = numpy.linspace(min(y), max(y), 100)
        zi = griddata(x, y, w, xi, yi)
        pylab.figure()
        #pylab.imshow(zi)
        pylab.scatter(x,y,marker='o',c=w,s=50)
        pylab.xlim(-self.source.parameters.sx/2,self.source.parameters.sx/2)
        pylab.ylim(-self.source.parameters.sy/2,self.source.parameters.sy/2)
        pylab.colorbar()
        pylab.title('Connection field from %s to %s of neuron %d' % (self.source.name,
                                                                     self.target.name,
                                                                     index))
    def get_feature_mask(self, mask):
        """
        Parameters
        ----------
        mask: (N, F) or (F,) ndarray of bool
            True for unobserved features.

        Returns
        -------
        feature_mask: (N, D) or (D,) ndarray of bool
            True for unobserved features.
        """
        assert(mask.dtype == bool)
        if mask.ndim == 1:
            assert(mask.shape[0] == self.F)
            feature_mask = np.ones(self.D).astype(bool)
            for ind in np.flatnonzero(~mask):
                feature_mask[slice(*self.feature_bounds[ind])] = False
        else:
            assert(mask.shape[1] == self.F)
            N = mask.shape[0]
            feature_mask = np.ones((N, self.D)).astype(bool)
            for i in xrange(N):
                for ind in np.flatnonzero(~mask[i]):
                    feature_mask[i, slice(*self.feature_bounds[ind])] = False
        return feature_mask
Example #13
0
 def _split_indices(self, vec):
     null = pd.isnull(vec)
     ix_obs = np.flatnonzero(~null)
     ix_miss = np.flatnonzero(null)
     if len(ix_obs) == 0:
         raise ValueError("variable to be imputed has no observed values")
     return ix_obs, ix_miss
Example #14
0
def elam_spline(xin, yin, yspl_in, xout):
    """
    interpolate values from Elam photoabsorption and
    scattering tables, according to Elam, and following
    standard interpolation methods.  Calc borrowed from D. Dale.

    Parameters:
        xin (ndarray): x values for interpolation data
        yin (ndarray): y values for interpolation data
        yspl_in (ndarray): spline coefficients (second derivatives of y) for
                       interpolation data
        xout(float or ndarray): x values to be evaluated at

    Returns:
        ndarray: interpolated values
    """
    x = as_ndarray(xout)
    x[np.where(x < min(xin))] = min(xin)
    x[np.where(x > max(xin))] = max(xin)

    lo, hi = np.array([(np.flatnonzero(xin < e)[-1],
                        np.flatnonzero(xin > e)[0])
                       for e in x]).transpose()

    diff = xin[hi] - xin[lo]
    if any(diff <= 0):
        raise ValueError('x must be strictly increasing')
    a = (xin[hi] - x) / diff
    b = (x - xin[lo]) / diff
    return (a * yin[lo] + b * yin[hi] +
            (diff*diff/6) * ((a*a - 1) * a * yspl_in[lo] +
                             (b*b - 1) * b * yspl_in[hi]))
Example #15
0
    def retrieve_coverage_information(self, cov_src=""):
        u"""
        Retrieve coverage information, i.e. whether locations to have spectra
        extracted have valid data and are not covered by clouds, margins etc.
        Information is retrieved from an external file.
        """
        self.coverage = dict()
        if cov_src and os.path.isfile(cov_src):
            for img_id in self.image_data:
                self.coverage[img_id] = list()
            try:
                data = np.loadtxt(cov_src)
            except:
                data = np.genfromtxt(cov_src, converters={0: lambda s: s.strip()})

            for d in data:
                # extracting row_id, usually a location id, i.e. for a plot
                try:
                    row_id = int(d[0])
                except:
                    row_id = d[0]
                # retrieving non-zero elements for each row, i.e. indicators
                # for visibility and coverage for each image id
                try:
                    nz = np.flatnonzero(d[1:])
                except:
                    # this line is necessary to deal with zero-rank arrays
                    # that are generated by np.genfromtxt above if the
                    # original location ids cannot be converted to float,
                    # i.e. if they contain letters
                    nz = np.flatnonzero(np.array(d.tolist()[1:]))
                for n in nz:
                    self.coverage[n].append(row_id)
    def __init__(self, alphabet, **kw):
        _ContinuousSubstitutionModel.__init__(self, alphabet, **kw)

        alphabet = self.getAlphabet() # as may be altered by recode_gaps etc.
        mask = self._instantaneous_mask
        N = len(alphabet)
        self.param_pick = numpy.zeros([N,N], int)
        self.parameter_order = []
        self.last_in_column = []
        for (d, (row, col)) in enumerate(zip(mask, mask.T)):
            row = list(numpy.flatnonzero(row[d:])+d)
            col = list(numpy.flatnonzero(col[d:])+d)
            if col:
                self.last_in_column.append((col.pop(), d))
            else:
                assert not row
            inst = [(d,j) for j in row] + [(i,d) for i in col]
            
            for (i, j) in inst:
                (x,y) = [alphabet[k] for k in [i,j]]
                self.parameter_order.append('%s/%s'%(x,y))
                self.param_pick[i,j] = len(self.parameter_order)
        if self._do_scaling:
            const_param = self.parameter_order.pop()
        self.symmetric = False
        self.checkParamsExist()
Example #17
0
    def readPulsar(self, psr, psrname):
        print("WARNING: readPulsar has been deprecated!")
        psr.name = psrname

        # Read the content of the par/tim files in a string
        psr.parfile_content = str(self.getData(psrname, 'parfile', required=False))
        psr.timfile_content = str(self.getData(psrname, 'timfile', required=False))

        # Read the timing model parameter descriptions
        psr.ptmdescription = map(str, self.getData(psrname, 'tmp_name'))
        psr.ptmpars = np.array(self.getData(psrname, 'tmp_valpre'))
        psr.ptmparerrs = np.array(self.getData(psrname, 'tmp_errpre'))
        psr.flags = map(str, self.getData(psrname, 'efacequad', 'Flags'))

        # Read the position of the pulsar
        if self.hasField(psrname, 'raj'):
            psr.raj = np.float(self.getData(psrname, 'raj'))
        else:
            rajind = np.flatnonzero(np.array(psr.ptmdescription) == 'RAJ')
            psr.raj = np.array(self.getData(psrname, 'tmp_valpre'))[rajind]

        if self.hasField(psrname, 'decj'):
            psr.decj = np.float(self.getData(psrname, 'decj'))
        else:
            decjind = np.flatnonzero(np.array(psr.ptmdescription) == 'DECJ')
            psr.decj = np.array(self.getData(psrname, 'tmp_valpre'))[decjind]

        # Obtain residuals, TOAs, etc.
        psr.toas = np.array(self.getData(psrname, 'TOAs'))
        psr.toaerrs = np.array(self.getData(psrname, 'toaErr'))
        psr.prefitresiduals = np.array(self.getData(psrname, 'prefitRes'))
        psr.residuals = np.array(self.getData(psrname, 'postfitRes'))
        psr.detresiduals = np.array(self.getData(psrname, 'prefitRes'))
        psr.freqs = np.array(self.getData(psrname, 'freq'))
        psr.Mmat = np.array(self.getData(psrname, 'designmatrix'))
Example #18
0
def split_trajectory_by_gripper(seg_info, thresh = .04, ms_thresh = 2):
    # thresh: open/close threshold
    rgrip = asarray(seg_info["r_gripper_joint"])
    lgrip = asarray(seg_info["l_gripper_joint"])

    n_steps = len(lgrip)


    # indices BEFORE transition occurs
    l_openings = np.flatnonzero((lgrip[1:] >= thresh) & (lgrip[:-1] < thresh))
    r_openings = np.flatnonzero((rgrip[1:] >= thresh) & (rgrip[:-1] < thresh))
    l_closings = np.flatnonzero((lgrip[1:] < thresh) & (lgrip[:-1] >= thresh))
    r_closings = np.flatnonzero((rgrip[1:] < thresh) & (rgrip[:-1] >= thresh))

    before_transitions = np.r_[l_openings, r_openings, l_closings, r_closings]
    after_transitions = before_transitions+1
    seg_starts = np.unique(np.r_[0, after_transitions])
    seg_ends = np.unique(np.r_[before_transitions, n_steps-1])

    # Only count gripper open/closes if the number of time steps is greater
    # than ms_thresh
    lr_open = {lr:[] for lr in 'lr'}
    new_seg_starts = []
    new_seg_ends = []
    for i in range(len(seg_starts)):
        if seg_ends[i]- seg_starts[i] >= ms_thresh:
            new_seg_starts.append(seg_starts[i])
            new_seg_ends.append(seg_ends[i])
            lval = True if lgrip[seg_starts[i]] >= thresh else False
            lr_open['l'].append(lval)
            rval = True if rgrip[seg_starts[i]] >= thresh else False
            lr_open['r'].append(rval)

    return new_seg_starts, new_seg_ends, lr_open
Example #19
0
def write_file(file_cc,file_bim,outfile):
    """
    This function write a output file (outfile) from information in .epi.cc and .bim.
    The output file has 11 columns and the last two columns are ID (prb1 and prb2) for each snp.

    columns:
    CHR1 | SNP1 | CHR2 | SNP2 | OR_INT | STAT | P | POSITION1 | POSITION2 | prb1 | prb2
    """

    print "Reading ..."
    data_cc = get_data_cc(file_cc)
    data_bim = get_data_bim(file_bim)

    rs =    np.unique(np.append( data_cc['SNP1'],data_cc['SNP2'] ))

    shape_data=np.shape(data_cc)

    ofile = open(outfile,'w')    # open file for writing 
    print "Writing ..."
    for i in range(shape_data[0]):

        ofile.write(str(data_cc[i]['CHR1'])+" "
        +str(data_cc[i]['SNP1'])+" "
        +str(data_cc[i]['CHR2'])+" "
        +str(data_cc[i]['SNP2'])+" "
        +str(data_cc[i]['OR_INT'])+" "
        +str(data_cc[i]['STAT'])+" "
        +str(data_cc[i]['P'])+" "  
        +str(data_bim['position'][data_bim['rs']==data_cc[i]['SNP1']][0])+" " #position1
        +str(data_bim['position'][data_bim['rs']==data_cc[i]['SNP2']][0])+" " #position2  
        +str(np.flatnonzero(rs==data_cc[i]['SNP1'])[0])+" " #id1
        +str(np.flatnonzero(rs==data_cc[i]['SNP2'])[0])+" " #id2  np.flatnonzero(
        +"\n"   )
    
    ofile.close
def gen_crossed_logit_pandas(nc, cs, s1, s2):

    np.random.seed(3799)

    a = np.kron(np.arange(nc), np.ones(cs))
    b = np.kron(np.ones(cs), np.arange(nc))
    fe = np.ones(nc * cs)

    vc = np.zeros(nc * cs)
    for i in np.unique(a):
        ii = np.flatnonzero(a == i)
        vc[ii] += s1*np.random.normal()
    for i in np.unique(b):
        ii = np.flatnonzero(b == i)
        vc[ii] += s2*np.random.normal()

    lp = -0.5 * fe + vc
    pr = 1 / (1 + np.exp(-lp))
    y = 1*(np.random.uniform(size=nc*cs) < pr)

    ident = np.zeros(2*nc, dtype=np.int)
    ident[nc:] = 1

    df = pd.DataFrame({"fe": fe, "a": a, "b": b, "y": y})

    return df
def getDBFormat(factX, modes, axisDict):
    """ 
    Get the database format for this set of factorizations
    
    Parameters
    ------------
    factX : the list of tensor factorizations
    modes: a list of all the unique modes in the tensor list
    axisDict : the mapping between the tensor modes and the axis information
    
    Output
    -----------
    outStack: a 2-D array with 4 columns with mode, feature_name (derived from axisDict), factor, and value
    """
    outStack = np.zeros((1, 4))
    ## Get the lambdas
    for i in range(len(factX)):
        idx = np.flatnonzero(factX[i].lmbda)
        tmp = np.column_stack((np.repeat(-1 * (i+1), len(idx)), np.repeat("lambda", len(idx)).astype('S100'), idx, factX[i].lmbda[idx]))
        outStack = np.vstack((outStack, tmp))
    
    midx = 0
    for m in modes:
        tensorIdx = m.flatten()[0]
        tensorDim = m.flatten()[1]
        tTuple = (tensorIdx, tensorDim)
        print tTuple
        for r in range(factX[tensorIdx].R):
            idx = np.flatnonzero(factX[tensorIdx].U[tensorDim][:,r])
            tmp = np.column_stack((np.repeat(midx, len(idx)), np.array(axisDict[tTuple], dtype="S100")[idx], np.repeat(r, len(idx)), factX[tensorIdx].U[tensorDim][idx, r]))
            outStack = np.vstack((outStack, tmp))
        midx = midx + 1
    
    return outStack
Example #22
0
def __entrofy(X, k, w=None, q=None, pre_selects=None):
    '''See entrofy() for documentation'''

    n_participants, n_attributes = X.shape

    if w is None:
        w = np.ones(n_attributes)

    if q is None:
        q = 0.5 * np.ones(n_attributes)

    assert 0 < k <= n_participants
    assert not np.any(w < 0)
    assert np.all(q >= 0.0) and np.all(q <= 1.0)
    assert len(w) == n_attributes
    assert len(q) == n_attributes

    if k == n_participants:
        return np.arange(n_participants)

    # Initialization
    y = np.zeros(n_participants, dtype=bool)

    if pre_selects is None:
        # Select one at random
        pre_selects = np.random.choice(n_participants, size=1)

    y[pre_selects] = True

    # Where do we have missing data?
    Xn = np.isnan(X)

    while True:
        i = y.sum()
        if i >= k:
            break

        # Initialize the distribution vector
        p = np.nanmean(X[y], axis=0)
        p[np.isnan(p)] = 0.0

        # Compute the candidate distributions
        p_new = (p * i + X) / (i + 1.0)

        # Wherever X is nan, propagate the old p since we have no new information
        p_new[Xn] = (Xn * p)[Xn]

        # Compute marginal gain for each candidate
        delta = obj(p_new, w, q) - obj(p, w, q)

        # Knock out the points we've already taken
        delta[y] = -np.inf

        # Select the top score.  Break near-ties randomly.
        target_score = delta.max()
        target_score = target_score - 1e-3 * np.abs(target_score)
        new_idx = np.random.choice(np.flatnonzero(delta >= target_score))
        y[new_idx] = True

    return obj(np.nanmean(X[y], axis=0), w, q), np.flatnonzero(y)
Example #23
0
    def __ComputeHistogram__(self, x, x_range):
        """Function to compute the histogram of the data
        
        Parameters
        ----------
        x: ndarray
            Array containing the data
        x_range: tuple
            Tuple containing the minimum and maxumum to consider
            to build the histogram of the data x

        Returns
        -------
        pdf: 1d-array 
            Return the PDF of x using the range provided by the user
        bin_edge: 1d-array
            Return the bins corresponding to the PDF
    
        """        
        # Compute the histogram for the data x with unit bins
        pdf_rel, bin_edges_rel = np.histogram(x, bins=(np.max(x) - np.min(x)), density=True)

        # We need to translate the pdf depending of the range given by x_range
        ### Create an array with unit bins depending of x_range
        ### We need max - min + 1 bins
        pdf_abs = np.zeros((x_range[1] - x_range[0],))
        bin_edges_abs = np.array(range(x_range[0], x_range[1] + 1))
        ### Copy the relative pdf at the right position
        pdf_abs[np.flatnonzero(bin_edges_abs==bin_edges_rel[0])[0] : np.flatnonzero(bin_edges_abs==bin_edges_rel[-1])[0]] = pdf_rel[:]

        return (pdf_abs, bin_edges_abs)
Example #24
0
def fixgaps(x):
    """FIXGAPS: Linearly interpolates gaps in a time series
     YOUT=FIXGAPS(YIN) linearly interpolates over NaN in the input time
     series (may be complex), but ignores trailing and leading NaNs.
     R. Pawlowicz 11/6/99
     Version 1.0
    """
    

    #find nans
    bd = np.isnan(x)

    #early exit if there are no nans  
    if not bd.any():
        return x
    
    #find nonnans index numbers
    gd = np.flatnonzero(~bd)

    #ignore leading and trailing nans
    bd[:gd.min()]=False
    bd[(gd.max()+1):]=False
    
    #interpolate nans
    x[bd] = np.interp(np.flatnonzero(bd),gd,x[gd])

    return x
def preprocessRawData(raw_data,y_raw):
    # if age >= 38, labeled with 1
    # else, labeled with -1.
    labelvec = np.array(y_raw)
    y = np.ones(len(labelvec))
    neg = labelvec < 38
    y[neg] = -1 

    # positive examples: larger than or equal to 38
    # negative examples: smaller than 38
    num_pos = len(np.flatnonzero(y > 0))
    num_neg = len(np.flatnonzero(y < 0))
    #print('Number of positive/negative examples = %d/%d' % (num_pos, num_neg))

    headers = list(raw_data.columns.values) # get the features' name
    raw_feat = np.array(raw_data[headers]) # feature matrix without age feature  

    avg = np.mean(raw_feat,axis = 0)
    std_dev = np.std(raw_feat, axis = 0)


    X = (raw_feat-avg)/std_dev # scaled features matrix [-1,1]
    #print X.shape # X is N x n, where N is 342 subjects, n is 7 features
    #print y.shape # y is N x 1, where N is 342 subjects
    return X,y
def plotTrialPath(jsonData, trialNum, preTime=0, postTime=0, bPreRelStart=True, bPostRelEnd=True, bFolded=False, bLabels=True):
	if type(trialNum) == type(int()):
		trialNum = [trialNum]
	for tn in trialNum:
		trial = jsonData['trials'][tn]
		trialPath = getTrialPath(jsonData, tn, preTime, postTime, bPreRelStart, bPostRelEnd, bFolded)	
		if trial['bAvoidedShock']:
			pyplot.plot(trialPath[:,1],trialPath[:,2], 'g')
		else:
			pyplot.plot(trialPath[:,1],trialPath[:,2], 'r')
		#plot the LED
		if bFolded or trial['side']==0:
			pyplot.plot(0,35/2.0,marker='o',mec='r',mfc='r',ms=10)
			pyplot.plot(80,35/2.0,marker='o',mec='r',mfc='None',ms=10)
		else:
			pyplot.plot(0,35/2.0,marker='o',mec='r',mfc='None',ms=10)
			pyplot.plot(80,35/2.0,marker='o',mec='r',mfc='r',ms=10)	
		#plot dot where LED turned on.
		ndxLED = np.flatnonzero(trialPath[:,0]>0)
		if len(ndxLED)>0:
			pyplot.plot(trialPath[ndxLED[0],1],trialPath[ndxLED[0],2],marker='o',mec='r',mfc='r',ms=5)  
		#plot dot where Shock starts.
		ndxShock = np.flatnonzero(trialPath[:,0]>jsonData['parameters']['LEDTimeMS']/1000.0 - .1)
		if not trial['bAvoidedShock'] and len(ndxShock)>0:
			pyplot.plot(trialPath[ndxShock[0],1],trialPath[ndxShock[0],2],marker='o',mec='y',mfc='y',ms=5)  
		pyplot.text(trialPath[-1,1],trialPath[-1,2], str(tn))
	pyplot.xlim([0,80])
	pyplot.ylim([0,35])
	pyplot.axvline(x=getEscapePosition(jsonData),color='k',ls='--') 
	if bLabels:
		pyplot.xlabel('mm')
		pyplot.ylabel('mm')
def get_square_stim_characteristics(i, t, no_test_pulse=False):
    '''
    Identify the start time, duration, amplitude, start index, and
    end index of a square stimulus.
    This assumes that there is a test pulse followed by the stimulus square.
    '''

    di = np.diff(i)
    up_idx = np.flatnonzero(di > 0)
    down_idx = np.flatnonzero(di < 0)

    idx = 0 if no_test_pulse else 1

    # second square is the stimulus
    if up_idx[idx] < down_idx[idx]: # positive square
        start_idx = up_idx[idx] + 1 # shift by one to compensate for diff()
        end_idx = down_idx[idx] + 1
    else: # negative square
        start_idx = down_idx[idx] + 1
        end_idx = up_idx[idx] + 1

    stim_start = float(t[start_idx])
    stim_dur = float(t[end_idx] - t[start_idx])
    stim_amp = float(i[start_idx])

    return (stim_start, stim_dur, stim_amp, start_idx, end_idx)
def do_pca_analysis(profiles, lens, name='', plot=False):
	L = np.array(0.446*(lens-np.mean(lens)), dtype='float64')
	pr = []
	for i,p in enumerate(profiles):
		mask = np.isnan(p)
		p[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), p[~mask])
		av, va = moving_average(np.log(p+0.001), 46, 100)
		pr.append(av)
	y = np.array(pr)
	pca = PCA(n_components=2)
	pca.fit(y)
	print pca.explained_variance_ratio_
	yp = pca.transform(y)
	m,b,r,p,_ = stats.linregress(L, yp[:,0])
	p1 = [p]
	r1 = [r]
	for _ in xrange(300):
		sample = np.random.choice(L.shape[0], L.shape[0], replace=True)
		m,b,r,p,_ = stats.linregress(L[~sample], yp[~sample,0])
		p1.append(p)
		r1.append(r)
	m,b,r,p,_ = stats.linregress(L, yp[:,1])
	p2 = [p]
	r2 = [r]
	for _ in xrange(300):
		sample = np.random.choice(L.shape[0], L.shape[0], replace=True)
		m,b,r,p,_ = stats.linregress(L[~sample], yp[~sample,1])
		p2.append(p)
		r2.append(r)
	if plot:
		plot_pca(y, pca, yp, L, name)
	return r1, p1, r2, p2, L.shape[0], name, np.std(L)
Example #29
0
def splint(xa, ya, y2a, x):
    '''spline interpolation'''
    try:
        len(x)
    except TypeError:
        x = np.array([x])

    try:
        klo, khi = np.array([
            (np.flatnonzero(xa < i)[-1], np.flatnonzero(xa > i)[0])
            for i in x
            ]).transpose()
    except IndexError:
        raise ValueError(
            'Input values must be between %s and %s'
            % (np.exp(xa[0]), np.exp(xa[-1]))
            )

    h = xa[khi] - xa[klo]
    if any(h <= 0):
        raise ValueError, 'xa input must be strictly increasing'
    a = (xa[khi] - x) / h
    b = (x - xa[klo]) / h

    res = (
        a * ya[klo]
        + b * ya[khi]
        + ((a**3 - a) * y2a[klo] + (b**3 - b) * y2a[khi]) * (h**2) / 6
        )
    return res
Example #30
0
 def time_of_state_transition(self,prevStateID,nextStateID):
     '''Returns the time of the transition from prevStateID to nextStateID on each trial.
     If prevStateID is an empty string it calculates transition from any state.
     It assumes that the states have the same ID on all trials.
     If the transition did not occur, it returns NaN for that trial.
     If it occurs more than once, it returns the first time it happens.
     '''
     # -- If input is a state name, find its ID --
     if isinstance(nextStateID,str):
         nextStateIDval = self.find_stateID(nextStateID)
     else:
         nextStateIDval = nextStateID
     if len(prevStateID):
         USE_PREVSTATE = True
         if isinstance(prevStateID,str):
             prevStateIDval = self.find_stateID(prevStateID)
         else:
             prevStateIDval = prevStateID
     else:
         USE_PREVSTATE = False
     transitionEventTimes = np.empty(self['nTrials'])
     transitionEventTimes.fill(np.nan)
     eventsIndList = events_each_trial(self['RawEventsTrialID'])
     for trialInd,eventsInd in enumerate(eventsIndList):
         if USE_PREVSTATE:
             transitions = (self['RawEvents'][eventsInd,0]==prevStateIDval) &\
                           (self['RawEvents'][eventsInd,3]==nextStateIDval)
             transitionEventInds = np.flatnonzero(transitions)
         else:
             transitionEventInds = np.flatnonzero(self['RawEvents'][eventsInd,3]==nextStateIDval)
         if len(transitionEventInds):
             # Note that we store only the first transition
             eventOfInterest = eventsInd[transitionEventInds[0]]
             transitionEventTimes[trialInd] = self['RawEvents'][eventOfInterest,2]
     return transitionEventTimes
def plot_predictions(predictions,
                     view_date_begin=None,
                     view_date_end=None,
                     ax=None):
    """Creates a plot of the observed series and model predictions.

  Creates a `matplotlib` plot of the observed time series with intervals from
  the model, predictions for any unobserved points, and the locations of any
  anomalies.

  Args:
    predictions: instance of `PredictionOutput` as returned by
      `detect_anomalies`. This should contain predictions for a single series
      with no batch dimensions.
    view_date_begin: Optional `datetime.datetime` instance.
    view_date_end: Optional `datetime.datetime` instance.
    ax: Optional `matplotlib` figure axis.
  """
    # pylint: disable=g-import-not-at-top
    from matplotlib import pylab as plt
    from matplotlib import dates as mdates
    # pylint: enable=g-import-not-at-top

    if len(predictions.observed_time_series.shape) > 1:
        raise ValueError(
            'Time series must be one-dimensional; batches are not '
            'supported. Saw shape: {}.'.format(
                predictions.observed_time_series.shape))

    num_steps = len(predictions.times)
    time_delta = predictions.times[1] - predictions.times[0]
    time_period_length = predictions.times[-1] - predictions.times[0]
    if view_date_begin is None:
        view_date_begin = predictions.times[0] - 0.04 * time_period_length
    if view_date_end is None:
        view_date_end = predictions.times[-1] + 0.04 * time_period_length
    if not (isinstance(view_date_begin, datetime.datetime)
            and isinstance(view_date_end, datetime.datetime)):
        raise ValueError('View date start and end must be `datetime.datetime` '
                         'instances.')

    if ax is None:  # Create default axis.
        fig = plt.figure(figsize=(15, 5), constrained_layout=True)
        ax = fig.add_subplot(1, 1, 1)

    # Plot series with upper and lower limits.
    ax.plot(predictions.times,
            predictions.observed_time_series,
            color='black',
            alpha=0.8)
    ax.fill_between(predictions.times,
                    predictions.lower_limit,
                    predictions.upper_limit,
                    color='tab:blue',
                    alpha=0.3)
    # At steps where no time series was observed, plot the predictive mean.
    ax.plot(predictions.times,
            np.where(np.isnan(predictions.observed_time_series),
                     predictions.mean, np.nan),
            color='black',
            alpha=0.8,
            ls='--')

    # Highlight anomalies.
    for anomaly_idx in np.flatnonzero(predictions.is_anomaly):
        x = predictions.times[anomaly_idx]
        y = predictions.observed_time_series[anomaly_idx]
        ax.scatter(x, y, s=100, alpha=0.4, c='red')
        ax.annotate(str(x), (x, y))
    ax.set_ylabel('Series')
    ax.label_outer()

    # Use smart date formatting for the x axis.
    locator = mdates.AutoDateLocator(minticks=3, maxticks=7)
    ax.xaxis.set_major_locator(locator)
    ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(locator))
    ax.set_xlim([view_date_begin, view_date_end])
    ax.grid(True, color='whitesmoke')
    # Display the grid *underneath* the rest of the plot
    # (see https://github.com/matplotlib/matplotlib/issues/5045/).
    ax.set_axisbelow(True)
Example #32
0
    def fit(self, X, y):

        X, y = check_X_y(X, y, accept_sparse=True)
        self.is_fitted_ = True

        # transforming data
        raw_data = np.insert(X, 0, y, axis=1)
        N = raw_data.shape[0]

        # setup Y vector and Y_name
        Y_col_idx = [0]
        Y = raw_data[:, Y_col_idx]
        Y_name = self.data_headers[Y_col_idx[0]]
        Y[Y == 0] = -1

        # setup X and X_names
        X_col_idx = [j for j in range(raw_data.shape[1]) if j not in Y_col_idx]
        X = raw_data[:, X_col_idx]
        variable_names = [self.data_headers[j] for j in X_col_idx]

        # insert a column of ones to X for the intercept
        X = np.insert(arr=X, obj=0, values=np.ones(N), axis=1)
        variable_names.insert(0, '(Intercept)')

        if self.sample_weights is None or len(self.sample_weights) != N:
            self.sample_weights = np.ones(N)

        self.data = {
            'X': X,
            'Y': Y,
            'variable_names': variable_names,
            'outcome_name': Y_name,
            'sample_weights': self.sample_weights,
        }

        #load folds
        if self.fold_csv_file is not None:
            if not os.path.isfile(self.fold_csv_file):
                raise IOError('could not find fold_csv_file: %s' %
                              self.fold_csv_file)
            else:
                fold_idx = pd.read_csv(self.fold_csv_file,
                                       sep=',',
                                       header=None)
                fold_idx = fold_idx.values.flatten()
                K = max(fold_idx)
                all_fold_nums = np.sort(np.unique(fold_idx))
                assert len(
                    fold_idx
                ) == N, "dimension mismatch: read %r fold indices (expected N = %r)" % (
                    len(fold_idx), N)
                assert np.all(all_fold_nums == np.arange(
                    1, K +
                    1)), "folds should contain indices between 1 to %r" % K
                assert fold_num in np.arange(
                    0, K + 1
                ), "fold_num should either be 0 or an integer between 1 to %r" % K
                if fold_num >= 1:
                    test_idx = fold_num == fold_idx
                    train_idx = fold_num != fold_idx
                    data['X'] = data['X'][train_idx, ]
                    data['Y'] = data['Y'][train_idx]
                    data['sample_weights'] = data['sample_weights'][train_idx]

        assert check_data(self.data)

        # create coefficient set and set the value of the offset parameter
        coef_set = CoefficientSet(variable_names=self.data['variable_names'],
                                  lb=-self.max_coefficient,
                                  ub=self.max_coefficient,
                                  sign=0)
        conservative_offset = get_conservative_offset(self.data, coef_set,
                                                      self.max_L0_value)
        self.max_offset = min(self.max_offset, conservative_offset)
        coef_set['(Intercept)'].ub = self.max_offset
        coef_set['(Intercept)'].lb = -self.max_offset

        # edit contraints here
        constraints = {
            'L0_min': 0,
            'L0_max': self.max_L0_value,
            'coef_set': coef_set,
        }

        # initialize MIP for lattice CPA
        mip_objects = setup_lattice_cpa(self.data, constraints, self.settings)

        # add operational constraints
        mip = mip_objects['mip']
        indices = mip_objects['indices']
        get_alpha_name = lambda var_name: 'alpha_' + str(self.data[
            'variable_names'].index(var_name))
        get_alpha_ind = lambda var_names: [
            get_alpha_name(v) for v in var_names
        ]

        # applies mutual exclusivity feature contraints
        if self.op_constraints is not None:

            names = []
            expressions = []

            for key in self.op_constraints.keys():
                names.append("mutually_exclusive_%s" % key)
                expressions.append(
                    cplex.SparsePair(
                        ind=get_alpha_ind(self.op_constraints[key]),
                        val=[1.0] * len(self.op_constraints[key])))

            mip.linear_constraints.add(
                names=names,
                lin_expr=expressions,
                senses=["L"] * len(self.op_constraints.keys()),
                rhs=[1.0] * len(self.op_constraints.keys()))

        mip_objects['mip'] = mip

        # fit using ltca
        model_info, mip_info, lcpa_info = finish_lattice_cpa(
            self.data, constraints, mip_objects, self.settings)
        rho = model_info['solution']
        self.model_info = model_info

        if np.sum(rho[1:]) != 0:
            print_model(model_info['solution'], self.data)
        print("solver_time = %d" % model_info['solver_time'])
        print("optimality_gap = %.3f" % model_info['optimality_gap'])
        print(rho)

        variable_names = self.data['variable_names']
        rho_values = np.copy(rho)
        rho_names = list(variable_names)

        # removes intercept value or sets it to 0
        if '(Intercept)' in rho_names:
            intercept_ind = variable_names.index('(Intercept)')
            self.intercept_val = int(rho[intercept_ind])
            rho_values = np.delete(rho_values, intercept_ind)
            rho_names.remove('(Intercept)')
        else:
            self.intercept_val = 0

        self.filter_mask = np.array(rho_values) != 0

        # removes zero values
        if not self.show_omitted_variables:
            selected_ind = np.flatnonzero(rho_values)
            self.rho_values = rho_values[selected_ind]
            self.rho_names = [rho_names[i] for i in selected_ind]

        return self
Example #33
0
    def read(self,
             t=None,
             z=None,
             y=None,
             x=None,
             N=None,
             K=None,
             J=None,
             I=None,
             var=None,
             nonan=True,
             result='full',
             profile=False,
             dummy=False):
        """Reads dataset.

        PARAMETERS
            t, z, y, x (array like, optional) :
                Sets the time, height, latitude and longitude for which
                the data will be read.
            N, K, J, I (array like, optional) :
                Sets the temporal, vertical, meridional and zonal
                indices for which the data will be read.
            var (string, optional) :
                Indicates which variable of the grid will be read. If
                the parameter is a list of variables, then the data will
                be returned as a list of arrays.
            nonan (boolean, optional) :
                If set to true (default) changes data values containing
                NaN to zero, preserving the mask.
            result (string, optional) :
                Determines wheter all time, height, latitude, longitude
                and data will be returned ('full', default), if
                temporal, vertical, meridional and zonal indices
                are returned instead ('indices'), or if only
                variable data is returned ('var only').
            components (list, optional) :
                A list containing which components will be included in
                the calculation. Options are the seasonal cycle
                ('seasonal'), westward propagating planetary waves
                ('planetary'), eddy fields ('eddy') and noise ('noise').
            profile (boolean, optional) :
                Sets whether the status is send to screen.
            dummy (boolean, optional) :
                If set to true, does not load data and returns the shape
                of the array that would have been returned.

        RETURNS
            t, z, y, x, dat (array like) :
                If 'result' is set to 'full', then all coordinates and
                data variables are returned.
            N, K, J, I, var (array like) :
                If 'result' is set to 'indices', then all indices and
                data variables are returned.
            dat (array like) :
                If 'result' is set to 'var only', then the data is
                returned.

        """
        global DEBUG
        t1 = time()

        # Checks input variables for consistency.
        if (t != None) & (N != None):
            raise ValueError('Both time and temporal index were provided.')
        if (z != None) & (K != None):
            raise ValueError('Both height and vertical index were provided.')
        if (y != None) & (J != None):
            raise ValueError(
                'Both latitude and meridional index were provided.')
        if (x != None) & (I != None):
            raise ValueError('Both latitude and zonal index were provided.')

        # Checks for variables indices. Intersects desired input values with
        # dataset dimesion data. In this dataset, since only surface data is
        # available, the height values are always zero.
        if t != None:
            N = flatnonzero(in1d(self.variables['time'].data, t))
        elif N == None:
            N = arange(self.dimensions['n'])
        if z != None:
            K = [0]
        elif K == None:
            K = [0]
        elif K != None:
            K = [0]
        if y != None:
            J = flatnonzero(in1d(self.variables['latitude'].data, y))
        elif J == None:
            J = arange(self.dimensions['j'])
        if x != None:
            I = flatnonzero(in1d(self.variables['longitude'].data, x))
        elif I == None:
            I = arange(self.dimensions['i'])

        # Sets the shape of the data array.
        shape = (len(N), 1, len(J), len(I))
        if dummy:
            return shape
        # Selects data according to indices.
        t = self.variables['time'].data[N]
        z = self.variables['height'].data
        y = self.variables['latitude'].data[J]
        x = self.variables['longitude'].data[I]
        xx, yy = meshgrid(x, y)
        II, JJ = meshgrid(I, J)
        var = ma.zeros(shape)
        # Walks through every time index and loads data range from maps.
        for n, T in enumerate(t):
            t2 = time()
            if profile:
                s = '\rLoading data... %s ' % (profiler(
                    shape[0], n + 1, 0, t1, t2), )
                stdout.write(s)
                stdout.flush()
            # Uncompresses and reads HDF file
            HDF = self._open_HDF(
                '%s/%s' %
                (self.params['path'], self.params['file_list'][N[n]]))
            HDF_att = HDF.attributes()
            # Loads scientific dataset (SDS) and calculates the parameter
            # value using scalling equation, slope and intercept.
            SDS_name = HDF.datasets().keys()[0]
            SDS = HDF.select(SDS_name)
            SDS_att = SDS.attributes()
            if (('lon_i' in self.params.keys()) &
                ('lat_j' in self.params.keys())):
                P = SDS[:, :][self.params['lat_j'], self.params['lon_i']][JJ,
                                                                          II]
            else:
                P = SDS[:, :][JJ, II]
            P[P <= SDS_att['Fill']] = nan
            P = (SDS_att['Slope'] * P + SDS_att['Intercept'])
            P = ma.masked_where(isnan(P), P)
            if nonan:
                P.data[P.mask] = 0
            #
            var[n, 0, :, :] = P[None, None, :, :]

        if profile:
            stdout.write('\r\n')
            stdout.flush()

        if DEBUG:
            print 't: ', t
            print 'z: ', z
            print 'y:', y
            print 'x:', x
            print 'var: ', var
            print 'N: ', N
            print 'K: ', K
            print 'J: ', J
            print 'I:', I
            print 'shape: ', shape

        if result == 'full':
            return t, z, y, x, var
        elif result == 'indices':
            return N, K, J, I, var
        elif result == 'var only':
            return var
        else:
            raise Warning("Result parameter set imporperly to '%s', "
                          "assuming 'var only'." % (result))
            return var
Example #34
0
File: ops.py Project: zeneli/pandas
 def reconstructed_codes(self) -> List[np.ndarray]:
     # get unique result indices, and prepend 0 as groupby starts from the first
     return [np.r_[0, np.flatnonzero(self.bins[1:] != self.bins[:-1]) + 1]]
Example #35
0
def onset_backtrack(events, energy):
    '''Backtrack detected onset events to the nearest preceding local
    minimum of an energy function.

    This function can be used to roll back the timing of detected onsets
    from a detected peak amplitude to the preceding minimum.

    This is most useful when using onsets to determine slice points for
    segmentation, as described by [1]_.

    .. [1] Jehan, Tristan.
           "Creating music by listening"
           Doctoral dissertation
           Massachusetts Institute of Technology, 2005.

    Parameters
    ----------
    events : np.ndarray, dtype=int
        List of onset event frame indices, as computed by `onset_detect`

    energy : np.ndarray, shape=(m,)
        An energy function

    Returns
    -------
    events_backtracked : np.ndarray, shape=events.shape
        The input events matched to nearest preceding minima of `energy`.

    Examples
    --------
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      offset=30, duration=2.0)
    >>> oenv = librosa.onset.onset_strength(y=y, sr=sr)
    >>> # Detect events without backtracking
    >>> onset_raw = librosa.onset.onset_detect(onset_envelope=oenv,
    ...                                        backtrack=False)
    >>> # Backtrack the events using the onset envelope
    >>> onset_bt = librosa.onset.onset_backtrack(onset_raw, oenv)
    >>> # Backtrack the events using the RMS energy
    >>> rmse = librosa.feature.rmse(S=np.abs(librosa.stft(y=y)))
    >>> onset_bt_rmse = librosa.onset.onset_backtrack(onset_raw, rmse[0])

    >>> # Plot the results
    >>> import matplotlib.pyplot as plt
    >>> plt.figure()
    >>> plt.subplot(2,1,1)
    >>> plt.plot(oenv, label='Onset strength')
    >>> plt.vlines(onset_raw, 0, oenv.max(), label='Raw onsets')
    >>> plt.vlines(onset_bt, 0, oenv.max(), label='Backtracked', color='r')
    >>> plt.legend(frameon=True, framealpha=0.75)
    >>> plt.subplot(2,1,2)
    >>> plt.plot(rmse[0], label='RMSE')
    >>> plt.vlines(onset_bt_rmse, 0, rmse.max(), label='Backtracked (RMSE)', color='r')
    >>> plt.legend(frameon=True, framealpha=0.75)
    '''

    # Find points where energy is non-increasing
    # all points:  energy[i] <= energy[i-1]
    # tail points: energy[i] < energy[i+1]
    minima = np.flatnonzero((energy[1:-1] <= energy[:-2])
                            & (energy[1:-1] < energy[2:]))

    # Pad on a 0, just in case we have onsets with no preceding minimum
    # Shift by one to account for slicing in minima detection
    minima = util.fix_frames(1 + minima, x_min=0)

    # Only match going left from the detected events
    return minima[util.match_events(events, minima, right=False)]
Example #36
0
    def plot(self,
             column=None,
             plot_close=True,
             plot_zones=True,
             close_trace_kwargs=None,
             entry_trace_kwargs=None,
             exit_trace_kwargs=None,
             exit_profit_trace_kwargs=None,
             exit_loss_trace_kwargs=None,
             active_trace_kwargs=None,
             profit_shape_kwargs=None,
             loss_shape_kwargs=None,
             add_trace_kwargs=None,
             xref='x', yref='y',
             fig=None,
             **layout_kwargs):  # pragma: no cover
        """Plot orders.

        Args:
            column (str): Name of the column to plot.
            plot_close (bool): Whether to plot `Trades.close`.
            plot_zones (bool): Whether to plot zones.

                Set to False if there are many trades within one position.
            close_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for `Trades.close`.
            entry_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Entry" markers.
            exit_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit" markers.
            exit_profit_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit - Profit" markers.
            exit_loss_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit - Loss" markers.
            active_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Active" markers.
            profit_shape_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Figure.add_shape` for profit zones.
            loss_shape_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Figure.add_shape` for loss zones.
            add_trace_kwargs (dict): Keyword arguments passed to `add_trace`.
            xref (str): X coordinate axis.
            yref (str): Y coordinate axis.
            fig (plotly.graph_objects.Figure): Figure to add traces to.
            **layout_kwargs: Keyword arguments for layout.

        ## Example

        ```python-repl
        >>> trades.plot()
        ```

        ![](/vectorbt/docs/img/trades_plot.png)"""
        from vectorbt.settings import color_schema, contrast_color_schema

        self_col = self.select_series(column=column, group_by=False)

        if close_trace_kwargs is None:
            close_trace_kwargs = {}
        close_trace_kwargs = merge_dicts(dict(
            line_color=color_schema['blue'],
            name='Close' if self_col.wrapper.name is None else self_col.wrapper.name
        ), close_trace_kwargs)
        if entry_trace_kwargs is None:
            entry_trace_kwargs = {}
        if exit_trace_kwargs is None:
            exit_trace_kwargs = {}
        if exit_profit_trace_kwargs is None:
            exit_profit_trace_kwargs = {}
        if exit_loss_trace_kwargs is None:
            exit_loss_trace_kwargs = {}
        if active_trace_kwargs is None:
            active_trace_kwargs = {}
        if profit_shape_kwargs is None:
            profit_shape_kwargs = {}
        if loss_shape_kwargs is None:
            loss_shape_kwargs = {}
        if add_trace_kwargs is None:
            add_trace_kwargs = {}

        if fig is None:
            fig = FigureWidget()
        fig.update_layout(**layout_kwargs)

        # Plot close
        if plot_close:
            fig = self_col.close.vbt.plot(trace_kwargs=close_trace_kwargs, add_trace_kwargs=add_trace_kwargs, fig=fig)

        if len(self_col.values) > 0:
            # Extract information
            _id = self_col.values['id']
            _id_str = 'Trade Id' if self.trade_type == TradeType.Trade else 'Position Id'
            size = self_col.values['size']
            entry_idx = self_col.values['entry_idx']
            entry_price = self_col.values['entry_price']
            entry_fees = self_col.values['entry_fees']
            exit_idx = self_col.values['exit_idx']
            exit_price = self_col.values['exit_price']
            exit_fees = self_col.values['exit_fees']
            pnl = self_col.values['pnl']
            ret = self_col.values['return']
            direction_value_map = to_value_map(TradeDirection)
            direction = self_col.values['direction']
            direction = np.vectorize(lambda x: str(direction_value_map[x]))(direction)
            status = self_col.values['status']

            def get_duration_str(from_idx, to_idx):
                if isinstance(self_col.wrapper.index, DatetimeTypes):
                    duration = self_col.wrapper.index[to_idx] - self_col.wrapper.index[from_idx]
                elif self_col.wrapper.freq is not None:
                    duration = self_col.wrapper.to_time_units(to_idx - from_idx)
                else:
                    duration = to_idx - from_idx
                return np.vectorize(str)(duration)

            duration = get_duration_str(entry_idx, exit_idx)

            if len(entry_idx) > 0:
                # Plot Entry markers
                entry_customdata = np.stack((
                    _id,
                    size,
                    entry_fees,
                    direction,
                    *((self_col.values['position_id'],)
                      if self.trade_type == TradeType.Trade else ())
                ), axis=1)
                entry_scatter = go.Scatter(
                    x=self_col.wrapper.index[entry_idx],
                    y=entry_price,
                    mode='markers',
                    marker=dict(
                        symbol='square',
                        color=contrast_color_schema['blue'],
                        size=7,
                        line=dict(
                            width=1,
                            color=adjust_lightness(contrast_color_schema['blue'])
                        )
                    ),
                    name='Entry',
                    customdata=entry_customdata,
                    hovertemplate=_id_str + ": %{customdata[0]}"
                                            "<br>Date: %{x}"
                                            "<br>Avg. Price: %{y}"
                                            "<br>Size: %{customdata[1]:.6f}"
                                            "<br>Fees: %{customdata[2]:.6f}"
                                            "<br>Direction: %{customdata[3]}"
                                  + ("<br>Position Id: %{customdata[4]}"
                                     if self.trade_type == TradeType.Trade else '')
                )
                entry_scatter.update(**entry_trace_kwargs)
                fig.add_trace(entry_scatter, **add_trace_kwargs)

            # Plot end markers
            def _plot_end_markers(mask, name, color, kwargs):
                if np.any(mask):
                    customdata = np.stack((
                        _id[mask],
                        duration[mask],
                        size[mask],
                        exit_fees[mask],
                        pnl[mask],
                        ret[mask],
                        direction[mask],
                        *((self_col.values['position_id'][mask],)
                          if self.trade_type == TradeType.Trade else ())
                    ), axis=1)
                    scatter = go.Scatter(
                        x=self_col.wrapper.index[exit_idx[mask]],
                        y=exit_price[mask],
                        mode='markers',
                        marker=dict(
                            symbol='square',
                            color=color,
                            size=7,
                            line=dict(
                                width=1,
                                color=adjust_lightness(color)
                            )
                        ),
                        name=name,
                        customdata=customdata,
                        hovertemplate=_id_str + ": %{customdata[0]}"
                                                "<br>Date: %{x}"
                                                "<br>Duration: %{customdata[1]}"
                                                "<br>Avg. Price: %{y}"
                                                "<br>Size: %{customdata[2]:.6f}"
                                                "<br>Fees: %{customdata[3]:.6f}"
                                                "<br>PnL: %{customdata[4]:.6f}"
                                                "<br>Return: %{customdata[5]:.2%}"
                                                "<br>Direction: %{customdata[6]}"
                                      + ("<br>Position Id: %{customdata[7]}"
                                         if self.trade_type == TradeType.Trade else '')
                    )
                    scatter.update(**kwargs)
                    fig.add_trace(scatter, **add_trace_kwargs)

            # Plot Exit markers
            _plot_end_markers(
                (status == TradeStatus.Closed) & (pnl == 0.),
                'Exit',
                contrast_color_schema['gray'],
                exit_trace_kwargs
            )

            # Plot Exit - Profit markers
            _plot_end_markers(
                (status == TradeStatus.Closed) & (pnl > 0.),
                'Exit - Profit',
                contrast_color_schema['green'],
                exit_profit_trace_kwargs
            )

            # Plot Exit - Loss markers
            _plot_end_markers(
                (status == TradeStatus.Closed) & (pnl < 0.),
                'Exit - Loss',
                contrast_color_schema['red'],
                exit_loss_trace_kwargs
            )

            # Plot Active markers
            _plot_end_markers(
                status == TradeStatus.Open,
                'Active',
                contrast_color_schema['orange'],
                active_trace_kwargs
            )

            if plot_zones:
                profit_mask = pnl > 0.
                if np.any(profit_mask):
                    # Plot profit zones
                    for i in np.flatnonzero(profit_mask):
                        fig.add_shape(**merge_dicts(dict(
                            type="rect",
                            xref=xref,
                            yref=yref,
                            x0=self_col.wrapper.index[entry_idx[i]],
                            y0=entry_price[i],
                            x1=self_col.wrapper.index[exit_idx[i]],
                            y1=exit_price[i],
                            fillcolor='green',
                            opacity=0.2,
                            layer="below",
                            line_width=0,
                        ), profit_shape_kwargs))

                loss_mask = pnl < 0.
                if np.any(loss_mask):
                    # Plot loss zones
                    for i in np.flatnonzero(loss_mask):
                        fig.add_shape(**merge_dicts(dict(
                            type="rect",
                            xref=xref,
                            yref=yref,
                            x0=self_col.wrapper.index[entry_idx[i]],
                            y0=entry_price[i],
                            x1=self_col.wrapper.index[exit_idx[i]],
                            y1=exit_price[i],
                            fillcolor='red',
                            opacity=0.2,
                            layer="below",
                            line_width=0,
                        ), loss_shape_kwargs))

        return fig
Example #37
0
    def _send_procs_to_workers(self, updatehash=False, graph=None):
        """
        Sends jobs to workers when system resources are available.
        """

        # Check to see if a job is available (jobs with all dependencies run)
        # See https://github.com/nipy/nipype/pull/2200#discussion_r141605722
        # See also https://github.com/nipy/nipype/issues/2372
        jobids = np.flatnonzero(~self.proc_done
                                & (self.depidx.sum(axis=0) == 0).__array__())

        # Check available resources by summing all threads and memory used
        free_memory_gb, free_processors = self._check_resources(
            self.pending_tasks)

        stats = (
            len(self.pending_tasks),
            len(jobids),
            free_memory_gb,
            self.memory_gb,
            free_processors,
            self.processors,
        )
        if self._stats != stats:
            tasks_list_msg = ""

            if logger.level <= INFO:
                running_tasks = [
                    "  * %s" % self.procs[jobid].fullname
                    for _, jobid in self.pending_tasks
                ]
                if running_tasks:
                    tasks_list_msg = "\nCurrently running:\n"
                    tasks_list_msg += "\n".join(running_tasks)
                    tasks_list_msg = indent(tasks_list_msg, " " * 21)
            logger.info(
                "[MultiProc] Running %d tasks, and %d jobs ready. Free "
                "memory (GB): %0.2f/%0.2f, Free processors: %d/%d.%s",
                len(self.pending_tasks),
                len(jobids),
                free_memory_gb,
                self.memory_gb,
                free_processors,
                self.processors,
                tasks_list_msg,
            )
            self._stats = stats

        if free_memory_gb < 0.01 or free_processors == 0:
            logger.debug("No resources available")
            return

        if len(jobids) + len(self.pending_tasks) == 0:
            logger.debug("No tasks are being run, and no jobs can "
                         "be submitted to the queue. Potential deadlock")
            return

        jobids = self._sort_jobs(jobids,
                                 scheduler=self.plugin_args.get("scheduler"))

        # Run garbage collector before potentially submitting jobs
        gc.collect()

        # Submit jobs
        for jobid in jobids:
            # First expand mapnodes
            if isinstance(self.procs[jobid], MapNode):
                try:
                    num_subnodes = self.procs[jobid].num_subnodes()
                except Exception:
                    traceback = format_exception(*sys.exc_info())
                    self._clean_queue(jobid,
                                      graph,
                                      result={
                                          "result": None,
                                          "traceback": traceback
                                      })
                    self.proc_pending[jobid] = False
                    continue
                if num_subnodes > 1:
                    submit = self._submit_mapnode(jobid)
                    if not submit:
                        continue

            # Check requirements of this job
            next_job_gb = min(self.procs[jobid].mem_gb, self.memory_gb)
            next_job_th = min(self.procs[jobid].n_procs, self.processors)

            # If node does not fit, skip at this moment
            if next_job_th > free_processors or next_job_gb > free_memory_gb:
                logger.debug(
                    "Cannot allocate job %d (%0.2fGB, %d threads).",
                    jobid,
                    next_job_gb,
                    next_job_th,
                )
                continue

            free_memory_gb -= next_job_gb
            free_processors -= next_job_th
            logger.debug(
                "Allocating %s ID=%d (%0.2fGB, %d threads). Free: "
                "%0.2fGB, %d threads.",
                self.procs[jobid].fullname,
                jobid,
                next_job_gb,
                next_job_th,
                free_memory_gb,
                free_processors,
            )

            # change job status in appropriate queues
            self.proc_done[jobid] = True
            self.proc_pending[jobid] = True

            # If cached and up-to-date just retrieve it, don't run
            if self._local_hash_check(jobid, graph):
                continue

            # updatehash and run_without_submitting are also run locally
            if updatehash or self.procs[jobid].run_without_submitting:
                logger.debug("Running node %s on master thread",
                             self.procs[jobid])
                try:
                    self.procs[jobid].run(updatehash=updatehash)
                except Exception:
                    traceback = format_exception(*sys.exc_info())
                    self._clean_queue(jobid,
                                      graph,
                                      result={
                                          "result": None,
                                          "traceback": traceback
                                      })

                # Release resources
                self._task_finished_cb(jobid)
                self._remove_node_dirs()
                free_memory_gb += next_job_gb
                free_processors += next_job_th
                # Display stats next loop
                self._stats = None

                # Clean up any debris from running node in main process
                gc.collect()
                continue

            # Task should be submitted to workers
            # Send job to task manager and add to pending tasks
            if self._status_callback:
                self._status_callback(self.procs[jobid], "start")
            tid = self._submit_job(deepcopy(self.procs[jobid]),
                                   updatehash=updatehash)
            if tid is None:
                self.proc_done[jobid] = False
                self.proc_pending[jobid] = False
            else:
                self.pending_tasks.insert(0, (tid, jobid))
            # Display stats next loop
            self._stats = None
        if I_minute[C_i] == 0:
            #slice out BT images for the current basin
            C_BTemp = xr.open_dataset(IRDIR + "\\" +
                                      BTemp_filename)['Tb'].values[0][
                                          DIM_BOUND[0]:DIM_BOUND[1] + 1,
                                          DIM_BOUND[2]:DIM_BOUND[3] + 1]
            C_lat = xr.open_dataset(
                IRDIR + "\\" +
                BTemp_filename)['lat'].values[DIM_BOUND[0]:DIM_BOUND[1] + 1]
            C_lon = xr.open_dataset(
                IRDIR + "\\" +
                BTemp_filename)['lon'].values[DIM_BOUND[2]:DIM_BOUND[3] + 1]
            #interpolate NaN values in BT images
            mask = np.isnan(C_BTemp)
            C_BTemp[mask] = np.interp(np.flatnonzero(mask),
                                      np.flatnonzero(~mask), C_BTemp[~mask])

        elif I_minute[C_i] == 30:
            #slice out BT images for the current basin
            C_BTemp = xr.open_dataset(IRDIR + "\\" +
                                      BTemp_filename)['Tb'].values[1][
                                          DIM_BOUND[0]:DIM_BOUND[1] + 1,
                                          DIM_BOUND[2]:DIM_BOUND[3] + 1]
            C_lat = xr.open_dataset(
                IRDIR + "\\" +
                BTemp_filename)['lat'].values[DIM_BOUND[0]:DIM_BOUND[1] + 1]
            C_lon = xr.open_dataset(
                IRDIR + "\\" +
                BTemp_filename)['lon'].values[DIM_BOUND[2]:DIM_BOUND[3] + 1]
            #interpolate NaN values in BT images
def digest_in_buckets(v, buckets_number):
    digested = set()
    for i in np.flatnonzero(v).flat:
        digested.add(i % buckets_number)
    return tuple(digested)
Example #40
0
    def unsaturated_case_3(self, dt, ia, case_3):
        """
        Solve Green-Ampt model for second unsaturated case:

        1) Soil is unsaturated
        2) Available rainfall is greater than saturated hydraulic conductivity

        Inputs:
        -------
        dt: float
            Time step (seconds)
        ia: np.ndarray (float)
            Available rainfall (m/s)
        case_3: np.ndarray (bool)
            Indicates whether case 3 applies to given element
        """
        # 5. If availble rainfall rate exceeds saturated hydr. conductivity
        orig_ia = np.copy(ia)
        ia = ia[case_3]
        is_saturated = self.is_saturated[case_3]
        Ks = self.Ks[case_3]
        Lu = self.Lu[case_3]
        theta_dmax = self.theta_dmax[case_3]
        psi_f = self.psi_f[case_3]
        theta_d = self.theta_d[case_3]
        # Variables to write
        F = self.F[case_3]
        f = self.f[case_3]
        theta_du = self.theta_du[case_3]
        # Reset recovery time
        self.T[case_3] = self.Tr[case_3]
        # Compute volume needed to saturate surface layer
        Fs = Ks * psi_f * theta_d / (ia - Ks)
        cond_0 = (F >= Fs)
        cond_1 = (F + ia * dt < Fs) & (~cond_0)
        cond_2 = (~cond_0 & ~cond_1)
        # For first case
        if cond_0.any():
            # Set to saturated
            is_saturated[cond_0] = True
            self.is_saturated[case_3] = is_saturated
            # Run saturated case
            # TODO: This is kind of confusing and probably not efficient
            sat_case = np.zeros(self.N, dtype=bool)
            sat_case[np.flatnonzero(case_3)[np.flatnonzero(cond_0)]] = True
            self.saturated_case(dt, orig_ia, sat_case)
            F[cond_0] = self.F[case_3][cond_0]
            f[cond_0] = self.f[case_3][cond_0]
            theta_du[cond_0] = self.theta_du[case_3][cond_0]
        # Run second case
        if cond_1.any():
            f[cond_1] = ia[cond_1]
            dF = ia[cond_1] * dt
            F[cond_1] += dF
            theta_du[cond_1] -= dF / Lu[cond_1]
            theta_du[cond_1] = np.minimum(np.maximum(theta_du[cond_1], 0),
                                                theta_dmax[cond_1])
        # Run third case
        # Solve integrated equation
        if cond_2.any():
            sub_dt = dt - (Fs[cond_2] - F[cond_2]) / ia[cond_2]
            n = sub_dt.size
            F_2 = np.zeros(n, dtype=float)
            # Run green ampt
            F_2 = run_green_ampt_newton(F_2, F[cond_2], Fs[cond_2], sub_dt, Ks[cond_2],
                                        theta_d[cond_2], psi_f[cond_2], ia[cond_2])
            dF = F_2 - F[cond_2]
            F[cond_2] = F_2
            theta_du[cond_2] -= dF / Lu[cond_2]
            theta_du[cond_2] = np.minimum(np.maximum(theta_du[cond_2], 0),
                                                theta_dmax[cond_2])
            f[cond_2] = dF / dt
        # Export instance variables
        self.F[case_3] = F
        self.f[case_3] = f
        self.theta_du[case_3] = theta_du
Example #41
0
def convertNansToZeros(ma):
    nan_elements = np.flatnonzero(np.isnan(ma.data))
    if len(nan_elements) > 0:
        ma.data[nan_elements] = 0.0
    return ma
Example #42
0
def _join(left,
          right,
          keys=None,
          join_type='inner',
          uniq_col_name='{col_name}_{table_name}',
          table_names=['1', '2'],
          col_name_map=None,
          metadata_conflicts='warn'):
    """
    Perform a join of the left and right Tables on specified keys.

    Parameters
    ----------
    left : Table
        Left side table in the join
    right : Table
        Right side table in the join
    keys : str or list of str
        Name(s) of column(s) used to match rows of left and right tables.
        Default is to use all columns which are common to both tables.
    join_type : str
        Join type ('inner' | 'outer' | 'left' | 'right'), default is 'inner'
    uniq_col_name : str or None
        String generate a unique output column name in case of a conflict.
        The default is '{col_name}_{table_name}'.
    table_names : list of str or None
        Two-element list of table names used when generating unique output
        column names.  The default is ['1', '2'].
    col_name_map : empty dict or None
        If passed as a dict then it will be updated in-place with the
        mapping of output to input column names.

    Returns
    -------
    joined_table : `~astropy.table.Table` object
        New table containing the result of the join operation.
    """
    # Store user-provided col_name_map until the end
    _col_name_map = col_name_map

    if join_type not in ('inner', 'outer', 'left', 'right'):
        raise ValueError(
            "The 'join_type' argument should be in 'inner', "
            "'outer', 'left' or 'right' (got '{}' instead)".format(join_type))

    # If we have a single key, put it in a tuple
    if keys is None:
        keys = tuple(name for name in left.colnames if name in right.colnames)
        if len(keys) == 0:
            raise TableMergeError(
                'No keys in common between left and right tables')
    elif isinstance(keys, str):
        keys = (keys, )

    # Check the key columns
    for arr, arr_label in ((left, 'Left'), (right, 'Right')):
        for name in keys:
            if name not in arr.colnames:
                raise TableMergeError(
                    '{} table does not have key column {!r}'.format(
                        arr_label, name))
            if hasattr(arr[name], 'mask') and np.any(arr[name].mask):
                raise TableMergeError(
                    '{} key column {!r} has missing values'.format(
                        arr_label, name))
            if not isinstance(arr[name], np.ndarray):
                raise ValueError(
                    "non-ndarray column '{}' not allowed as a key column".
                    format(name))

    len_left, len_right = len(left), len(right)

    if len_left == 0 or len_right == 0:
        raise ValueError(
            'input tables for join must both have at least one row')

    # Joined array dtype as a list of descr (name, type_str, shape) tuples
    col_name_map = get_col_name_map([left, right], keys, uniq_col_name,
                                    table_names)
    out_descrs = get_descrs([left, right], col_name_map)

    # Make an array with just the key columns.  This uses a temporary
    # structured array for efficiency.
    out_keys_dtype = [descr for descr in out_descrs if descr[0] in keys]
    out_keys = np.empty(len_left + len_right, dtype=out_keys_dtype)
    for key in keys:
        out_keys[key][:len_left] = left[key]
        out_keys[key][len_left:] = right[key]
    idx_sort = out_keys.argsort(order=keys)
    out_keys = out_keys[idx_sort]

    # Get all keys
    diffs = np.concatenate(([True], out_keys[1:] != out_keys[:-1], [True]))
    idxs = np.flatnonzero(diffs)

    # Main inner loop in Cython to compute the cartesian product
    # indices for the given join type
    int_join_type = {'inner': 0, 'outer': 1, 'left': 2, 'right': 3}[join_type]
    masked, n_out, left_out, left_mask, right_out, right_mask = \
        _np_utils.join_inner(idxs, idx_sort, len_left, int_join_type)

    out = _get_out_class([left, right])()

    for out_name, dtype, shape in out_descrs:

        left_name, right_name = col_name_map[out_name]
        if left_name and right_name:  # this is a key which comes from left and right
            cols = [left[left_name], right[right_name]]

            col_cls = _get_out_class(cols)
            if not hasattr(col_cls.info, 'new_like'):
                raise NotImplementedError(
                    'join unavailable for mixin column type(s): {}'.format(
                        col_cls.__name__))

            out[out_name] = col_cls.info.new_like(cols, n_out,
                                                  metadata_conflicts, out_name)

            if issubclass(col_cls, Column):
                out[out_name][:] = np.where(right_mask,
                                            left[left_name].take(left_out),
                                            right[right_name].take(right_out))
            else:
                # np.where does not work for mixin columns (e.g. Quantity) so
                # use a slower workaround.
                left_mask = ~right_mask
                if np.any(left_mask):
                    out[out_name][left_mask] = left[left_name].take(left_out)
                if np.any(right_mask):
                    out[out_name][right_mask] = right[right_name].take(
                        right_out)
            continue
        elif left_name:  # out_name came from the left table
            name, array, array_out, array_mask = left_name, left, left_out, left_mask
        elif right_name:
            name, array, array_out, array_mask = right_name, right, right_out, right_mask
        else:
            raise TableMergeError('Unexpected column names (maybe one is ""?)')

        # Select the correct elements from the original table
        col = array[name][array_out]

        # If the output column is masked then set the output column masking
        # accordingly.  Check for columns that don't support a mask attribute.
        if masked and np.any(array_mask):
            # If col is a Column but not MaskedColumn then upgrade at this point
            # because masking is required.
            if isinstance(col, Column) and not isinstance(col, MaskedColumn):
                col = out.MaskedColumn(col, copy=False)

            # array_mask is 1-d corresponding to length of output column.  We need
            # make it have the correct shape for broadcasting, i.e. (length, 1, 1, ..).
            # Mixin columns might not have ndim attribute so use len(col.shape).
            array_mask.shape = (col.shape[0], ) + (1, ) * (len(col.shape) - 1)

            # Now broadcast to the correct final shape
            array_mask = np.broadcast_to(array_mask, col.shape)

            try:
                col[array_mask] = col.info.mask_val
            except Exception:  # Not clear how different classes will fail here
                raise NotImplementedError(
                    "join requires masking column '{}' but column"
                    " type {} does not support masking".format(
                        out_name, col.__class__.__name__))

        # Set the output table column to the new joined column
        out[out_name] = col

    # If col_name_map supplied as a dict input, then update.
    if isinstance(_col_name_map, Mapping):
        _col_name_map.update(col_name_map)

    return out
# submission = submission.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)
# for i in range(len(submission['Confidence'].values)):
#     if submission['Confidence'][i] < df_final['Confidence'][i]:
#         df_final['Confidence'][i] = 0.8*df_final['Confidence'][i] + 0.2*submission['Confidence'][i]


# print(df_final)

test = pd.read_csv(PATH+'test.csv')
for i in range(len(test.values)):
    week = test['Weeks'][i]
    patient = test['Patient'][i]
    FVC = test['FVC'][i]
   
    mask = df_final['Patient_Week'] == str(patient+'_'+str(week))
    pos = np.flatnonzero(mask)
    FVC_pre = df_final['FVC'][int(pos)]
    # print(pos)
    # print(FVC)
    # print(FVC_pre)
    if (FVC_pre - FVC) > 100:
        mask = df_final['Patient_Week'] == str(patient+'_'+'131')
        pos = np.flatnonzero(mask)
        df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5)
        df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))
        print('131_week',df_final['FVC'][int(pos)])
        
        mask = df_final['Patient_Week'] == str(patient+'_'+'132')
        pos = np.flatnonzero(mask)
        df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5)
        df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))
Example #44
0
def convertNansToOnes(pArray):
    nan_elements = np.flatnonzero(np.isnan(pArray))
    if len(nan_elements) > 0:
        pArray[nan_elements] = 1.0
    return pArray
                                0,
                                samples_eig_vecs,
                                weight_type='linear',
                                norm='L2',
                                int_type='CI',
                                bounds='upper')
signal_weighted_adjacency_matrix = weighted_adjacency_matrix[
    reject_dict['signal_inds']][:, reject_dict['signal_inds']]

print(dt.datetime.now().isoformat() + ' INFO: ' +
      'Constructing final signal network without leaves...')
biggest_signal_comp, biggest_signal_inds, biggest_signal_assing, biggest_signal_size = nnr.getBiggestComponent(
    signal_weighted_adjacency_matrix)
signal_comp_inds = reject_dict['signal_inds'][biggest_signal_inds]
strength_distn = biggest_signal_comp.sum(axis=0)
leaf_inds = np.flatnonzero(strength_distn == 1)
keep_inds = np.flatnonzero(strength_distn > 1)
signal_final_inds = signal_comp_inds[keep_inds]
signal_leaf_inds = signal_comp_inds[leaf_inds]
final_weighted_adjacency_matrix = biggest_signal_comp[keep_inds][:, keep_inds]

saveVariablesForClustering(
    data_file_name.split('.')[0], npy_dir, reject_dict,
    weighted_adjacency_matrix, exceeding_space_dims, signal_final_inds,
    expected_wcm, final_weighted_adjacency_matrix,
    loadNodeLabels(data_file_name, data))

print(dt.datetime.now().isoformat() + ' INFO: ' +
      'Plotting modularity matrix eigenspectrum against null eigenspectrum...')
plt.figure()
nnr.plotModEigValsVsNullEigHist(network_modularity_matrix, samples_eig_vals)
Example #46
0
def chanvese(I,
             init_mask,
             max_its=200,
             alpha=0.2,
             thresh=0,
             color='r',
             display=False):
    I = I.astype(np.float)

    # Create a signed distance map (SDF) from mask
    phi = mask2phi(init_mask)

    if display:
        plt.ion()
        fig, axes = plt.subplots(ncols=2)
        show_curve_and_phi(fig, I, phi, color)
        #plt.savefig('levelset_start.png', bbox_inches='tight')

    # Main loop
    its = 0
    stop = False
    prev_mask = init_mask
    c = 0

    while (its < max_its and not stop):
        # Get the curve's narrow band
        idx = np.flatnonzero(np.logical_and(phi <= 1.2, phi >= -1.2))

        if len(idx) > 0:
            # Intermediate output
            if display:
                if np.mod(its, 50) == 0:
                    print('iteration: {0}'.format(its))
                    show_curve_and_phi(fig, I, phi, color)
            else:
                if np.mod(its, 10) == 0:
                    print('iteration: {0}'.format(its))

            # Find interior and exterior mean
            upts = np.flatnonzero(phi <= 0)  # interior points
            vpts = np.flatnonzero(phi > 0)  # exterior points
            u = np.sum(I.flat[upts]) / (len(upts) + eps)  # interior mean
            v = np.sum(I.flat[vpts]) / (len(vpts) + eps)  # exterior mean

            # Force from image information
            F = (I.flat[idx] - u)**2 - (I.flat[idx] - v)**2
            # Force from curvature penalty
            curvature = get_curvature(phi, idx)

            # Gradient descent to minimize energy
            dphidt = F / np.max(np.abs(F)) + alpha * curvature

            # Maintain the CFL condition
            dt = 0.45 / (np.max(np.abs(dphidt)) + eps)

            # Evolve the curve
            phi.flat[idx] += dt * dphidt

            # Keep SDF smooth
            phi = sussman(phi, 0.5)

            new_mask = phi <= 0
            c = convergence(prev_mask, new_mask, thresh, c)

            if c <= 5:
                its = its + 1
                prev_mask = new_mask
            else:
                stop = True

        else:
            break

    # Final output
    if display:
        show_curve_and_phi(fig, I, phi, color)
        #plt.savefig('levelset_end.png', bbox_inches='tight')

    # Make mask from SDF
    seg = phi <= 0  # Get mask from levelset

    return seg, phi, its
Example #47
0
def get_topup_inputs_from(dwi_file,
                          bval_file,
                          b0_threshold,
                          topup_prefix,
                          bids_origin_files,
                          epi_fmaps=None,
                          max_per_spec=3,
                          topup_requested=False):
    """Create a datain spec and a slspec from a concatenated dwi series.

    Create inputs for TOPUP that come from data in ``dwi/`` and epi fieldmaps in ``fmap/``.
    The ``nii_file`` input may be the result of concatenating a number of scans with different
    distortions present. The original source of each volume in ``nii_file`` is listed in
    ``bids_origin_files``.

    The strategy is to select ``max_per_spec`` b=0 images from each distortion group.
    Here, distortion group uses the FSL definition of a phase encoding direction and
    total readout time, as specified in the datain file used by TOPUP (i.e. "0 -1 0 0.087").


    Case: Two opposing PE direction dwi series
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

    For example if the following b=0 images are found at the following indices into
    ``nii_file``:

    ============ ============================= ==================
    Image Index  BIDS source file for a b=0    Distortion Group
    ------------ ----------------------------- ------------------
    0            sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    15           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    30           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    45           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    60           sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087``
    75           sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087``
    90           sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087``
    105          sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087``
    120          sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087``
    135          sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087``
    150          sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087``
    165          sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087``
    ============ ============================= ==================

    This will select images 0, 45 and 105 to represent the distortion group ``0 -1 0 0.087`` and
    images 120, 135 and 165 to represent ``0 1 0 0.087``. The ``--datain`` file would then
    contain::

        0 -1 0 0.087
        0 -1 0 0.087
        0 -1 0 0.087
        0 1 0 0.087
        0 1 0 0.087
        0 1 0 0.087

    Case: one DWI series and an EPI fieldmap
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

    If a reverse-phase encoding fieldmap image (or images) are passed in through ``epi_fmaps``,
    these will undergo the same selection process using ``max_per_spec``. The images will be
    added to the *end* of the image series, though, to ensure that the fieldmap correction will
    be aligned to the first b=0 image in ``nii_file``. For example if ``nii_file`` contains

    ============ ============================= ==================
    Image Index  BIDS source file for a b=0    Distortion Group
    ------------ ----------------------------- ------------------
    0            sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    15           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    30           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    45           sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087``
    ============ ============================= ==================

    and the file from fmaps contains

    ============ ============================= ==================
    Image Index  BIDS source file for a b=0    Distortion Group
    ------------ ----------------------------- ------------------
    0            sub-1_dir-PA_epi.nii.gz       ``0 1 0 0.087``
    1            sub-1_dir-PA_epi.nii.gz       ``0 1 0 0.087``
    ============ ============================= ==================

    images 0, 15 and 45 would be selected to represent ``0 -1 0 0.087`` and images 0 and 1
    would be selected to represent ``0 1 0 0.087``, resulting in a ``--datain`` file that
    contains::

        0 -1 0 0.087
        0 -1 0 0.087
        0 -1 0 0.087
        0 1 0 0.087
        0 1 0 0.087


    Parameters:
    ===========

        nii_file : str
            A 4D DWI Series
        b0_indices: array-like
            indices into nii_file that can be used by topup
        topup_prefix: str
            file prefix for topup inputs
        bids_origin_files: list
            A list with the original bids file of each image in ``nii_file``. This is
            necessary because merging may have happened earlier in the pipeline
        epi_fmaps:
            A list of b=0 images from the fmaps/ directory.
        max_per_spec: int
            The maximum number of b=0 images to extract from a PE direction / image set

    """

    # Start with the DWI file. Determine which images are b=0
    bvals = np.loadtxt(bval_file)
    b0_indices = np.flatnonzero(bvals < b0_threshold)
    if not b0_indices.size:
        raise RuntimeError("No b=0 images available for TOPUP from the dwi.")
    dwi_nii = load_img(dwi_file)
    # Gather images from just the dwi series
    dwi_spec_lines, dwi_imain, dwi_report, _ = topup_inputs_from_4d_file(
        dwi_nii,
        b0_indices,
        bids_origin_files,
        image_source="combined DWI series",
        max_per_spec=max_per_spec)

    # If there are EPI fieldmaps, add them to the END of the topup spec
    if epi_fmaps and isdefined(epi_fmaps):
        topup_imain, topup_spec_lines, fmap_report = add_epi_fmaps_to_dwi_b0s(
            epi_fmaps, b0_threshold, max_per_spec, dwi_spec_lines, dwi_imain)
        topup_text = dwi_report + fmap_report
    else:
        topup_imain = dwi_imain
        topup_spec_lines = dwi_spec_lines
        topup_text = dwi_report

    imain_output = topup_prefix + "imain.nii.gz"
    imain_img = to_lps(topup_imain, new_axcodes=('L', 'A', 'S'))
    assert imain_img.shape[3] == len(topup_spec_lines)
    imain_img.to_filename(imain_output)

    # Write the datain text file and make sure it's usable if it's needed
    if len(set(topup_spec_lines)) < 2 and topup_requested:
        print(topup_spec_lines)
        raise Exception(
            "Unable to run TOPUP: not enough distortion groups. "
            "Check \"IntendedFor\" fields or consider using --ignore fieldmaps."
        )

    datain_file = topup_prefix + "datain.txt"
    with open(datain_file, "w") as f:
        f.write("\n".join(topup_spec_lines))

    return datain_file, imain_output, topup_text
Example #48
0
def compute_reho(in_file, mask_file, cluster_size):
    """
    Computes the ReHo Map, by computing tied ranks of the timepoints,
    followed by computing Kendall's coefficient concordance(KCC) of a
    timeseries with its neighbours

    Parameters
    ----------

    in_file : nifti file
        4D EPI File 

    mask_file : nifti file
        Mask of the EPI File(Only Compute ReHo of voxels in the mask)

    cluster_size : integer
        for a brain voxel the number of neighbouring brain voxels to use for
        KCC.


    Returns
    -------

    out_file : nifti file
        ReHo map of the input EPI image

    """

    out_file = None

    res_fname = (in_file)
    res_mask_fname = (mask_file)
    CUTNUMBER = 10

    if not (cluster_size == 27 or cluster_size == 19 or cluster_size == 7):
        cluster_size = 27

    nvoxel = cluster_size

    res_img = nb.load(res_fname)
    res_mask_img = nb.load(res_mask_fname)

    res_data = res_img.get_data()
    res_mask_data = res_mask_img.get_data()

    print(res_data.shape)
    (n_x, n_y, n_z, n_t) = res_data.shape

    # "flatten" each volume of the timeseries into one big array instead of
    # x,y,z - produces (timepoints, N voxels) shaped data array
    res_data = np.reshape(res_data, (n_x * n_y * n_z, n_t), order='F').T

    # create a blank array of zeroes of size n_voxels, one for each time point
    Ranks_res_data = np.tile((np.zeros((1, (res_data.shape)[1]))),
                             [(res_data.shape)[0], 1])

    # divide the number of total voxels by the cutnumber (set to 10)
    # ex. end up with a number in the thousands if there are tens of thousands
    # of voxels
    segment_length = np.ceil(float((res_data.shape)[1]) / float(CUTNUMBER))

    for icut in range(0, CUTNUMBER):

        segment = None

        # create a Numpy array of evenly spaced values from the segment
        # starting point up until the segment_length integer
        if not (icut == (CUTNUMBER - 1)):
            segment = np.array(
                np.arange(icut * segment_length, (icut + 1) * segment_length))
        else:
            segment = np.array(
                np.arange(icut * segment_length, (res_data.shape[1])))

        segment = np.int64(segment[np.newaxis])

        # res_data_piece is a chunk of the original timeseries in_file, but
        # aligned with the current segment index spacing
        res_data_piece = res_data[:, segment[0]]
        nvoxels_piece = res_data_piece.shape[1]

        # run a merge sort across the time axis, re-ordering the flattened
        # volume voxel arrays
        res_data_sorted = np.sort(res_data_piece, 0, kind='mergesort')
        sort_index = np.argsort(res_data_piece, axis=0, kind='mergesort')

        # subtract each volume from each other
        db = np.diff(res_data_sorted, 1, 0)

        # convert any zero voxels into "True" flag
        db = db == 0

        # return an n_voxel (n voxels within the current segment) sized array
        # of values, each value being the sum total of TRUE values in "db"
        sumdb = np.sum(db, 0)

        temp_array = np.array(np.arange(0, n_t))
        temp_array = temp_array[:, np.newaxis]

        sorted_ranks = np.tile(temp_array, [1, nvoxels_piece])

        if np.any(sumdb[:]):

            tie_adjust_index = np.flatnonzero(sumdb)

            for i in range(0, len(tie_adjust_index)):

                ranks = sorted_ranks[:, tie_adjust_index[i]]

                ties = db[:, tie_adjust_index[i]]

                tieloc = np.append(np.flatnonzero(ties), n_t + 2)
                maxties = len(tieloc)
                tiecount = 0

                while (tiecount < maxties - 1):
                    tiestart = tieloc[tiecount]
                    ntied = 2
                    while (tieloc[tiecount + 1] == (tieloc[tiecount] + 1)):
                        tiecount += 1
                        ntied += 1

                    ranks[tiestart:tiestart + ntied] = np.ceil(
                        np.float32(np.sum(ranks[tiestart:tiestart + ntied])) /
                        np.float32(ntied))
                    tiecount += 1

                sorted_ranks[:, tie_adjust_index[i]] = ranks

        del db, sumdb
        sort_index_base = np.tile(
            np.multiply(np.arange(0, nvoxels_piece), n_t), [n_t, 1])
        sort_index += sort_index_base
        del sort_index_base

        ranks_piece = np.zeros((n_t, nvoxels_piece))

        ranks_piece = ranks_piece.flatten(order='F')
        sort_index = sort_index.flatten(order='F')
        sorted_ranks = sorted_ranks.flatten(order='F')

        ranks_piece[sort_index] = np.array(sorted_ranks)

        ranks_piece = np.reshape(ranks_piece, (n_t, nvoxels_piece), order='F')

        del sort_index, sorted_ranks

        Ranks_res_data[:, segment[0]] = ranks_piece

        sys.stdout.write('.')

    Ranks_res_data = np.reshape(Ranks_res_data, (n_t, n_x, n_y, n_z),
                                order='F')

    K = np.zeros((n_x, n_y, n_z))

    mask_cluster = np.ones((3, 3, 3))

    if nvoxel == 19:
        mask_cluster[0, 0, 0] = 0
        mask_cluster[0, 2, 0] = 0
        mask_cluster[2, 0, 0] = 0
        mask_cluster[2, 2, 0] = 0
        mask_cluster[0, 0, 2] = 0
        mask_cluster[0, 2, 2] = 0
        mask_cluster[2, 0, 2] = 0
        mask_cluster[2, 2, 2] = 0

    elif nvoxel == 7:

        mask_cluster[0, 0, 0] = 0
        mask_cluster[0, 1, 0] = 0
        mask_cluster[0, 2, 0] = 0
        mask_cluster[0, 0, 1] = 0
        mask_cluster[0, 2, 1] = 0
        mask_cluster[0, 0, 2] = 0
        mask_cluster[0, 1, 2] = 0
        mask_cluster[0, 2, 2] = 0
        mask_cluster[1, 0, 0] = 0
        mask_cluster[1, 2, 0] = 0
        mask_cluster[1, 0, 2] = 0
        mask_cluster[1, 2, 2] = 0
        mask_cluster[2, 0, 0] = 0
        mask_cluster[2, 1, 0] = 0
        mask_cluster[2, 2, 0] = 0
        mask_cluster[2, 0, 1] = 0
        mask_cluster[2, 2, 1] = 0
        mask_cluster[2, 0, 2] = 0
        mask_cluster[2, 1, 2] = 0
        mask_cluster[2, 2, 2] = 0

    for i in range(1, n_x - 1):
        for j in range(1, n_y - 1):
            for k in range(1, n_z - 1):

                block = Ranks_res_data[:, i - 1:i + 2, j - 1:j + 2,
                                       k - 1:k + 2]
                mask_block = res_mask_data[i - 1:i + 2, j - 1:j + 2,
                                           k - 1:k + 2]

                if not (int(mask_block[1, 1, 1]) == 0):

                    if nvoxel == 19 or nvoxel == 7:
                        mask_block = np.multiply(mask_block, mask_cluster)

                    R_block = np.reshape(block, (block.shape[0], 27),
                                         order='F')
                    mask_R_block = R_block[:,
                                           np.argwhere(
                                               np.reshape(mask_block, (1, 27),
                                                          order='F') > 0)[:,
                                                                          1]]

                    K[i, j, k] = f_kendall(mask_R_block)

    img = nb.Nifti1Image(K,
                         header=res_img.get_header(),
                         affine=res_img.get_affine())
    reho_file = os.path.join(os.getcwd(), 'ReHo.nii.gz')
    img.to_filename(reho_file)
    out_file = reho_file

    return out_file
Example #49
0
def hot_to_indices(bool_array):
    """
    Only works for 1D-vector.
    """
    index_array = np.flatnonzero(bool_array)
    return index_array
 def __call__(self, atts):
     i = numpy.random.choice(numpy.flatnonzero(atts == atts.max()))
     dist = numpy.zeros(len(atts))
     dist[i] = 1
     return dist
Example #51
0
def cut_prime_meridian(vertices):
    """Cut a polygon across the prime meridian, possibly splitting it into
    multiple polygons. Vertices consist of (longitude, latitude) pairs where
    longitude is always given in terms of a wrapped angle (between 0 and 2π).

    This routine is not meant to cover all possible cases; it will only work
    for convex polygons that extend over less than a hemisphere."""

    # Ensure that the list of vertices does not contain a repeated endpoint.
    if (vertices[0] == vertices[-1]).all():
        vertices = vertices[:-1]

    # Ensure that the longitudes are wrapped from 0 to 2π.
    vertices = np.column_stack((wrapped_angle(vertices[:, 0]), vertices[:, 1]))

    # Test if the segment consisting of points i-1 and i croses the meridian.
    #
    # If the two longitudes are in [0, 2π), then the shortest arc connecting
    # them crosses the meridian if the difference of the angles is greater
    # than π.
    phis = vertices[:, 0]
    phi0, phi1 = np.sort(np.row_stack((np.roll(phis, 1), phis)), axis=0)
    crosses_meridian = (phi1 - phi0 > np.pi)

    # Count the number of times that the polygon crosses the meridian.
    meridian_crossings = np.sum(crosses_meridian)

    if meridian_crossings == 0:
        # There were zero meridian crossings, so we can use the
        # original vertices as is.
        out_vertices = [vertices]
    elif meridian_crossings == 1:
        # There was one meridian crossing, so the polygon encloses the pole.
        # Any meridian-crossing edge has to be extended
        # into a curve following the nearest polar edge of the map.
        i, = np.flatnonzero(crosses_meridian)
        v0 = vertices[i - 1]
        v1 = vertices[i]

        # Find the latitude at which the meridian crossing occurs by
        # linear interpolation.
        delta_lon = abs(reference_angle(v1[0] - v0[0]))
        lat = (abs(reference_angle(v0[0])) / delta_lon * v0[1] +
               abs(reference_angle(v1[0])) / delta_lon * v1[1])

        # FIXME: Use this simple heuristic to decide which pole to enclose.
        sign_lat = np.sign(np.sum(vertices[:, 1]))

        # Find the closer of the left or the right map boundary for
        # each vertex in the line segment.
        lon_0 = 0. if v0[0] < np.pi else 2 * np.pi
        lon_1 = 0. if v1[0] < np.pi else 2 * np.pi

        # Set the output vertices to the polar cap plus the original
        # vertices.
        out_vertices = [
            np.vstack((vertices[:i], [[lon_0, lat],
                                      [lon_0, sign_lat * np.pi / 2],
                                      [lon_1, sign_lat * np.pi / 2],
                                      [lon_1, lat]], vertices[i:]))
        ]
    elif meridian_crossings == 2:
        # Since the polygon is assumed to be convex, if there is an even number
        # of meridian crossings, we know that the polygon does not enclose
        # either pole. Then we can use ordinary Euclidean polygon intersection
        # algorithms.

        out_vertices = []

        # Construct polygon representing map boundaries.
        frame_poly = geometry.Polygon(
            np.asarray([[0., np.pi / 2], [0., -np.pi / 2],
                        [2 * np.pi, -np.pi / 2], [2 * np.pi, np.pi / 2]]))

        # Intersect with polygon re-wrapped to lie in [-π, π) or [π, 3π).
        for shift in [0, 2 * np.pi]:
            poly = geometry.Polygon(
                np.column_stack(
                    (reference_angle(vertices[:, 0]) + shift, vertices[:, 1])))
            intersection = poly.intersection(frame_poly)
            if intersection:
                assert isinstance(intersection, geometry.Polygon)
                assert intersection.is_simple
                out_vertices += [np.asarray(intersection.exterior)]
    else:
        # There were more than two intersections. Not implemented!
        raise NotImplemented('The polygon intersected the map boundaries two '
                             'or more times, so it is probably not simple and '
                             'convex.')

    # Done!
    return out_vertices
# In[ ]:


model_object = CNN_Model(data,len(data.all_encoded.keys()))


# In[ ]:


classifier = model_object.fit_generator(num_epochs=10, batch_size=16)


# # MRR metric
classifier = load_model("Code/Models/RCNN_ResNext.h5")
test_data = ImageDataGenerator(data.x_test[:10000], data.y_test[:10000], 32, 16)predictions = classifier.predict_generator(test_data)species_embedding = data.embed_vectors2['species_glc_id']y_test = np.array(data.y_test)[:,-1,:]preds = predictions[:,-1,:]
trues = [np.flatnonzero((species_embedding == i).all(1)) for i in y_test]from sklearn.metrics.pairwise import euclidean_distances

def prediction_to_species(ypred, unique_species_id, max_ranks=5):
    e2 = pkl.load(open("Data/Embed2.pkl", "rb"))
    species_embedding = e2['species_glc_id'][-3336:]

    embedding_distance = euclidean_distances(species_embedding, ypred.reshape(-1,10))
    top_indices = embedding_distance.flatten().argsort()[:max_ranks]
    species_glc_id = pd.Series(unique_species_id)[top_indices].reshape(-1,1)
    return species_glc_ids = 0
for i in range(len(trues)):
    if(i%100 == 1): print(i, s/i)
    try:
        ypred = prediction_to_species(preds[i],data.species, 100)
        r = ypred.index(trues[i])
    except:
Example #53
0
    T = fits_table(fn)
    #T.about()
    events = fits_table(fn, ext=2)

    steps, Istep = np.unique(T.step, return_index=True)
    stepmap = dict([(s, i) for i, s in enumerate(steps)])
    print('Steps:', len(steps))
    unixtimes = T.unixtime[Istep]
    t0 = min(unixtimes)
    xaxis = unixtimes - t0

    parent_pid = T._header['PPID']
    T.mine = np.logical_or(T.pid == parent_pid, T.ppid == parent_pid)
    T.main = (T.pid == parent_pid)

    I = np.flatnonzero(T.mine)
    mypids = np.unique(T.pid[I])
    print(len(I), 'measurements of my PIDs')
    print(len(mypids), 'PIDs')

    plt.clf()

    ps_icpu = np.zeros(len(steps), np.float32)
    plotted_worker = False
    for pid in mypids:
        II = I[T.pid[I] == pid]
        # Fill in the non-empty steps
        J = np.array([stepmap[s] for s in T.step[II]])

        cmds = np.unique(T.command[II])
        #print('PID', pid, ':', cmds)
Example #54
0
    def fit(self, X, y, max_features=None, overwrite_X=False, **params):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training data.

        y : array-like, shape = [n_samples]
            Target values.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        self._set_params(**params)

        X = np.atleast_2d(X)
        y = np.atleast_1d(y)

        X, y, Xmean, ymean = LinearModel._center_data(X, y, self.fit_intercept)
        alpha = getattr(self, 'alpha', 0.)

        if self.normalize:
            norms = np.sqrt(np.sum(X**2, axis=0))
            nonzeros = np.flatnonzero(norms)
            if not overwrite_X:
                X = X.copy()
                overwrite_X = True
            X[:, nonzeros] /= norms[nonzeros]

        # precompute if n_samples > n_features
        precompute = self.precompute
        if hasattr(precompute, '__array__'):
            # copy as it's going to be modified
            Gram = precompute.copy()
        elif precompute == 'auto':
            Gram = 'auto'
        else:
            Gram = None

        self.alphas_, self.active_, self.coef_path_ = lars_path(
            X,
            y,
            Gram=Gram,
            overwrite_X=overwrite_X,
            overwrite_Gram=True,
            alpha_min=alpha,
            method=self.method,
            verbose=self.verbose,
            max_features=max_features,
            max_iter=self.max_iter)

        if self.normalize:
            self.coef_path_ /= norms[:, np.newaxis]
        self.coef_ = self.coef_path_[:, -1]

        self._set_intercept(Xmean, ymean)

        return self
def samp_entropy(a, m, r, tau=1, relative_r=True):

    """
    Compute the sample entropy [RIC00]_ of a signal with embedding dimension `de` and delay `tau` [PYEEG]_.
    Vectorised version of the eponymous PyEEG function.
    In addition, this function can also be used to vary tau and therefore compute Multi-Scale Entropy(MSE) [COS05]_ by
    coarse grainning the time series (see example bellow).
    By default, r is expressed as relatively to the standard deviation of the signal.
    :param a: a one dimensional floating-point array representing a time series.
    :type a: :class:`~numpy.ndarray` or :class:`~pyrem.time_series.Signal`
    :param m: the scale
    :type m: int
    :param r: The tolerance
    :type r: float
    :param tau: The scale for coarse grainning.
    :type tau: int
    :param relative_r: whether the argument r is relative to the standard deviation. If false, an absolute value should be given for r.
    :type relative_r: true
    :return: the approximate entropy, a scalar
    :rtype: float
    Example:
    """

    if len(a) <= 2:
        return np.nan

    coarse_a = _coarse_grainning(a, tau)

    if relative_r:
        coarse_a /= np.std(coarse_a)

    embsp = _embed_seq(coarse_a, 1, m + 1)
    embsp_last = embsp[:, -1]
    embs_mini = embsp[:, :-1]

    # Buffers are preallocated chunks of memory storing temporary results.
    # see the `out` argument in numpy *ufun* documentation

    dist_buffer = np.zeros(embsp.shape[0] - 1, dtype=np.float32)
    subtract_buffer = np.zeros((dist_buffer.size, m), dtype=np.float32)
    in_range_buffer = np.zeros_like(dist_buffer, dtype=np.bool)
    sum_cm, sum_cmp = 0.0, 0.0

    # we iterate through all templates (rows), except last one.
    for i, template in enumerate(embs_mini[:-1]):

        # these are just views to the buffer arrays. to store intermediary matrices
        dist_b_view = dist_buffer[i:]
        sub_b_view = subtract_buffer[i:]
        range_b_view = in_range_buffer[i:]
        embsp_view = embsp_last[i+1:]

        # substract the template from each subsequent row of the embedded matrix
        np.subtract(embs_mini[i+1:],  template, out=sub_b_view)
        # Absolute distance
        np.abs(sub_b_view, out=sub_b_view)
        # Maximal absolute difference between a scroll and a template is the distance
        np.max(sub_b_view, axis=1, out=dist_b_view)
        # we compare this distance to a tolerance r
        np.less_equal(dist_b_view, r, out=range_b_view)
        # score one for this template for each match
        in_range_sum = np.sum(range_b_view)
        sum_cm += in_range_sum

        # reuse the buffers for last column
        dist_b_view = dist_buffer[:in_range_sum]

        where = np.flatnonzero(range_b_view)
        dist_b_view = np.take(embsp_view, where, out=dist_b_view)
        range_b_view = in_range_buffer[range_b_view]
        # score one to TODO for each match of the last element
        dist_b_view -= embsp_last[i]
        np.abs(dist_b_view, out=dist_b_view)
        np.less_equal(dist_b_view, r, out=range_b_view)
        sum_cmp += np.sum(range_b_view)

    if sum_cm == 0 or sum_cmp ==0:
        return np.NaN

    return np.log(sum_cm/sum_cmp)
z_test = z_m_test
print 'z_test_shape: ' ,z_test.shape
print 's_test: ' ,s_test.shape

X_test_dec = net.Dec.predict([z_test,s_test],batch_size=params['batch_size'])
sdim = s_test.shape[1]

n=10
full_inds = np.zeros((n,))
y_vals = np.unique(y_test)
if len(y_vals)>n:
    y_vals = np.random.permutation(y_vals)
for i in range(n):
    dig = y_vals[i%len(y_vals)]
    #print dig
    cur_inds = np.flatnonzero(y_test==dig)
    perm_inds = np.random.permutation(range(len(cur_inds)))
    full_inds[i] = cur_inds[perm_inds[i]]

#print full_inds
full_inds = full_inds.astype(np.int)
z_rand = np.zeros((n*n,z_test.shape[1]))
#z_rand = z_test[:n*n,:]
s_rand = np.zeros((n*n,sdim))
for i in range(n):
    for jj in range(n):
        z_rand[i*n+jj,:] = z_test[full_inds[i],:]
        if sdim>1:
            s_rand[i*n+jj,:] = s_test[full_inds[jj],:]
        else:
            s_rand[i*n+jj] = s_test[full_inds[jj]]
Example #57
0
    def __init__(self,
                 zscores,
                 null_lb=-1,
                 null_ub=1,
                 estimate_mean=True,
                 estimate_scale=True,
                 estimate_null_proportion=False):

        # Extract the null z-scores
        ii = np.flatnonzero((zscores >= null_lb) & (zscores <= null_ub))
        if len(ii) == 0:
            raise RunTimeError("No Z-scores fall between null_lb and null_ub")
        zscores0 = zscores[ii]

        # Number of Z-scores, and null Z-scores
        n_zs, n_zs0 = len(zscores), len(zscores0)

        # Unpack and transform the parameters to the natural scale, hold
        # parameters fixed as specified.
        def xform(params):

            mean = 0.
            sd = 1.
            prob = 1.

            ii = 0
            if estimate_mean:
                mean = params[ii]
                ii += 1
            if estimate_scale:
                sd = np.exp(params[ii])
                ii += 1
            if estimate_null_proportion:
                prob = 1 / (1 + np.exp(-params[ii]))

            return mean, sd, prob

        from scipy.stats.distributions import norm

        def fun(params):
            """
            Negative log-likelihood of z-scores.

            The function has three arguments, packed into a vector:

            mean : location parameter
            logscale : log of the scale parameter
            logitprop : logit of the proportion of true nulls

            The implementation follows section 4 from Efron 2008.
            """

            d, s, p = xform(params)

            # Mass within the central region
            central_mass = (norm.cdf((null_ub - d) / s) - norm.cdf(
                (null_lb - d) / s))

            # Probability that a Z-score is null and is in the central region
            cp = p * central_mass

            # Binomial term
            rval = n_zs0 * np.log(cp) + (n_zs - n_zs0) * np.log(1 - cp)

            # Truncated Gaussian term for null Z-scores
            zv = (zscores0 - d) / s
            rval += np.sum(-zv**2 / 2) - n_zs0 * np.log(s)
            rval -= n_zs0 * np.log(central_mass)

            return -rval

        # Estimate the parameters
        from scipy.optimize import minimize
        # starting values are mean = 0, scale = 1, p0 ~ 1
        mz = minimize(fun, np.r_[0., 0, 3], method="Nelder-Mead")
        mean, sd, prob = xform(mz['x'])

        self.mean = mean
        self.sd = sd
        self.null_proportion = prob
Example #58
0
File: nb.py Project: yamen/vectorbt
def true_choice_nb(col, from_i, to_i, a):
    """`choice_func_nb` that returns the indices of signals in `a`."""
    return from_i + np.flatnonzero(a[from_i:to_i, col])
Example #59
0
LFtous = numpy.zeros(Npoints)
LFine = numpy.zeros(Npoints)
LFelas = numpy.zeros(Npoints)

#Define number of particles/bunch (or filling pattern array)
Np = I0 * mA

#Calculates the bunch length and saves data in the file BunchLength.txt
#Also prints some statistics at the end
Calc_Sigma(Vh, ph, Vm, pm, I0, Npoints, param)

#Average pressure in the machine
Pmed = 1  #[nTorr]

#Check places where I0 differnte then and get the indexes
not_zero = numpy.flatnonzero(I0)

print '-----------------------------------------------'
print 'Calculates IBS effects and Lifetime results'
for j in range(len(not_zero)):
    #Redefines parameter Np
    param['Np'] = I0[not_zero[j]] * mA
    #Calculates IBS effects on the emittances
    (exi[j], eyi[j], spi[j],
     ssi[j]) = Iterate_emittances3HC(twiss, param, pm[not_zero[j]],
                                     Vm[not_zero[j]], ph[not_zero[j]],
                                     Vh[not_zero[j]])
    #Uses the IBS results to calculates lifetimes
    (LFtous[j], LFine[j],
     LFelas[j]) = Calc_Lifetime(Pmed, param, I0[not_zero[j]], twiss, exi[j],
                                eyi[j], spi[j], ssi[j])
Example #60
0
ax.set_xlabel('Threshold [Filter output sdev]')
ax.set_ylabel('Rate [cps]')

for ifile, file in enumerate(files):
    with np.load(file) as arch:

        thr = arch['thr']
        thrcounts = arch['thrcounts']
        thr_theory = arch['thr_theory']
        thrcounts_theory = arch['thrcounts_theory']
        effnsamples = arch['effnsamples']
        nevents = arch['nevents']
        freq = arch['freq']
        sdev = arch['sdev']

        nz = np.flatnonzero(thrcounts)
        start = max(0, nz[0] - 1)
        end = min(len(thr), nz[-1] + 2)
        s = slice(start, end)

        ratefactor = freq / (nevents * effnsamples)
        cond = thr_theory >= np.min(thr)
        cond &= thr_theory <= np.max(thr)
        cond &= thrcounts_theory >= np.min(thrcounts[thrcounts > 0])

        label = os.path.split(file)[1].replace('.npz', '')

        kw = dict(color=f'C{ifile}')
        ax.plot(thr_theory[cond] / sdev, ratefactor * thrcounts_theory[cond],
                '-', **kw)
        ax.plot(thr[s] / sdev,