Example #1
0
def prepare_train_data(data):

	labels = []
	flattened = []
	features = []
	images = []
	images_norm = [0] * 7
	for i in range(7):
		labels.append(np.array([int(el[0]) for el in data[0][i][0][0][1]]))
		# labels.append(1.0 * data[0][i][0][0][1])
		images.append(1.0 * data[0][i][0][0][0])
		#print images[0].shape
		#print labels[0].shape
		# images.append(1.0 * np.array([float(el[0]) for el in data[0][i][0][0][0]]))
		reshaped = np.reshape(images[i], (784, labels[i].shape[0]))
		reshaped = np.transpose(reshaped) 

		#Center around 0, normalized
		mean = np.mean(reshaped, axis = 1)
		std = np.std(reshaped, axis =1)
		for k in xrange(std.shape[0]):
			if std[k] == 0: 
				std[k] = 1				
		reshaped = reshaped - mean[:,None]
		reshaped_normalized = reshaped / std[:,None]

		#reshaped_normalized = 1.0 * reshaped / sum(reshaped)
		reshaped_normalized = np.transpose(reshaped_normalized)
		flattened.append(reshaped_normalized)
		features.append(np.transpose(flattened[i]))

	return (features, labels)
Example #2
0
def read_abinit(filename):
    with open(filename) as f:
        abinit_in = AbinitIn(f.readlines())
    tags = abinit_in.get_variables()
    acell = tags['acell']
    rprim = tags['rprim'].T
    scalecart = tags['scalecart']
    lattice = rprim * acell
    if scalecart is not None:
        for i in range(3):
            lattice[i] *= scalecart[i]

    if tags['xcart'] is not None:
        pos_bohr = np.transpose(tags['xcart'])
        positions = np.dot(np.linalg.inv(lattice), pos_bohr).T
    elif tags['xangst'] is not None:
        pos_bohr = np.transpose(tags['xangst']) / Bohr
        positions = np.dot(np.linalg.inv(lattice), pos_bohr).T
    elif tags['xred'] is not None:
        positions = tags['xred']

    numbers = [tags['znucl'][x - 1] for x in tags['typat']]

    return Atoms(numbers=numbers,
                 cell=lattice.T,
                 scaled_positions=positions)
Example #3
0
def spatial_batchnorm_backward(dout, cache):
    """
    Computes the backward pass for spatial batch normalization.
    
    Inputs:
    - dout: Upstream derivatives, of shape (N, C, H, W)
    - cache: Values from the forward pass
    
    Returns a tuple of:
    - dx: Gradient with respect to inputs, of shape (N, C, H, W)
    - dgamma: Gradient with respect to scale parameter, of shape (C,)
    - dbeta: Gradient with respect to shift parameter, of shape (C,)
    """
    dx, dgamma, dbeta = None, None, None

    #############################################################################
    # TODO: Implement the backward pass for spatial batch normalization.        #
    #                                                                           #
    # HINT: You can implement spatial batch normalization using the vanilla     #
    # version of batch normalization defined above. Your implementation should  #
    # be very short; ours is less than five lines.                              #
    #############################################################################
    N,C,H,W = dout.shape
    dout_temp = np.transpose(dout, [0, 2, 3, 1]).reshape(N*H*W, C)
    dx, dgamma, dbeta = batchnorm_backward(dout_temp, cache)
    dx = np.transpose(dx.reshape(N, H, W, C), [0, 3, 1, 2])
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

    return dx, dgamma, dbeta
Example #4
0
 def store_models(self, specs, ivar):
     self.models = n.zeros( (specs.shape) )
     for i in xrange(self.models.shape[0]):
         minloc = n.unravel_index( self.zchi2arr[i].argmin(),
                                  self.zchi2arr[i].shape )
         pmat = n.zeros( (specs.shape[-1],self.npoly+1) )
         this_temp = self.templates[minloc[:-1]]
         pmat[:,0] = this_temp[(minloc[-1]*self.npixstep) +
                               self.pixoffset:(minloc[-1]*self.npixstep) +
                               self.pixoffset+specs.shape[-1]]
         polyarr = poly_array(self.npoly, specs.shape[-1])
         pmat[:,1:] = n.transpose(polyarr)
         ninv = n.diag(ivar[i])
         try: # Some eBOSS spectra have ivar[i] = 0 for all i
             '''
             f = n.linalg.solve( n.dot(n.dot(n.transpose(pmat),ninv),pmat),
                             n.dot( n.dot(n.transpose(pmat),ninv),specs[i]) )
             '''
             f = nnls( n.dot(n.dot(n.transpose(pmat),ninv),pmat),
                      n.dot( n.dot(n.transpose(pmat),ninv),specs[i]) );\
                             f = n.array(f)[0]
             self.models[i] = n.dot(pmat, f)
         except Exception as e:
             self.models[i] = n.zeros(specs.shape[-1])
             print "Exception: %r" % r
Example #5
0
	def updateParameters(self, articlePicked, click,  userID):	
		self.counter +=1
		self.Wlong = vectorize(self.W)
		featureDimension = len(articlePicked.featureVector)
		T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) 
		self.A += np.outer(T_X, T_X)	
		self.b += click*T_X
		self.AInv = np.linalg.inv(self.A)
		self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) 

		Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum))
		Xi_Matirx.T[userID] = articlePicked.featureVector
		W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx))
		self.batchGradient +=evaluateGradient(W_X, click, self.Wlong, self.lambda_, self.regu  )

		if self.counter%self.windowSize ==0:
			self.Wlong -= 1/(float(self.counter/self.windowSize)+1)*self.batchGradient
			self.W = matrixize(self.Wlong, self.userNum)
			self.W = normalize(self.W, axis=0, norm='l1')
			#print 'SVD', self.W
			self.batchGradient = np.zeros(self.userNum*self.userNum)
			# Use Ridge regression to fit W
		'''
		plt.pcolor(self.W_b)
		plt.colorbar
		plt.show()
		'''
		if self.W.T[userID].any() <0 or self.W.T[userID].any()>1:
			print self.W.T[userID]

		self.CoTheta = np.dot(self.UserTheta, self.W)
		self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector)))
		self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW))
		self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def detrend(dataset,tvalues,order=1):
	"""Removes polinomial trends

  Given a linear input dataset, detrend it removing a polynomial of order
  N of type trend(t)=\sum_{i=0}^N a_i t^i

  Arguments:

    'dataset' -- Numpy array with the data to be detrended (along its _first_
                 dimension)

    'tvalues' -- Numpy array with the values of the time coordinate. Its 
                 length must be that of the first dimension of 'dataset'

  Optional arguments:

    'order' -- The order of the polinomial to be removed. Defaults to 1 (linear)

  The function returns the detrended (LINEAR) dataset and
  the parameters of the trend in a tuple.
  Of course, when removing a polynomial of the mentioned type, the
  mean is also removed !!!
	"""
	T=numpy.ones((len(dataset),)+(order+1,),numpy.float64)
	for iord in xrange(1,len(T[0])):
		T[:,iord]=T[:,iord-1]*tvalues
	V=mm(numpy.transpose(T),T)
	Tx=mm(numpy.transpose(T),dataset)
	A=mm(numpy.linalg.inv(V),Tx)
	trendterm=mm(T,A)
	residual=dataset-trendterm
	return residual,A
Example #7
0
    def _add_solids(self, X, Y, C):
        """
        Draw the colors using :meth:`~matplotlib.axes.Axes.pcolormesh`;
        optionally add separators.
        """
        if self.orientation == "vertical":
            args = (X, Y, C)
        else:
            args = (np.transpose(Y), np.transpose(X), np.transpose(C))
        kw = dict(cmap=self.cmap, norm=self.norm, alpha=self.alpha, edgecolors="None")
        # Save, set, and restore hold state to keep pcolor from
        # clearing the axes. Ordinarily this will not be needed,
        # since the axes object should already have hold set.
        _hold = self.ax.ishold()
        self.ax.hold(True)
        col = self.ax.pcolormesh(*args, **kw)
        self.ax.hold(_hold)
        # self.add_observer(col) # We should observe, not be observed...

        if self.solids is not None:
            self.solids.remove()
        self.solids = col
        if self.dividers is not None:
            self.dividers.remove()
            self.dividers = None
        if self.drawedges:
            linewidths = (0.5 * mpl.rcParams["axes.linewidth"],)
            self.dividers = collections.LineCollection(
                self._edges(X, Y), colors=(mpl.rcParams["axes.edgecolor"],), linewidths=linewidths
            )
            self.ax.add_collection(self.dividers)
Example #8
0
def test_arclength_half_circle():
    """ Here we define the tests for the lenght computer of our ArcLengthParametrizer, we try it with a half a 
    circle and a fan. 
    We test it both in 2d and 3d."""


    # Number of interpolation points minus one
    n = 5
    toll = 1.e-6
    points = np.linspace(0, 1, (n+1) ) 
    R = 1
    P = 1
    control_points_2d = np.asmatrix(np.zeros([n+1,2]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)]
    control_points_2d[:,0] = np.transpose(np.matrix([R*np.cos(1 * i * np.pi / (n + 1))for i in range(n+1)]))
    control_points_2d[:,1] = np.transpose(np.matrix([R*np.sin(1 * i * np.pi / (n + 1))for i in range(n+1)]))

    control_points_3d = np.asmatrix(np.zeros([n+1,3]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)]
    control_points_3d[:,0] = np.transpose(np.matrix([R*np.cos(1 * i * np.pi / (n + 1))for i in range(n+1)]))
    control_points_3d[:,1] = np.transpose(np.matrix([R*np.sin(1 * i * np.pi / (n + 1))for i in range(n+1)]))
    control_points_3d[:,2] = np.transpose(np.matrix([P*i for i in range(n+1)]))

    vsl = AffineVectorSpace(UniformLagrangeVectorSpace(n+1),0,1)
    dummy_arky_2d = ArcLengthParametrizer(vsl, control_points_2d)
    dummy_arky_3d = ArcLengthParametrizer(vsl, control_points_3d)
    length2d = dummy_arky_2d.compute_arclength()[-1,1]
    length3d = dummy_arky_3d.compute_arclength()[-1,1]
#    print (length2d)
#    print (n * np.sqrt(2))
    l2 = np.pi * R
    l3 = 2 * np.pi * np.sqrt(R * R + (P / (2 * np.pi)) * (P / (2 * np.pi)))
    print (length2d, l2)
    print (length3d, l3)
    assert (length2d - l2) < toll
    assert (length3d - l3) < toll
Example #9
0
def test_reparametrization():
    """ Here we define the tests for reparametrizer of our ArcLengthParametrizer, we try it with a half a 
    circle and a fan. 
    We test it both in 2d and 3d."""
    R = 1
    P = 1
    toll = 1.e-6

    n = 10
    ii = [0.5,0.8,4,6,7,8,9,9.6,10,10.1,11]
    control_points_3d = np.asmatrix(np.zeros([n+1,3]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)]
    control_points_3d[:,0] = np.transpose(np.matrix([R*np.cos(5*i * np.pi / (n + 1))for i in ii]))
    control_points_3d[:,1] = np.transpose(np.matrix([R*np.sin(5*i * np.pi / (n + 1))for i in ii]))
    control_points_3d[:,2] = np.transpose(np.matrix([P*i for i in range(n+1)]))
    #control_points_3d[3,:] += 32
    #print control_points_3d[0]
    vsl = AffineVectorSpace(UniformLagrangeVectorSpace(n+1),0,1)
    arky = ArcLengthParametrizer(vsl, control_points_3d)
    new_control_points_3d = arky.reparametrize()

    new_arky = ArcLengthParametrizer(vsl, new_control_points_3d)
    new_new_control_points_3d = arky.reparametrize()
    tt = np.linspace(0, 1, 128)

    new_new_vals = vsl.element(new_new_control_points_3d)(tt)
    #print vals
    new_vals = vsl.element(new_control_points_3d)(tt)
    #print vals.shape, new_vals.shape
    assert np.amax(np.abs(new_new_vals-new_vals)) < toll
Example #10
0
def display_grid(grid, **kwargs):
    fig = plt.figure()
    plt.axes().set_aspect('equal')

    if kwargs.get('mark_core_cells', True):
        core_cell_coords = grid._cell_nodes[1:-1, 1:-1]
        cellx, celly = core_cell_coords[:, :, 0], core_cell_coords[:, :, 1]
        plt.plot(cellx, celly, '-o', np.transpose(cellx), np.transpose(celly), '-o', color='red')

    if kwargs.get('mark_boundary_cells', True):
        boundary_cell_coords = grid._cell_nodes[0, :], \
                               grid._cell_nodes[-1, :], \
                               grid._cell_nodes[1:-1, 0], \
                               grid._cell_nodes[1:-1, -1]

        for coords in boundary_cell_coords:
            plt.plot(coords[:, 0], coords[:, 1], '-x', color='blue')

    if kwargs.get('show', False):
        plt.show()

    f = BytesIO()
    plt.savefig(f)

    return f
Example #11
0
 def sample_line_segment_mm_s(start_xy_mm, end_xy_mm, dt_s, mW=None, max_mm=5.0):
     """ Given a line segment in mm space, map it to galvo space.
         To make the line straight in mm space, samples may be added to 
         more-closely approximate a straight line.
         Returns: An array of shape nx3 (if mW is None) or nx4 (if mW is not None) 
                     of points time deltas in mm and seconds,
                     excluding start_xy_mm and including end_xy_mm,
                     possibly including samples along the way.
     """
     import FLP
     from numpy.linalg import norm
     dist_mm = norm(np.asarray(end_xy_mm) - start_xy_mm)
     if dist_mm <= max_mm:
         if mW is None:
             return np.array((tuple(end_xy_mm) + (dt_s,),)) # Just the end sample.
         else:
             return np.array((tuple(end_xy_mm) + (dt_s, mW),)) # Just the end sample.
     samples_s = np.linspace(0, dt_s, np.ceil(dist_mm / max_mm) + 1)
     timeRange_s = (0, dt_s)
     if mW is None:
         return np.transpose([np.interp(samples_s[1:], timeRange_s, (start_xy_mm[0], end_xy_mm[0])),
                              np.interp(samples_s[1:], timeRange_s, (start_xy_mm[1], end_xy_mm[1])),
                              np.diff(samples_s)])
     else:
         return np.transpose([np.interp(samples_s[1:], timeRange_s, (start_xy_mm[0], end_xy_mm[0])),
                              np.interp(samples_s[1:], timeRange_s, (start_xy_mm[1], end_xy_mm[1])),
                              np.diff(samples_s),
                              mW * np.ones_like(samples_s[1:])])
Example #12
0
def test_cl():
    ctx = cl.create_some_context()  # (interactive=False)

    # print 'ctx', ctx
    queue = cl.CommandQueue(ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
    f = open("part1.cl", "r")
    fstr = "".join(f.readlines())
    program = cl.Program(ctx, fstr).build()
    mf = cl.mem_flags

    cameraPos = np.array([0, 6, -1, 0])
    invView = la.inv(look_at((0, 6, -1), (0, 1, 1), (0, 1, 0)))
    invProj = la.inv(perspective(60, 1, 1, 1000))
    print "view", invView
    print "proj", invProj
    viewParamsData = (
        cameraPos.flatten().tolist()
        + np.transpose(invView).flatten().tolist()
        + np.transpose(invProj).flatten().tolist()
    )
    # print 'vpd', viewParamsData
    viewParams = struct.pack("4f16f16f", *viewParamsData)
    viewParams_buf = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=viewParams)
    num_pixels = 1000 * 1000
    # setup opencl
    dest = np.ndarray((1000, 1000, 4), dtype=np.float32)
    dest_buf = cl.Buffer(ctx, mf.WRITE_ONLY, dest.nbytes)
    local_shape = (8, 8)
    # run kernel
    evt = program.part1(queue, (dest.shape[0], dest.shape[1]), None, viewParams_buf, dest_buf)
    # evt = program.part1(queue, dest.shape, None, dest_buf)
    cl.enqueue_read_buffer(queue, dest_buf, dest).wait()
    print "time", (evt.profile.end - evt.profile.start) * 0.000001, "ms"
    return dest
Example #13
0
def get_moments(v,m,n=100):
  """ Get the first n moments of a certain vector
  using the Chebychev recursion relations"""
  mus = np.array([0.0j for i in range(2*n)]) # empty arrray for the moments
  a = v.copy() # first vector
  am = v.copy() # zero vector
  a = m*v  # vector number 1
  bk = (np.transpose(np.conjugate(v))*v)[0,0] # scalar product
  bk1 = (np.transpose(np.conjugate(v))*a)[0,0] # scalar product
  mus[0] = bk  # mu0
  mus[1] = bk1 # mu1
  for i in range(1,n): 
    ap = 2*m*a - am # recursion relation
    bk = (np.transpose(np.conjugate(a))*a)[0,0] # scalar product
    bk1 = (np.transpose(np.conjugate(ap))*a)[0,0] # scalar product
    mus[2*i] = 2.*bk
    mus[2*i+1] = 2.*bk1
    am = a +0. # new variables
    a = ap+0. # new variables
  mu0 = mus[0] # first
  mu1 = mus[1] # second
  for i in range(1,n): 
    mus[2*i] +=  - mu0
    mus[2*i+1] += -mu1 
  return mus
    def _train(self, input_list, target_list):
        """Internal training method for our neural network."""
        # 1) Get transposed arrays - one column.
        inputs = numpy.array(input_list, ndmin=2).T

        # 2) Forward propogation.
        hidden_inputs = numpy.dot(self.weights[ITH], inputs)
        hidden_outputs = self.activation_function(hidden_inputs)

        final_inputs = numpy.dot(self.weights[HTO], hidden_outputs)
        final_outputs = self.activation_function(final_inputs)

        # 3) Get error.
        targets = numpy.array(target_list, ndmin=2).T
        output_errors = targets - final_outputs
        hidden_errors = numpy.dot(self.weights[HTO].T,
                                  output_errors)

        self.weights[HTO] += self._learning_rate * numpy.dot(
            (output_errors * final_outputs * (1.0 - final_outputs)),
            numpy.transpose(hidden_outputs))

        self.weights[ITH] += self._learning_rate * numpy.dot(
            (hidden_errors * hidden_outputs * (1.0 - hidden_outputs)),
            numpy.transpose(inputs))
Example #15
0
    def __new__(cls, input_array, tol=1e-3):
        """
        Create an ElasticTensor object.  The constructor throws an error if
        the shape of the input_matrix argument is not 3x3x3x3, i. e. in true
        tensor notation.  Issues a warning if the input_matrix argument does
        not satisfy standard symmetries.  Note that the constructor uses
        __new__ rather than __init__ according to the standard method of
        subclassing numpy ndarrays.

        Args:
            input_array (3x3x3x3 array-like): the 3x3x3x3 array-like
                representing the elastic tensor

            tol (float): tolerance for initial symmetry test of tensor
        """

        obj = TensorBase(input_array).view(cls)
        if obj.shape != (3, 3, 3, 3):
            raise ValueError("Default elastic tensor constructor requires "
                             "input to be the true 3x3x3x3 representation. "
                             "To construct from an elastic tensor from "
                             "6x6 Voigt array, use ElasticTensor.from_voigt")

        if not ((obj - np.transpose(obj, (1, 0, 2, 3)) < tol).all() and
                    (obj - np.transpose(obj, (0, 1, 3, 2)) < tol).all() and
                    (obj - np.transpose(obj, (1, 0, 3, 2)) < tol).all() and
                    (obj - np.transpose(obj, (3, 2, 0, 1)) < tol).all()):
            warnings.warn("Input elasticity tensor does "
                          "not satisfy standard symmetries")

        return obj
Example #16
0
    def search(self, text, isSvd = False):

        if isSvd:
            print "Search with SVD"
        else:
            print "Search with LDA"

        start = time.time()
        vector = text.split()
        cleanedVector = cleanVector(vector)
        #cleanedVector = self.wordsToAsk

        print cleanedVector

        if isSvd:
            bagOfWords, indices = createBagOfWordsFromVector(cleanedVector, self.amountOfWords, self.dictOfWords, self.idfs)
            b = fasterCorrelations(self.matrix, indices, bagOfWords, self.amountOfFiles)

        else:
            bagOfWords = createBagOfWordsForLDA(cleanedVector, self.amountOfWords, self.dictOfWords)

            #print 'bow shape:'
            #print bagOfWords.shape

            #print 'topic word shape:'
            #print np.transpose(self.model.topic_word_).shape
            res = np.dot(bagOfWords, np.transpose(self.model.topic_word_))

            #print 'first mul shape:'
            #print res.shape
            #print res
            #print 'doc topic shape:'
            #print np.transpose(self.model.doc_topic_).shape

            res2 = np.dot(res, np.transpose(self.model.doc_topic_))

            #print 'final shape:'
            #print res2.shape

            bestValues = sorted(list(res2[0]), reverse=True)[:5]

            b = []

            for bestVal in bestValues:
                for ind, val in enumerate(list(res2[0])):
                    if val == bestVal:
                        b.append((ind, val))


        results = []
        dbMan = DatabaseManager()
        for x in b:
            results.append(dbMan.get_link(self.listOfArticles[x[0]]))

        for res in results:
            print res.url
            print res.title

        stop = time.time()
        return results, stop - start
Example #17
0
def getTopOutlierIndexAsc(data, per):
    res = np.argsort(np.transpose(data))
    resD = np.sort(np.transpose(data))
    lenth = len(data)
    topNum = int(np.floor(per*lenth))+1
    res = res[0]
    return res[:topNum]
Example #18
0
def lms2rgb(lms_Nx3, conversionMatrix=None):
    """Convert from cone space (Long, Medium, Short) to RGB.

    Requires a conversion matrix, which will be generated from generic
    Sony Trinitron phosphors if not supplied (note that you will not get
    an accurate representation of the color space unless you supply a
    conversion matrix)

    usage::

        rgb_Nx3 = lms2rgb(dkl_Nx3(el,az,radius), conversionMatrix)

    """

    lms_3xN = numpy.transpose(lms_Nx3)#its easier to use in the other orientation!

    if conversionMatrix==None:
        cones_to_rgb = numpy.asarray([ \
            #L        M        S
            [ 4.97068857, -4.14354132, 0.17285275],#R
            [-0.90913894, 2.15671326, -0.24757432],#G
            [-0.03976551, -0.14253782, 1.18230333]#B
            ])
        logging.warning('This monitor has not been color-calibrated. Using default LMS conversion matrix.')
    else: cones_to_rgb=conversionMatrix

    rgb_to_cones = numpy.linalg.pinv(cones_to_rgb)#get inverse
    rgb = numpy.dot(cones_to_rgb, lms_3xN)
    return numpy.transpose(rgb)#return in the shape we received it
Example #19
0
 def test(self):
     # Expected
     in_channels = 3
     in_dim = 11
     out_channels = 5
     out_dim = (in_dim/2 + 1)
     img = np.arange(0,in_dim*in_dim*in_channels*1, dtype=np.float32)
     img = np.reshape(img,[in_dim,in_dim,in_channels,1])
     filter = np.arange(0,3*3*in_channels*out_channels, dtype=np.float32)
     filter = np.reshape(filter,[3,3,in_channels,out_channels])
     bias = np.zeros([5])
     expected = np.zeros([out_dim,out_dim,out_channels])
     for och in range(out_channels):
         tmp = np.zeros([out_dim,out_dim,1])
         for ich in range(in_channels):
             imgslice = np.reshape(img[:,:,ich,0],[in_dim,in_dim])
             filterslice = np.reshape(filter[:,:,ich,och],[3,3])
             tmp += np.reshape(convolve(imgslice,filterslice,mode='constant',cval = 0.0)[::2,::2] , [out_dim, out_dim, 1])
         expected[:,:,och] = np.squeeze(tmp) + bias[och]
         
     # test
     owlimg = owl.from_numpy(np.transpose(img))
     owlfilter = owl.from_numpy(np.transpose(filter))
     owlbias = owl.from_numpy(bias)
     convolver = owl.conv.Convolver(1,1,2,2)   
     test = convolver.ff(owlimg, owlfilter, owlbias)
     
     print 'Expected\n',expected
     print "Actual\n",test.to_numpy()
     self.assertTrue(np.allclose(expected, test))
Example #20
0
def calc_alm_chisq_fromfits(almfile, clfile):
    alm = pyfits.open(almfile)[0].data
    cls = pyfits.open(clfile)[0].data
    numiter = alm.shape[0]
    numchain = alm.shape[1]
    alm = np.reshape(alm, (numiter*numchain,alm.shape[2], alm.shape[3], alm.shape[4], alm.shape[5]))
    alm = alm[:, :, :, 2:, :]
    cls = cls[1:]
    cls = np.reshape(cls, (numiter * numchain, cls.shape[2], cls.shape[3]))
    if alm.shape[1] == 3:
        cls = np.concatenate((cls[:, 0:1, :], cls[:, 3:4, :], cls[:, 5:6, :]), 1)
    elif alm.shape[1] == 1:
        cls = cls[:, 0:1, :]
    cls = cls[:, :, 2:]
    cls = np.transpose(cls).copy()
    alm = np.transpose(alm).copy()
    chisq = np.zeros(cls.shape)
    for i in range(cls.shape[0]):
        l = i + 2
        for m in range(l):
            if m == 0:
                chisq[i, :, :] += alm[0, i, m, :, :] ** 2
            else:
                chisq[i, :, :] += np.sum(2 * alm[:, i, m, :, :] ** 2, 0)
        chisq[i, :, :] = chisq[i, :, :] / cls[i, :, :] / (2 * l + 1) * (l * (l + 1)) / (2 * np.pi)
    return chisq
Example #21
0
File: test.py Project: Jorge-C/bipy
def permute_2d(m, p):
    """Performs 2D permutation of matrix m according to p."""
    return m[p][:, p]
    # unused below
    m_t = np.transpose(m)
    r_t = np.take(m_t, p, axis=0)
    return np.take(np.transpose(r_t), p, axis=0)
Example #22
0
def _csd_array(x, sfreq, window_fun, eigvals, freq_mask, freq_mask_mt, n_fft,
               mode, mt_adaptive):
    """Calculate Fourier transform using multitaper module.

    The arguments correspond to the values in `compute_csd_epochs` and
    `csd_array`.
    """
    x_mt, _ = _mt_spectra(x, window_fun, sfreq, n_fft)

    if mt_adaptive:
        # Compute adaptive weights
        _, weights = _psd_from_mt_adaptive(x_mt, eigvals, freq_mask,
                                           return_weights=True)
        # Tiling weights so that we can easily use _csd_from_mt()
        weights = weights[:, np.newaxis, :, :]
        weights = np.tile(weights, [1, x_mt.shape[0], 1, 1])
    else:
        # Do not use adaptive weights
        if mode == 'multitaper':
            weights = np.sqrt(eigvals)[np.newaxis, np.newaxis, :, np.newaxis]
        else:
            # Hack so we can sum over axis=-2
            weights = np.array([1.])[:, np.newaxis, np.newaxis, np.newaxis]

    x_mt = x_mt[:, :, freq_mask_mt]

    # Calculating CSD
    # Tiling x_mt so that we can easily use _csd_from_mt()
    x_mt = x_mt[:, np.newaxis, :, :]
    x_mt = np.tile(x_mt, [1, x_mt.shape[0], 1, 1])
    y_mt = np.transpose(x_mt, axes=[1, 0, 2, 3])
    weights_y = np.transpose(weights, axes=[1, 0, 2, 3])
    csds = _csd_from_mt(x_mt, y_mt, weights, weights_y)

    return csds
def wide_dataset_large():
  print("Reading in Arcene training data for binomial modeling.")
  trainDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train_labels.labels"), delimiter=' ')
  trainDataResponse = np.where(trainDataResponse == -1, 0, 1)
  trainDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train.data"), delimiter=' ')
  xtrain = np.transpose(trainDataFeatures).tolist()
  ytrain = trainDataResponse.tolist()
  trainData = h2o.H2OFrame.fromPython([ytrain]+xtrain)

  trainData[0] = trainData[0].asfactor()

  print("Run model on 3250 columns of Arcene with strong rules off.")
  model = H2OGeneralizedLinearEstimator(family="binomial", lambda_search=False, alpha=1)
  model.train(x=range(1,3250), y=0, training_frame=trainData)

  print("Test model on validation set.")
  validDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid_labels.labels"), delimiter=' ')
  validDataResponse = np.where(validDataResponse == -1, 0, 1)
  validDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid.data"), delimiter=' ')
  xvalid = np.transpose(validDataFeatures).tolist()
  yvalid = validDataResponse.tolist()
  validData = h2o.H2OFrame.fromPython([yvalid]+xvalid)
  prediction = model.predict(validData)

  print("Check performance of predictions.")
  performance = model.model_performance(validData)

  print("Check that prediction AUC better than guessing (0.5).")
  assert performance.auc() > 0.5, "predictions should be better then pure chance"
Example #24
0
    def prettyPrint(self, include_header=False, transpose_data=False, column_limit=None, col_sep="\t"):
        """Returns a string method of the data and character order.

        include_header: include charcter order or not
        transpose_data: data as is (rows are positions) or transposed
            (rows are characters) to line it up with an alignment
        column_limit = int, maximum number of columns displayed
        col_sep = string, column separator
        """
        h = self.CharOrder
        d = self.Data
        if column_limit is None:
            max_col_idx = d.shape[1]
        else:
            max_col_idx = column_limit
        if include_header and not transpose_data:
            r = [h] + d.tolist()
        elif include_header and transpose_data:
            r = [[x] + y for x, y in zip(h, transpose(d).tolist())]
        elif transpose_data:
            r = transpose(d).tolist()
        else:
            r = d.tolist()
        # resize the result based on the column limit
        if column_limit is not None:
            r = [row[:column_limit] for row in r]
        # nicely format the table content, discard the header (already included)
        if r:
            new_header, formatted_res = formattedCells(r)
        else:
            formatted_res = r
        return "\n".join([col_sep.join(map(str, i)) for i in formatted_res])
	def _forward( self, inputs ):
		""" Forward phase.
		Returns the calculated outputs.
		"""

		ones = -ny.ones( ( len( inputs ), 1 ) )

		# Activation in hidden layer
		self.hidden = ny.dot( inputs, self.weights_layer1 )
		self.hidden = 1.0 / ( 1.0 + ny.exp( -self.beta * self.hidden ) )
		self.hidden = ny.concatenate( ( self.hidden, ones ), axis = 1 )

		# Acitvation in output layer
		outputs = ny.dot( self.hidden, self.weights_layer2 )

		if self.outtype == "linear":
			pass
		elif self.outtype == "logistic":
			outputs = 1.0 / ( 1.0 + ny.exp( -self.beta * outputs ) )
		elif self.outtype == 'softmax':
			normalizers = ny.sum( ny.exp( outputs ), axis = 1 ) * ny.ones( ( 1, ny.shape( outputs )[0] ) )
			outputs = ny.transpose( ny.transpose( ny.exp( outputs ) ) / normalizers )
		else:
			print "ERROR: Unknown outtype = %s" % outtype

		return outputs
Example #26
0
    def test_putting_filters_back_together_from_fitted_basis_function_coefficients(self):
        """
        Make sure we can properly put filters back together if we have found their coefficients.
        """
        cc = np.concatenate

        tsm = time_series.Munger()
        tsm.delay = 10  # we will assume there is a 10 timestep delay between input and response

        # we will assume x1 is used directly and x2, x3, and y are filtered, and that their
        # filters are represented by sums of sinusoidal basis functions
        # (the details don't matter as we're just making sure all the arrays are getting
        # rearranged correctly)
        t = np.linspace(0, np.pi, 20)
        sin_basis = np.transpose([np.sin(t), np.sin(2*t), np.sin(3*t), np.sin(4*t)])
        t_short = t[:10]
        sin_basis_short = np.transpose([np.sin(t_short), np.sin(2*t_short)])
        tsm.basis_in = [None, sin_basis, sin_basis]
        tsm.basis_out = sin_basis_short

        coeffs = cc([[-1.], [.3], [1, 2, 3, 4], [5, 6, 7, 8], [-3, -4]])
        constant, in_filters, out_filter = tsm.filters_from_coeffs(coeffs)

        self.assertAlmostEqual(constant, coeffs[0])
        self.assertAlmostEqual(in_filters[0], coeffs[1])
        np.testing.assert_array_almost_equal(in_filters[1], sin_basis.dot(coeffs[2:6]))
        np.testing.assert_array_almost_equal(in_filters[2], sin_basis.dot(coeffs[6:10]))
        np.testing.assert_array_almost_equal(out_filter, sin_basis_short.dot(coeffs[10:12]))
Example #27
0
 def open_file(self,filename):
     self.statusBar().showMessage('Loading {}'.format(os.path.basename(filename)))
     t=time.time()
     Tiff=tifffile.TiffFile(filename)
     try:
         metadata=Tiff[0].image_description
         metadata = self.txt2dict(metadata)
     except AttributeError:
         metadata=dict()
     tif=Tiff.asarray().astype(np.float64)
     Tiff.close()        
     #tif=imread(filename,plugin='tifffile').astype(g.m.settings['internal_data_type'])
     if len(tif.shape)>3: # WARNING THIS TURNS COLOR movies TO BLACK AND WHITE BY AVERAGING ACROSS THE THREE CHANNELS
         tif=np.mean(tif,3)
     tif=np.squeeze(tif) #this gets rid of the meaningless 4th dimention in .stk files
     if len(tif.shape)==3: #this could either be a movie or a colored still frame
         if tif.shape[2]==3: #this is probably a colored still frame
             tif=np.mean(tif,2)
             tif=np.transpose(tif,(1,0)) # This keeps the x and y the same as in FIJI. 
         else:
             tif=np.transpose(tif,(0,2,1)) # This keeps the x and y the same as in FIJI. 
     elif len(tif.shape)==2: # I haven't tested whether this preserved the x y and keeps it the same as in FIJI.  TEST THIS!!
         tif=np.transpose(tif,(0,1))
     self.statusBar().showMessage('{} successfully loaded ({} s)'.format(os.path.basename(filename), time.time()-t))
     return tif  
 def testImplicitLargeDiag(self):
   mu = np.array([[1., 2, 3],
                  [11, 22, 33]])      # shape: [b, k] = [2, 3]
   u = np.array([[[1., 2],
                  [3, 4],
                  [5, 6]],
                 [[0.5, 0.75],
                  [1, 0.25],
                  [1.5, 1.25]]])      # shape: [b, k, r] = [2, 3, 2]
   m = np.array([[0.1, 0.2],
                 [0.4, 0.5]])         # shape: [b, r] = [2, 2]
   scale = np.stack([
       np.eye(3) + np.matmul(np.matmul(u[0], np.diag(m[0])),
                             np.transpose(u[0])),
       np.eye(3) + np.matmul(np.matmul(u[1], np.diag(m[1])),
                             np.transpose(u[1])),
   ])
   cov = np.stack([np.matmul(scale[0], scale[0].T),
                   np.matmul(scale[1], scale[1].T)])
   logging.vlog(2, "expected_cov:\n{}".format(cov))
   with self.test_session():
     mvn = ds.MultivariateNormalDiagPlusLowRank(
         loc=mu,
         scale_perturb_factor=u,
         scale_perturb_diag=m)
     self.assertAllClose(cov, mvn.covariance().eval(), atol=0., rtol=1e-6)
Example #29
0
def fitt(XX, TT, DD, lam, p, fun=fmin_ncg):
    X = XX
    T = np.reshape(TT, (len(TT), 1))
    D = np.reshape(DD, (1, len(DD)))

    sdi = sum(DD)
    ss = float(( np.dot(D,np.log(T)) ) * (p-1) + (math.log(lam*p))*sdi)
    #print ss, sdi
    ltpt  = np.reshape(lam*np.power(T, p), (len(T), 1) )
    ltp  = np.transpose(ltpt)
    Dead = D
    Time = T
    DT = np.transpose(D)
    XT = np.transpose(X)

    def obj_grad  (beta):
        K = np.dot(X , beta)
        v = ( DT - np.multiply( ltpt , derexpp(K).reshape( (len(T), 1)  )  )  )
        vl =  - (np.dot (XT , v) )
        return vl[:,0]

    def obj_value (beta):
        K = np.dot(X , beta)
        vl= -(ss + np.dot(Dead , K) - np.dot(ltp,expp(K)))
        return float(vl)

    bopt = fun(obj_value, np.zeros( np.shape(X)[1] ), fprime=obj_grad)
    return bopt
Example #30
0
def one_schedule_var(P, mu, K, q, N, ucb_avg, l, R):
    # one scheduling step
    # P : permutation matrix
    # mu : actual transmission rate
    # K : number of queues
    # q : current queuelengths
    # N : number of samplings to be updates
    # avg : number of samplings
    # R : maximum alphabet size
    rates = np.diag(P * np.transpose(mu))
    # print P
    # print "rates", rates
    for i in range(K):
        bit = random.random()
        if bit <= l[i]:
            q[i] = q[i] + 1
    nodes = P * np.transpose(np.matrix(range(K)))
    # print nodes
    for i in range(K):
        bit = random.random()
        if bit <= float(rates[i] / R):
            # print "testing"
            if q[i] - R >= 0:
                q[i] = q[i] - R
            else:
                q[i] = 0
            ucb_avg[i, int(nodes[i])] = (N[i, int(nodes[i])] * ucb_avg[i, int(nodes[i])] + 1) / (
                N[i, int(nodes[i])] + 1
            )
            N[i, int(nodes[i])] = N[i, int(nodes[i])] + 1
        else:
            ucb_avg[i, int(nodes[i])] = (N[i, int(nodes[i])] * ucb_avg[i, int(nodes[i])]) / (N[i, int(nodes[i])] + 1)
            N[i, int(nodes[i])] = N[i, int(nodes[i])] + 1
Example #31
0
def imshow(img):
    img = img / 2 + 0.5  # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()
    def _augment_images_with_patch(
        x: np.ndarray,
        patch: np.ndarray,
        random_location: bool,
        channels_first: bool,
        mask: Optional[np.ndarray] = None,
    ) -> Tuple[np.ndarray, List[Dict[str, int]]]:
        """
        Augment images with patch.

        :param x: Sample images.
        :param patch: The patch to be applied.
        :param random_location: If True apply patch at randomly shifted locations, otherwise place patch at origin
                                (top-left corner).
        :param channels_first: Set channels first or last.
        :param mask: An boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x`
                     (N, H, W) without their channel dimensions. Any features for which the mask is True can be the
                     center location of the patch during sampling.
        :type mask: `np.ndarray`
        """
        transformations = list()
        x_copy = x.copy()
        patch_copy = patch.copy()

        if channels_first:
            x_copy = np.transpose(x_copy, (0, 2, 3, 1))
            patch_copy = np.transpose(patch_copy, (1, 2, 0))

        for i_image in range(x.shape[0]):

            if random_location:
                if mask is None:
                    i_x_1 = random.randint(0, x_copy.shape[1] - 1 - patch_copy.shape[0])
                    i_y_1 = random.randint(0, x_copy.shape[2] - 1 - patch_copy.shape[1])
                else:

                    if mask.shape[0] == 1:
                        mask_2d = mask[0, :, :]
                    else:
                        mask_2d = mask[i_image, :, :]

                    edge_x_0 = patch_copy.shape[0] // 2
                    edge_x_1 = patch_copy.shape[0] - edge_x_0
                    edge_y_0 = patch_copy.shape[1] // 2
                    edge_y_1 = patch_copy.shape[1] - edge_y_0

                    mask_2d[0:edge_x_0, :] = False
                    mask_2d[-edge_x_1:, :] = False
                    mask_2d[:, 0:edge_y_0] = False
                    mask_2d[:, -edge_y_1:] = False

                    num_pos = np.argwhere(mask_2d).shape[0]
                    pos_id = np.random.choice(num_pos, size=1)
                    pos = np.argwhere(mask_2d > 0)[pos_id[0]]
                    i_x_1 = pos[0] - edge_x_0
                    i_y_1 = pos[1] - edge_y_0

            else:
                i_x_1 = 0
                i_y_1 = 0

            i_x_2 = i_x_1 + patch_copy.shape[0]
            i_y_2 = i_y_1 + patch_copy.shape[1]

            transformations.append({"i_x_1": i_x_1, "i_y_1": i_y_1, "i_x_2": i_x_2, "i_y_2": i_y_2})

            x_copy[i_image, i_x_1:i_x_2, i_y_1:i_y_2, :] = patch_copy

        if channels_first:
            x_copy = np.transpose(x_copy, (0, 3, 1, 2))

        return x_copy, transformations
Example #33
0
def grad_loss(X, w, y, alpha):
    predictions = probability_prediction(X, w)
    return (1.0 / y.shape[0]) * np.dot(np.transpose(X), (predictions-y)) + alpha * w
Example #34
0
def eval_loss(X, w, y, alpha):
   predictions = probability_prediction(X, w)
   return ((-1.0 / y.shape[0]) * np.sum(np.log((1 - y) * (1 - predictions) + (y) * (predictions)))) + ((alpha / (2.0*y.shape[0])) * (np.dot(np.transpose(w), w)))
Example #35
0
    def evaluate(self, outputs, res_folder, metric='PCKh', **kwargs):
        """Evaluate PCKh for MPII dataset. Adapted from
        https://github.com/leoxiaobin/deep-high-resolution- net.pytorch
        Original licence: Copyright (c) Microsoft, under the MIT License.

        Note:
            batch_size: N
            num_keypoints: K
            heatmap height: H
            heatmap width: W

        Args:
            outputs(list(preds, boxes, image_path, output_heatmap)):

                * preds(np.ndarray[1,K,3]): The first two dimensions are
                  coordinates, score is the third dimension of the array.
                * boxes(np.ndarray[1,6]): [center[0], center[1], scale[0]
                  , scale[1],area, score]
                * image_path(list[str]): For example, ['0', '0',
                  '0', '0', '0', '1', '1', '6', '3', '.', 'j', 'p', 'g']
                * output_heatmap (np.ndarray[N, K, H, W]): model outputs.

            res_folder(str): Path of directory to save the results.
            metric (str | list[str]): Metrics to be performed.
                Defaults: 'PCKh'.

        Returns:
            dict: PCKh for each joint
        """

        metrics = metric if isinstance(metric, list) else [metric]
        allowed_metrics = ['PCKh']
        for metric in metrics:
            if metric not in allowed_metrics:
                raise KeyError(f'metric {metric} is not supported')

        preds = np.stack([kpts[0] for kpts, _, _, _ in outputs])

        # convert 0-based index to 1-based index,
        # and get the first two dimensions.
        preds = preds[..., :2] + 1.0

        if res_folder:
            pred_file = os.path.join(res_folder, 'pred.mat')
            savemat(pred_file, mdict={'preds': preds})

        SC_BIAS = 0.6
        threshold = 0.5

        gt_file = os.path.join(os.path.dirname(self.ann_file),
                               'mpii_gt_val.mat')
        gt_dict = loadmat(gt_file)
        dataset_joints = gt_dict['dataset_joints']
        jnt_missing = gt_dict['jnt_missing']
        pos_gt_src = gt_dict['pos_gt_src']
        headboxes_src = gt_dict['headboxes_src']

        pos_pred_src = np.transpose(preds, [1, 2, 0])

        head = np.where(dataset_joints == 'head')[1][0]
        lsho = np.where(dataset_joints == 'lsho')[1][0]
        lelb = np.where(dataset_joints == 'lelb')[1][0]
        lwri = np.where(dataset_joints == 'lwri')[1][0]
        lhip = np.where(dataset_joints == 'lhip')[1][0]
        lkne = np.where(dataset_joints == 'lkne')[1][0]
        lank = np.where(dataset_joints == 'lank')[1][0]

        rsho = np.where(dataset_joints == 'rsho')[1][0]
        relb = np.where(dataset_joints == 'relb')[1][0]
        rwri = np.where(dataset_joints == 'rwri')[1][0]
        rkne = np.where(dataset_joints == 'rkne')[1][0]
        rank = np.where(dataset_joints == 'rank')[1][0]
        rhip = np.where(dataset_joints == 'rhip')[1][0]

        jnt_visible = 1 - jnt_missing
        uv_error = pos_pred_src - pos_gt_src
        uv_err = np.linalg.norm(uv_error, axis=1)
        headsizes = headboxes_src[1, :, :] - headboxes_src[0, :, :]
        headsizes = np.linalg.norm(headsizes, axis=0)
        headsizes *= SC_BIAS
        scale = headsizes * np.ones((len(uv_err), 1), dtype=np.float32)
        scaled_uv_err = uv_err / scale
        scaled_uv_err = scaled_uv_err * jnt_visible
        jnt_count = np.sum(jnt_visible, axis=1)
        less_than_threshold = (scaled_uv_err <= threshold) * jnt_visible
        PCKh = 100. * np.sum(less_than_threshold, axis=1) / jnt_count

        # save
        rng = np.arange(0, 0.5 + 0.01, 0.01)
        pckAll = np.zeros((len(rng), 16), dtype=np.float32)

        for r, threshold in enumerate(rng):
            less_than_threshold = (scaled_uv_err <= threshold) * jnt_visible
            pckAll[r, :] = 100. * np.sum(less_than_threshold,
                                         axis=1) / jnt_count

        PCKh = np.ma.array(PCKh, mask=False)
        PCKh.mask[6:8] = True

        jnt_count = np.ma.array(jnt_count, mask=False)
        jnt_count.mask[6:8] = True
        jnt_ratio = jnt_count / np.sum(jnt_count).astype(np.float64)

        name_value = [('Head', PCKh[head]),
                      ('Shoulder', 0.5 * (PCKh[lsho] + PCKh[rsho])),
                      ('Elbow', 0.5 * (PCKh[lelb] + PCKh[relb])),
                      ('Wrist', 0.5 * (PCKh[lwri] + PCKh[rwri])),
                      ('Hip', 0.5 * (PCKh[lhip] + PCKh[rhip])),
                      ('Knee', 0.5 * (PCKh[lkne] + PCKh[rkne])),
                      ('Ankle', 0.5 * (PCKh[lank] + PCKh[rank])),
                      ('PCKh', np.sum(PCKh * jnt_ratio)),
                      ('[email protected]', np.sum(pckAll[11, :] * jnt_ratio))]
        name_value = OrderedDict(name_value)

        return name_value
Example #36
0
 def on_enter(pts):
     x, y = np.transpose(pts)
     print("length = %0.2f" %
           np.sqrt(np.diff(x)**2 + np.diff(y)**2))
def makeSubmissionScript(featureMatrixPath,
                         outputSubmissionPath,
                         trainFakeTrips=200,
                         digits=5,
                         foldsize=10):
    #Read Feature Matrix
    featureMatrix = readFeatureMatrix.totalFeatureMatrix(featureMatrixPath)

    #ShortCut, to make loading faster, change the path if this is used.
    #featureMatrix = np.load('D:\\Documents\\Data\\MLiP\\features.npy')
    #np.save('D:\\Documents\\Data\\MLiP\\features', featureMatrix)

    print(np.shape(featureMatrix))

    #some features that are not very informative, so they are ignored
    featureMatrix = np.delete(
        featureMatrix,
        [17, 39, 42, 46, 49, 52, 53, 85, 86, 87, 126, 138, 144, 148], 0)

    #Get the driver numbers as the names of the csv files in the feature matrix path
    drivernrs = readFeatureMatrix.getdrivernrs(featureMatrixPath)
    print('Done Reading Feature matrix!')
    print(np.shape(featureMatrix))

    numFeat, _, numDrivers = np.shape(featureMatrix)
    numTrips = 200  #The number of trips to make the submission out of, always 200
    numfolds = float(
        numTrips / foldsize
    )  #the number of folds that will be made, aka the number of models per driver
    importances = np.zeros(
        (numFeat, numDrivers))  #Storage for feature importances

    #The probabilities that a trip belongs to a driver
    probabilities = np.zeros((numTrips, 2, numDrivers))

    for i in range(numDrivers):
        #First get the trainingtrips from the featurematrix. It is assumed that the real trips come first
        trainTrips = np.transpose(featureMatrix[:, :(numTrips +
                                                     trainFakeTrips), i])
        realTrips = trainTrips[:numTrips, :]
        trainLabels = np.hstack(
            (np.ones(numTrips - foldsize), np.zeros(trainFakeTrips)))

        #Training the model for each fold
        for b, e in [(k, k + foldsize)
                     for k in np.arange(0, numTrips, foldsize)]:
            testIndex = np.arange(b, e)
            trainIndex = np.hstack(
                (np.arange(0, b), np.arange(e, (numTrips + trainFakeTrips))))

            #Train the model, then predict the classes for that model
            model = learn.trainModel(trainTrips[trainIndex],
                                     trainLabels,
                                     criterion='entropy',
                                     n_trees=300,
                                     n_jobs=-1)
            #Keep track of the importance of each feature for this driver
            importances[:, i] = importances[:, i] + np.divide(
                model.feature_importances_, numfolds)

            tempprobs = learn.predictClass(model, realTrips[testIndex])

            #Append the tripnrs, then add to probability table
            probabilities[testIndex, :, i] = np.transpose(
                np.vstack((testIndex + 1, tempprobs)))

        #Progress report, it's so slow that we give a report every driver
        if i % 1 == 0:
            print("Done learning driver " + ` i `)
        #Appending the output to a file. This is reccomended when doing a long calculation because you can halt
        #the computation and still have the results for the drivers so far, but it's not needed.
        #CreateSubmission.appendProbabilities(outputSubmissionPath, drivernrs[i], probabilities[:,:,i],'%0.' + `digits` + 'f')
    print('Done calculating probabilities!')

    #Makes submission file
    fmtstring = '%0.' + ` digits ` + 'f'
    CreateSubmission.createSubmissionfileFrom3D(outputSubmissionPath,
                                                probabilities,
                                                drivernrs=drivernrs,
                                                fmtstring=fmtstring)

    return importances
Example #38
0
	plt.plot(G_losses,label="G")
	plt.plot(D_losses,label="C")
	plt.xlabel("iterations")
	plt.ylabel("Loss")
	plt.legend()
	plt.savefig(str(filestr + "_loss_plot.png"))
	plt.close()
	
	# Grab a batch of real images from the dataloader
	real_batch = next(iter(test_loader))

	# Plot the real images
	plt.figure(figsize=(15,15))
	plt.subplot(1,3,1)
	plt.axis("off")
	plt.title("Real Images")
	plt.imshow(np.transpose(vutils.make_grid(real_batch[0].to(device)[:64], padding=5, normalize=True).cpu(),(1,2,0)))

	plt.subplot(1,3,2)
	plt.axis("off")
	plt.title("Fake Images after " + str(num_epochs//2) + " epochs")
	plt.imshow(np.transpose(img_list[num_epochs//2 +1],(1,2,0)))

	# Plot the fake images from the last epoch
	plt.subplot(1,3,3)
	plt.axis("off")
	plt.title("Fake Images after " + str(num_epochs) + " epochs")
	plt.imshow(np.transpose(img_list[-1],(1,2,0)))
	plt.savefig(str(filestr + "real_v_fake.png"))
	plt.close()
def sample_random_candidates(id, df, n_sample,VOXEL_SIZE, dfcandidates,fp_only,rotate=False):
	#choose random voxels from this id,
	#if they contain a nodule, return the index of this nodule in the dataframe
	#from that we can determine size, attributes
	#TODO: TRIM IMAGE
	
	df['ix'] = range(df.shape[0])
	dfcandidates = dfcandidates[dfcandidates['seriesuid'] == id]
	dfsub = df[df['seriesuid']==id]
	nodule_coords = []
	nodule_sizes = []
	nodule_ixs = []
	
	if len(dfsub) > 0:
		for i in range(dfsub.shape[0]):
			row = dfsub.iloc[i]
			nodule_coords.append((row['coordZ'], row['coordY'], row['coordX']))
			nodule_sizes.append(row['diameter_mm'])
			
			
	img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', id  + '.mhd'))
	voxel_coords = [worldToVoxelCoord(c,origin,spacing) for c in nodule_coords]
	

	# zoom = np.random.uniform(.99,1.01)
	numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0])
	# assert numZpix > 10, 'too few z pixels'
	numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1])
	# assert numYpix > 10, 'too few y pixels'
	numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2])
	# assert numXpix > 10, 'too few x pixels'
	
	voxels = []
	ixs = []
	for i in range(n_sample):

		#choose a random row from the candidates file
		#OR choose a random voxel. 50-50.
		
		if dfcandidates.shape[0] > 0 and (np.random.randint(0,2) == 0 or fp_only):
			row = dfcandidates.iloc[np.random.choice(dfcandidates.shape[0])]
			x_center_raw = row['coordX'] * np.random.uniform(.9,1.1)
			y_center_raw = row['coordY'] * np.random.uniform(.9,1.1)
			z_center_raw = row['coordZ'] * np.random.uniform(.9,1.1)
			vcords = worldToVoxelCoord((z_center_raw, y_center_raw, x_center_raw), origin, spacing)
			x_center = vcords[2]
			y_center = vcords[1]
			z_center = vcords[0]
			ix = -2
			#now - if the coordinates are too close to an edge just default to random ones
			if x_center < numXpix/2 or x_center > img.shape[2]-numXpix/2 or \
				y_center < numYpix/2 or y_center > img.shape[1]-numYpix/2 or \
				z_center < numZpix/2 or z_center > img.shape[0]-numZpix/2:
				x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2)
				y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2)
				z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2)
				ix = -1
		else:
			#print 'no candidates for id', id
			x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2)
			y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2)
			z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2)
			ix = -1
			
		z_start = np.clip(z_center-numZpix/2, 0, img.shape[0])
		z_end = np.clip(z_center+numZpix/2, 0, img.shape[0])
		y_start = np.clip(y_center-numYpix/2, 0, img.shape[1])
		y_end = np.clip(y_center+numYpix/2, 0, img.shape[1])
		x_start = np.clip(x_center-numXpix/2, 0, img.shape[2])
		x_end = np.clip(x_center+numXpix/2, 0, img.shape[2])

		voxel = img[z_start:z_end,y_start:y_end,x_start:x_end]

		voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE))

		voxel_norm = np.clip(voxel_norm, -1000, 400)
		if rotate:
			voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(1,0), order=1,reshape=False,cval=-1000,mode='nearest')
			voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(2,1), order=1,reshape=False,cval=-1000,mode='nearest')

		#apply a random rotation
		
		#determine index (if applicable)
		#if no match put -1.
		largest_diam = 0
		for i,(coord,size) in enumerate(zip(voxel_coords,nodule_sizes)):
			if (x_start  < coord[2] < x_end) and (y_start < coord[1] < y_end ) and (z_start < coord[0] < z_end):
				#we got one
				if dfsub.iloc[i]['diameter_mm'] > largest_diam:
					largest_diam = dfsub.iloc[i]['diameter_mm']
					ix = dfsub.iloc[i]['ix']
				#target = size
		voxels.append(np.transpose(voxel_norm, (2,1,0)))
		ixs.append(ix)
		
	return np.stack(voxels),np.stack(ixs)
Example #40
0
    x_mesh_centralise = x_mesh_centralise_all
    y_mesh_centralise = y_mesh_centralise_all

print('Data Collection and Binning Completed')
# ------------------------------------------End of SELECTION FOR EXCLUSION OF ZERO POINTS

# ------------------------------------------ Tabulate Poisson Likelihood

# Use the k_quad and latent intensity v array to obtain likelihood
# The k_quad now contains data from year 2014
# Latent_v_array was optimized based on


# Generate Objective Function: log(P(D|v))
exp_term = -1 * np.sum(np.exp(latent_v_array))
product_term = np.matmul(latent_v_array, np.transpose(k_quad))

factorial_k = scispec.gamma(k_quad + np.ones_like(k_quad))
# factorial_term = - np.sum(np.log(factorial_k))  # summation of logs = log of product
factorial_term = - np.sum(fn.log_special(factorial_k))  # summation of logs = log of product

log_p_likelihood = exp_term + product_term + factorial_term

print('The Log Poisson Likelihood is', log_p_likelihood)

# ------------------------------------------------------- Tabulate the GP Log Marginal Likelihood

# The Covariance Matrix and Prior mean are created here as a component of the objective function
prior_mean = mean_func_scalar(mean_optimal, xy_quad)

# Select Kernel and Construct Covariance Matrix
Example #41
0
                           Level_ratio[car_id*(num_cars-1)+count, 1] = Level_ratio[car_id*(num_cars-1) + count, 1] + 0.5

                        Level_ratio[car_id*(num_cars-1)+count,:] = Level_ratio[car_id*(num_cars-1)+count,:]/ sum(Level_ratio[car_id*(num_cars-1)+count,:])   # normalizing
                        count = count+1
        
        print(Level_ratio)

                
        # State update
        X_new, R = Environment_Multi_Sim.Environment_Multi_Sim(X_old, Action_id, params.t_step_Sim, params)
        X_old = X_new


        # Reward plot
        color = ['b','r','m','g']
        R_history[episode, :, step] = np.transpose(R)
        
        # plt.figure(2)
        # plt.plot(step,R[0],color='b',marker='.',markersize=16)
        # plt.plot(step,R[1],color='r',marker='.',markersize=16)


        # Plot the level_history
        ego_car_id = 1  # AV
        opp_car_id = 3  # Car 4
        plot_level_ratio.plot_level_ratio(Level_ratio_history, ego_car_id, opp_car_id, params, step, episode, max_step, fig_lh)

        # Timer
        t1 = time.time()
        time_per_step = t1 - t0
        # print(time_per_step)
def get_bounding_voxels_new(patient, df, n,VOXEL_SIZE,rotate=False):
	#given the nodule index and the nodule dataframe
	#return n jittered views of the nodule and n copies of the row index
	#from the row index we can look up the nodule size, malignancy, etc.
	df['ix'] = range(df.shape[0])
	dfsub = df[df['seriesuid']==patient]
	if dfsub.shape[0] == 0:
		return None
	img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', patient  + '.mhd'))
	
	#now pick out VOXEL_SIZE mm of pixels in each dimension.
	# zoom = np.random.uniform(.99,1.01)
	numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0])
	# assert numZpix > 10, 'too few z pixels'
	numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1])
	# assert numYpix > 10, 'too few y pixels'
	numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2])
	# assert numXpix > 10, 'too few x pixels'
	
	
	voxels = []
	indices = []
	for i in range(n):
	
		#choose a random nodule from this patient
		row = dfsub.iloc[ np.random.choice(dfsub.shape[0]) ]
		coords = (row['coordZ'], row['coordY'], row['coordX'])
		diameter_mm = row['diameter_mm']
		
		voxel_coords = worldToVoxelCoord(coords, origin, spacing)
		voxel_coords = np.round(voxel_coords)
		
		#fuzz 
		max_z_fuzz = int((numZpix/2) * (1 - diameter_mm / VOXEL_SIZE))
		max_y_fuzz = int((numYpix/2) * (1 - diameter_mm / VOXEL_SIZE))
		max_x_fuzz = int((numXpix/2) * (1 - diameter_mm / VOXEL_SIZE))
		zfuzz = np.random.randint(-max_z_fuzz, max_z_fuzz+1) if max_z_fuzz > 0 else 0
		yfuzz = np.random.randint(-max_y_fuzz, max_y_fuzz+1) if max_y_fuzz > 0  else 0
		xfuzz = np.random.randint(-max_x_fuzz, max_x_fuzz+1) if max_x_fuzz > 0 else 0


		z_start = np.clip(voxel_coords[0] + zfuzz - numZpix/2, 0, img.shape[0])
		z_end = np.clip(voxel_coords[0] + zfuzz +numZpix/2, 0, img.shape[0])
		y_start = np.clip(voxel_coords[1]+ yfuzz -numYpix/2, 0, img.shape[1])
		y_end = np.clip(voxel_coords[1] + yfuzz +numYpix/2, 0, img.shape[1])
		x_start = np.clip(voxel_coords[2]+xfuzz-numXpix/2, 0, img.shape[2])
		x_end = np.clip(voxel_coords[2]+xfuzz +numXpix/2, 0, img.shape[2])
			
		#now let's see if this voxel contains more than one nodule
		num_nodules = 0
		maxdiam_ix = -1
		maxdiam = 0
		for j in range(dfsub.shape[0]):
			row_j = dfsub.iloc[j]
			row_coords = (row_j['coordZ'], row_j['coordY'], row_j['coordX'])
			row_voxel_coords = worldToVoxelCoord(row_coords, origin, spacing)
			
			if (row_voxel_coords[2] > x_start+0 and row_voxel_coords[2] < x_end-0) and \
				(row_voxel_coords[1] > y_start+0 and row_voxel_coords[1] < y_end-0) and \
				(row_voxel_coords[0] > z_start+0 and row_voxel_coords[0] < z_end-0):
				
				#found one
				num_nodules += 1
				if row_j['diameter_mm'] > maxdiam:
					maxdiam_ix = row_j['ix']
					maxdiam = row_j['diameter_mm']
		
		if num_nodules == 0:
			# print 'no nodules in region!'
			# print x_start, x_end, y_start, y_end, z_start, z_end
			# print voxel_coords
			maxdiam_ix = -1
			
		# assert num_nodules > 0, 'no nodules in region'
		# if num_nodules > 1:
			# print 'multiple nodules found in voxel. choosing largest'
			
		indices.append(maxdiam_ix)
		
		voxel = img[z_start:z_end,y_start:y_end,x_start:x_end]
		
	
		# print voxel.shape, spacing
		voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE))
		if rotate:
			voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(1,0), order=1,reshape=False,cval=-1000,mode='nearest')
			voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(2,1), order=1,reshape=False,cval=-1000,mode='nearest')

		# halfsize = size/2
		voxel_norm = np.clip(voxel_norm, -1000, 400)
		voxel_norm = np.transpose(voxel_norm, (2,1,0)) #X,Y,Z 
		voxels.append(voxel_norm)
		
	return np.stack(voxels), np.array(indices).astype('int32')
Example #43
0
    def fit(self, images, indices=[slice(None), slice(None)]):
        """
        This method uses the cnmf algorithm to find sources in data.
        it is calling every function from the cnmf folder
        you can find out more at how the functions are called and how they are laid out at the ipython notebook

        Args:
            images : mapped np.ndarray of shape (t,x,y[,z]) containing the images that vary over time.

            indices: list of slice objects along dimensions (x,y[,z]) for processing only part of the FOV

        Returns:
            self: updated using the cnmf algorithm with C,A,S,b,f computed according to the given initial values

        Raises:
        Exception 'You need to provide a memory mapped file as input if you use patches!!'

        See Also:
        ..image::docs/img/quickintro.png

        http://www.cell.com/neuron/fulltext/S0896-6273(15)01084-3

        """
        # Todo : to compartment
        if isinstance(indices, slice):
            indices = [indices]
        indices = [slice(None)] + indices
        if len(indices) < len(images.shape):
            indices = indices + [slice(None)
                                 ] * (len(images.shape) - len(indices))
        dims_orig = images.shape[1:]
        dims_sliced = images[tuple(indices)].shape[1:]
        is_sliced = (dims_orig != dims_sliced)
        if self.params.get('patch', 'rf') is None and (is_sliced or 'ndarray'
                                                       in str(type(images))):
            images = images[tuple(indices)]
            self.dview = None
            logging.warning("Parallel processing in a single patch "
                            "is not available for loaded in memory or sliced" +
                            " data.")

        T = images.shape[0]
        self.params.set('online', {'init_batch': T})
        self.dims = images.shape[1:]
        #self.params.data['dims'] = images.shape[1:]
        Y = np.transpose(images, list(range(1, len(self.dims) + 1)) + [0])
        Yr = np.transpose(np.reshape(images, (T, -1), order='F'))
        if np.isfortran(Yr):
            raise Exception(
                'The file is in F order, it should be in C order (see save_memmap function)'
            )

        logging.info((T, ) + self.dims)

        # Make sure filename is pointed correctly (numpy sets it to None sometimes)
        try:
            Y.filename = images.filename
            Yr.filename = images.filename
        except AttributeError:  # if no memmapping cause working with small data
            pass

        # update/set all options that depend on data dimensions
        # number of rows, columns [and depths]
        self.params.set(
            'spatial', {
                'medw': (3, ) * len(self.dims),
                'se': np.ones((3, ) * len(self.dims), dtype=np.uint8),
                'ss': np.ones((3, ) * len(self.dims), dtype=np.uint8)
            })

        logging.info(('Using ' + str(self.params.get('patch', 'n_processes')) +
                      ' processes'))
        if self.params.get('preprocess', 'n_pixels_per_process') is None:
            avail_memory_per_process = psutil.virtual_memory(
            )[1] / 2.**30 / self.params.get('patch', 'n_processes')
            mem_per_pix = 3.6977678498329843e-09
            npx_per_proc = np.int(avail_memory_per_process / 8. / mem_per_pix /
                                  T)
            npx_per_proc = np.int(
                np.minimum(
                    npx_per_proc,
                    np.prod(self.dims) //
                    self.params.get('patch', 'n_processes')))
            self.params.set('preprocess',
                            {'n_pixels_per_process': npx_per_proc})

        self.params.set(
            'spatial', {
                'n_pixels_per_process':
                self.params.get('preprocess', 'n_pixels_per_process')
            })

        logging.info(
            'using ' +
            str(self.params.get('preprocess', 'n_pixels_per_process')) +
            ' pixels per process')
        logging.info('using ' +
                     str(self.params.get('spatial', 'block_size_spat')) +
                     ' block_size_spat')
        logging.info('using ' +
                     str(self.params.get('temporal', 'block_size_temp')) +
                     ' block_size_temp')

        if self.params.get('patch', 'rf') is None:  # no patches
            logging.info('preprocessing ...')
            Yr = self.preprocess(Yr)
            if self.estimates.A is None:
                logging.info('initializing ...')
                self.initialize(Y)

            if self.params.get(
                    'patch',
                    'only_init'):  # only return values after initialization
                if not (self.params.get('init', 'method_init') == 'corr_pnr'
                        and self.params.get('init',
                                            'ring_size_factor') is not None):
                    self.compute_residuals(Yr)
                    self.estimates.bl = None
                    self.estimates.c1 = None
                    self.estimates.neurons_sn = None

                if self.remove_very_bad_comps:
                    logging.info('removing bad components : ')
                    final_frate = 10
                    r_values_min = 0.5  # threshold on space consistency
                    fitness_min = -15  # threshold on time variability
                    fitness_delta_min = -15
                    Npeaks = 10
                    traces = np.array(self.C)
                    logging.info('estimating the quality...')
                    idx_components, idx_components_bad, fitness_raw,\
                        fitness_delta, r_values = estimate_components_quality(
                            traces, Y, self.estimates.A, self.estimates.C, self.estimates.b, self.estimates.f,
                            final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min,
                            fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True, N=5)

                    logging.info(
                        ('Keeping ' + str(len(idx_components)) +
                         ' and discarding  ' + str(len(idx_components_bad))))
                    self.estimates.C = self.estimates.C[idx_components]
                    self.estimates.A = self.estimates.A[:,
                                                        idx_components]  # type: ignore # not provable that self.initialise provides a value
                    self.estimates.YrA = self.estimates.YrA[idx_components]

                self.estimates.normalize_components()

                return self

            logging.info('update spatial ...')
            self.update_spatial(Yr, use_init=True)

            logging.info('update temporal ...')
            if not self.skip_refinement:
                # set this to zero for fast updating without deconvolution
                self.params.set('temporal', {'p': 0})
            else:
                self.params.set('temporal',
                                {'p': self.params.get('preprocess', 'p')})
            logging.info('deconvolution ...')

            self.update_temporal(Yr)

            if not self.skip_refinement:
                logging.info('refinement...')
                if self.params.get('merging', 'do_merge'):
                    logging.info('merging components ...')
                    self.merge_comps(Yr, mx=50, fast_merge=True)

                logging.info('Updating spatial ...')

                self.update_spatial(Yr, use_init=False)
                # set it back to original value to perform full deconvolution
                self.params.set('temporal',
                                {'p': self.params.get('preprocess', 'p')})
                logging.info('update temporal ...')
                self.update_temporal(Yr, use_init=False)
            # else:
            #     todo : ask for those..
            # C, f, S, bl, c1, neurons_sn, g1, YrA, lam = self.estimates.C, self.estimates.f, self.estimates.S, self.estimates.bl, self.estimates.c1, self.estimates.neurons_sn, self.estimates.g, self.estimates.YrA, self.estimates.lam

            # embed in the whole FOV
            if is_sliced:
                FOV = np.zeros(dims_orig, order='C')
                FOV[indices[1:]] = 1
                FOV = FOV.flatten(order='F')
                ind_nz = np.where(FOV > 0)[0].tolist()
                self.estimates.A = self.estimates.A.tocsc()
                A_data = self.estimates.A.data
                A_ind = np.array(ind_nz)[self.estimates.A.indices]
                A_ptr = self.estimates.A.indptr
                A_FOV = scipy.sparse.csc_matrix(
                    (A_data, A_ind, A_ptr),
                    shape=(FOV.shape[0], self.estimates.A.shape[-1]))
                b_FOV = np.zeros((FOV.shape[0], self.estimates.b.shape[-1]))
                b_FOV[ind_nz] = self.estimates.b
                self.estimates.A = A_FOV
                self.estimates.b = b_FOV

        else:  # use patches
            if self.params.get('patch', 'stride') is None:
                self.params.set('patch', {
                    'stride':
                    np.int(self.params.get('patch', 'rf') * 2 * .1)
                })
                logging.info(
                    ('Setting the stride to 10% of 2*rf automatically:' +
                     str(self.params.get('patch', 'stride'))))

            if type(images) is np.ndarray:
                raise Exception(
                    'You need to provide a memory mapped file as input if you use patches!!'
                )

            self.estimates.A, self.estimates.C, self.estimates.YrA, self.estimates.b, self.estimates.f, \
                self.estimates.sn, self.estimates.optional_outputs = run_CNMF_patches(
                    images.filename, self.dims + (T,), self.params,
                    dview=self.dview, memory_fact=self.params.get('patch', 'memory_fact'),
                    gnb=self.params.get('init', 'nb'), border_pix=self.params.get('patch', 'border_pix'),
                    low_rank_background=self.params.get('patch', 'low_rank_background'),
                    del_duplicates=self.params.get('patch', 'del_duplicates'),
                    indices=indices)

            self.estimates.bl, self.estimates.c1, self.estimates.g, self.estimates.neurons_sn = None, None, None, None
            logging.info("merging")
            self.estimates.merged_ROIs = [0]

            if self.params.get('init',
                               'center_psf'):  # merge taking best neuron
                if self.params.get('patch', 'nb_patch') > 0:

                    while len(self.estimates.merged_ROIs) > 0:
                        self.merge_comps(Yr, mx=np.Inf, fast_merge=True)

                    logging.info("update temporal")
                    self.update_temporal(Yr, use_init=False)

                    self.params.set(
                        'spatial', {
                            'se': np.ones(
                                (1, ) * len(self.dims), dtype=np.uint8)
                        })
                    logging.info('update spatial ...')
                    self.update_spatial(Yr, use_init=False)

                    logging.info("update temporal")
                    self.update_temporal(Yr, use_init=False)
                else:
                    while len(self.estimates.merged_ROIs) > 0:
                        self.merge_comps(Yr, mx=np.Inf, fast_merge=True)
                        if len(self.estimates.merged_ROIs) > 0:
                            not_merged = np.setdiff1d(
                                list(range(len(self.estimates.YrA))),
                                np.unique(
                                    np.concatenate(
                                        self.estimates.merged_ROIs)))
                            self.estimates.YrA = np.concatenate([
                                self.estimates.YrA[not_merged],
                                np.array([
                                    self.estimates.YrA[m].mean(0) for ind, m in
                                    enumerate(self.estimates.merged_ROIs)
                                    if not self.empty_merged[ind]
                                ])
                            ])
                    if self.params.get('init', 'nb') == 0:
                        self.estimates.W, self.estimates.b0 = compute_W(
                            Yr,
                            self.estimates.A.toarray(),
                            self.estimates.C,
                            self.dims,
                            self.params.get('init', 'ring_size_factor') *
                            self.params.get('init', 'gSiz')[0],
                            ssub=self.params.get('init', 'ssub_B'))
                    if len(self.estimates.C):
                        self.deconvolve()
                    else:
                        self.estimates.S = self.estimates.C
            else:
                while len(self.estimates.merged_ROIs) > 0:
                    self.merge_comps(Yr, mx=np.Inf)

                logging.info("update temporal")
                self.update_temporal(Yr, use_init=False)

        self.estimates.normalize_components()
        return self
def sample_random_voxels(id, df, n_sample,VOXEL_SIZE):
	#choose random voxels from this id,
	#if they contain a nodule, return the index of this nodule in the dataframe
	#from that we can determine size, attributes
	#TODO: TRIM IMAGE
	
	df['ix'] = range(df.shape[0])
	dfsub = df[df['seriesuid']==id]
	nodule_coords = []
	nodule_sizes = []
	nodule_ixs = []
	
	if len(dfsub) > 0:
		for i in range(dfsub.shape[0]):
			row = dfsub.iloc[i]
			nodule_coords.append((row['coordZ'], row['coordY'], row['coordX']))
			nodule_sizes.append(row['diameter_mm'])
			
			
	img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', id  + '.mhd'))
	voxel_coords = [worldToVoxelCoord(c,origin,spacing) for c in nodule_coords]
	

	# zoom = np.random.uniform(.99,1.01)
	numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0])
	# assert numZpix > 10, 'too few z pixels'
	numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1])
	# assert numYpix > 10, 'too few y pixels'
	numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2])
	# assert numXpix > 10, 'too few x pixels'
	
	voxels = []
	ixs = []
	for i in range(n_sample):

		x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2)
		y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2)
		z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2)
		
		z_start = np.clip(z_center-numZpix/2, 0, img.shape[0])
		z_end = np.clip(z_center+numZpix/2, 0, img.shape[0])
		y_start = np.clip(y_center-numYpix/2, 0, img.shape[1])
		y_end = np.clip(y_center+numYpix/2, 0, img.shape[1])
		x_start = np.clip(x_center-numXpix/2, 0, img.shape[2])
		x_end = np.clip(x_center+numXpix/2, 0, img.shape[2])

		voxel = img[z_start:z_end,y_start:y_end,x_start:x_end]

		voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE))

		voxel_norm = np.clip(voxel_norm, -1000, 400)

		#determine index (if applicable)
		#if no match put -1.
		ix = -1
		for i,(coord,size) in enumerate(zip(voxel_coords,nodule_sizes)):
			if (x_start + size*1.5 <= coord[2] <= x_end - size*1.5) and (y_start + size*1.5 <= coord[1] <= y_end - size*1.5) and (z_start + size*1.5 <= coord[0] <= z_end - size*1.5):
				#we got one
				ix = dfsub.iloc[i]['ix']
				# pdb.set_trace()
				#target = size
		voxels.append(np.transpose(voxel_norm, (2,1,0)))
		ixs.append(ix)
		
	return np.stack(voxels),np.stack(ixs)
Example #45
0
    def initialize_variables(self):
        # initialization of V doesn't matter as we learn V on the first step conditioned on U
        self.V_mu = np.zeros((self.K, self.num_studies))
        self.V_mu = np.random.randn(self.K, self.num_studies)
        print('start')
        pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True)
        pca.fit(np.random.randn(self.num_studies, 200).T)
        self.V_mu = np.transpose(pca.components_.T)
        for kk in range(self.K):
            self.V_mu[kk, :] = (self.V_mu[kk, :] - np.mean(
                self.V_mu[kk, :])) / np.std(self.V_mu[kk, :])
        print('end')
        self.V_var = np.ones((self.K, self.num_studies)) * 1.0
        # Initialization of intercept doesn't matter as we learn intercept in the first step, conditioned on U
        self.intercept_mu = np.zeros(self.num_studies)
        self.intercept_var = np.ones(self.num_studies) * 1.0
        # Initialization of U DOES matter
        self.U_mu = np.random.randn(self.num_snps, self.K)
        self.U_var = np.ones((self.num_snps, self.K)) * 1.0
        self.S_U = np.ones((self.num_snps, self.K))

        # Smart init for Residual variance
        ld_score = generate_ld_scores(np.ones(
            (self.U_mu.shape[0], 1)), self.pairwise_ld_files,
                                      self.pairwise_ld_indices_files)
        # Perform regression analysis independently for each study
        resid_varz = []
        for study_num in range(self.num_studies):
            # Extract relevent data for this study
            study_chi_sq = np.load(self.chi_squared_files[study_num])
            study_sample_size = self.study_sample_sizes[study_num]
            # Simple error checking
            if len(study_chi_sq) != ld_score.shape[0]:
                print('assumption error')
            # Fit regression model
            #clf = Lasso(alpha=.1)
            reg = LinearRegression().fit(ld_score * study_sample_size,
                                         study_chi_sq - 1.0)
            residual_var = np.sum(
                np.square(
                    reg.predict(ld_score * study_sample_size) -
                    (study_chi_sq - 1.0))) / (len(study_chi_sq) - 2)
            resid_varz.append(residual_var)
        mean_resid_var = np.mean(resid_varz) / 10.0
        print(mean_resid_var)

        # Variance params
        # resid var
        #self.tau_alpha = np.ones(self.num_studies)
        #self.tau_beta = np.ones(self.num_studies)
        #self.tau_beta = np.asarray(resid_varz)
        self.tau_alpha = 1.0
        self.tau_beta = mean_resid_var
        # U var
        self.gamma_U_alpha = np.ones(self.K) * 1.0
        self.gamma_U_beta = np.ones(self.K) * 10.0
        # V var
        self.gamma_V_alpha = np.ones(self.K) * 1.0
        self.gamma_V_beta = np.ones(self.K) * 10.0

        # Sparsity parameters
        self.theta_U_a = np.ones(self.K)
        self.theta_U_b = np.ones(self.K)
Example #46
0
    def step_centertrack(self, results, time_lag):
        if len(results) == 0:
            self.tracks = []
            return []
        else:
            temp = []
            for det in results:
                # filter out classes not evaluated for tracking
                if det['detection_name'] not in TRACKING_NAMES:
                    continue

                det['ct'] = np.array(det['translation'][:2])
                det['tracking'] = np.array(det['velocity'][:2]) * -1 * time_lag
                det['label_preds'] = TRACKING_NAMES.index(det['detection_name'])
                temp.append(det)

            results = temp

        N = len(results)
        M = len(self.tracks)

        # N X 2
        if 'tracking' in results[0]:
            dets = np.array(
            [det['ct'] + det['tracking'].astype(np.float32)
             for det in results], np.float32)
        else:
            dets = np.array(
                [det['ct'] for det in results], np.float32)

        item_cat = np.array([item['label_preds'] for item in results], np.int32)  # N
        track_cat = np.array([track['label_preds'] for track in self.tracks], np.int32)  # M

        max_diff = np.array([self.CLS_VELOCITY_ERROR[box['detection_name']] for box in results], np.float32)
        tracks = np.array(
            [pre_det['ct'] for pre_det in self.tracks], np.float32)  # M x 2

        if len(tracks) > 0:  # NOT FIRST FRAME
            dist = (((tracks.reshape(1, -1, 2) -
                dets.reshape(-1, 1, 2)) ** 2).sum(axis=2))  # N x M
            dist = np.sqrt(dist)  # absolute distance in meter

            invalid = ((dist > max_diff.reshape(N, 1)) +
                (item_cat.reshape(N, 1) != track_cat.reshape(1, M))) > 0

            dist = dist + invalid * 1e18
            if self.hungarian:
                dist[dist > 1e18] = 1e18
                matched_indices = linear_sum_assignment(copy.deepcopy(dist))
                matched_indices = np.transpose(np.asarray(matched_indices))
            else:
                matched_indices = greedy_assignment(copy.deepcopy(dist))
        else:  # first few frame
            assert M == 0
            matched_indices = np.array([], np.int32).reshape(-1, 2)

        matched_indices = np.array(matched_indices).reshape(-1, 2)
        unmatched_dets = [d for d in range(dets.shape[0]) if not (d in matched_indices[:, 0])]

        unmatched_tracks = [d for d in range(tracks.shape[0]) if not (d in matched_indices[:, 1])]

        if self.hungarian:
            matches = []
            for m in matched_indices:
                if dist[m[0], m[1]] > 1e16:
                    unmatched_dets.append(m[0])
                else:
                    matches.append(m)
            matches = np.array(matches).reshape(-1, 2)
        else:
            matches = matched_indices

        ret = []
        for m in matches:
            track = results[m[0]]
            track['tracking_id'] = self.tracks[m[1]]['tracking_id']      
            track['age'] = 1
            track['active'] = self.tracks[m[1]]['active'] + 1
            ret.append(track)

        for i in unmatched_dets:
            track = results[i]
            self.id_count += 1
            track['tracking_id'] = self.id_count
            track['age'] = 1
            track['active'] = 1
            ret.append(track)

        # still store unmatched tracks if its age doesn't exceed max_age, however, we shouldn't output
        # the object in current frame
        for i in unmatched_tracks:
            track = self.tracks[i]
            if track['age'] < self.max_age:
                track['age'] += 1
                track['active'] = 0
                ct = track['ct']

                # movement in the last second
                if 'tracking' in track:
                    offset = track['tracking'] * -1  # move forward
                    track['ct'] = ct + offset
                ret.append(track)

        self.tracks = ret
        return ret
Example #47
0
def predict(encoder, decoder):
    test_df = pd.read_csv(args.test_data, names=['input'])

    test_df['target_pos'] = test_df['input'].apply(lambda x: x.split('EOS')[
        1].split('NOP')[0].strip().split(' ')[:MAX_TGT_LEN])
    test_df['target_rhyme'] = test_df['input'].apply(
        lambda x: x.split('NOP')[1].split('NOE')[0].strip())
    test_df['target_length'] = test_df['input'].apply(
        lambda x: x.split('NOE')[1].split('NOR')[0].strip())
    test_df['input'] = test_df['input'].apply(
        lambda x: x.split('EOS')[0].strip().split(' ')[1:MAX_INPUT_LEN + 1])
    test_df['input'] = test_df['input'].apply(
        lambda x: x + [EOS_TOKEN] +
        [PAD_TOKEN
         for i in range(MAX_INPUT_LEN - len(x) - 1)])  # add EOS at the end
    target_rhyme = np.copy(test_df.target_rhyme.values)
    target_length = np.copy(test_df.target_length.values.astype(int))
    target_pos = np.copy(test_df.target_pos.values)

    test_df['target_rhyme'] = test_df['target_rhyme'].apply(
        lambda x: [x, "NOE"])
    test_df['target_length'] = test_df['target_length'].apply(
        lambda x: [x, "NOR"])
    test_df['target_pos'] = test_df['target_pos'].apply(
        lambda x: x + ["POS_PAD"
                       for i in range(MAX_TGT_LEN - len(x))] + ["NOP"])
    test_df['input'] = test_df['input'] + test_df["target_pos"] + test_df[
        "target_rhyme"] + test_df["target_length"]
    test_df['input'] = test_df['input'].apply(
        lambda x: [word2idx_mapping.get(i, 0) for i in x])

    test_data = test_df['input'].values

    test_data = torch.LongTensor(test_data.tolist()).t().to(device)

    encoder = encoder.eval().to(device)
    decoder = decoder.eval().to(device)

    # predict
    batch_size = 1000
    result = [[] for i in range(MAX_TGT_LEN)]
    for b in tqdm(range(test_data.shape[1] // batch_size)):
        start = batch_size * b
        end = None if (b == (test_data.shape[1] // batch_size) -
                       1) else batch_size * (b + 1)

        encoder_output, encoder_hidden = encoder(test_data[:, start:end])
        decoder_hidden = encoder_hidden
        decoder_input = torch.LongTensor([[
            word2idx_mapping[SOS_TOKEN]
        ]]).repeat(1, encoder_output.shape[1]).to(device)

        for i in range(MAX_TGT_LEN):
            if args.attn:
                decoder_output, decoder_hidden, attn_weights = decoder(
                    decoder_input, decoder_hidden, encoder_output,
                    tmp_tgt_lengths)
                _, top1 = decoder_output.topk(1, dim=1)
                decoder_input = top1.squeeze(-1).unsqueeze(0)
            else:
                decoder_output, decoder_hidden = decoder(
                    decoder_input, decoder_hidden)
                _, top1 = decoder_output.topk(1, dim=2)
                decoder_input = top1.squeeze(-1)
                result[i].extend(top1.cpu().numpy()[0].tolist())
                #_, top1 = decoder_output.topk(70, dim=2)  # output top 70 for rhyme
                #decoder_input = top1[:,:,0].squeeze(-1)
                #result[i].extend(top1.cpu().numpy()[0].tolist())

    ipdb.set_trace()
    # output
    result = np.transpose(np.array(result), (1, 0, 2))
    res_len = np.zeros(result.shape[0])
    new_result = [[] for i in range(result.shape[0])]
    cnt = 0
    all_pos_cnt = 0
    correct_pos_cnt = 0
    with open(args.output_file, "w") as f:
        for i, res in enumerate(result):
            for j, w in enumerate(res):
                if (idx2word_mapping[w[0]] == EOS_TOKEN) or (j
                                                             == len(res) - 1):
                    res_len[i] = j

                    # pick correct rhyme
                    #for k, candidate in enumerate(last):
                    #    if j > 0 and rhyme_corpus.get(idx2word_mapping.get(candidate, 0), 0) == tgt_rhyme[i]:
                    #        new_result[i][j-1] = idx2word_mapping.get(candidate, 0)
                    #        cnt += 1
                    #        break
                    if j > 0 and rhyme_corpus.get(
                            idx2word_mapping.get(last[0], 0),
                            0) == target_rhyme[i]:
                        cnt += 1

                    # calculate POS accuracy
                    all_pos_cnt += len(target_pos[i])
                    if len(new_result[i]) > 0:
                        _, pos = zip(*jieba.posseg.cut(''.join(new_result[i])))
                        pos_len = min(len(pos), len(target_pos[i]))
                        #pos = np.array([pos2idx_mapping.get(i, 0) for i in pos])
                        correct_pos_cnt += sum(
                            np.array(pos[:pos_len]) == np.array(target_pos[i]
                                                                [:pos_len]))
                    else:
                        print("", file=f)
                        break

                    for k, c in enumerate(new_result[i]):
                        if k == len(new_result[i]) - 1:
                            print(c, file=f)
                        else:
                            print(c, end=' ', file=f)
                    break
                new_result[i].append(idx2word_mapping[w[0]])
                last = w  # save last timestep's output
    print('pos:', correct_pos_cnt / all_pos_cnt)
    print('rhyme:', cnt / result.shape[0])
    print("length: ", sum(res_len == target_length) / res_len.shape[0])

    output = subprocess.run([
        'python3', '/home/victai/SDML/HW3/trigram_model/trigram_evaluate.py',
        '--testing_data', args.output_file
    ],
                            stderr=subprocess.STDOUT)
Example #48
0
    def __init__(self, tflist_arr, x_arr):
        if x_arr is not None:
            ndim = x_arr.ndim
        else:
            ndim = 0

        if ndim == 0:
            self.rs = tflist_arr[0].rs
            self.in_eng = tflist_arr[0].in_eng
            self.eng = tflist_arr[0].eng
            self.dlnz = tflist_arr[0].dlnz
            self.x = x_arr
            self.spec_type = tflist_arr[0].spec_type
            self.tflist_arr = tflist_arr

            self._grid_vals = tflist_arr[0].grid_vals
            if tflist_arr[0].tftype == 'eng':
                # grid_vals should have indices corresponding to
                # (rs, in_eng, eng).
                grid_vals = np.transpose(grid_vals, (1, 0, 2))
                # grid_vals are now (rs, in_eng, eng).

            if self.rs[0] - self.rs[1] > 0:
                # data points have been stored in decreasing rs.
                self.rs = np.flipud(self.rs)
                self._grid_vals = np.flip(self._grid_vals, 1)

        elif ndim == 1:
            if not arrays_equal([tflist.rs for tflist in tflist_arr]):
                raise TypeError('All redshift bins must be identical.')
            if not arrays_equal([tflist.in_eng for tflist in tflist_arr]):
                raise TypeError('All in_eng bins must be identical.')
            if not arrays_equal([tflist.eng for tflist in tflist_arr]):
                raise TypeError('All eng bins must be identical.')
            if len(set([tflist.dlnz for tflist in tflist_arr])) != 1:
                raise TypeError('All dlnz steps must be identical.')
            if len(set([tflist.spec_type for tflist in tflist_arr])) != 1:
                raise TypeError('All spec_type must be identical.')
            if len(set([tflist.tftype for tflist in tflist_arr])) != 1:
                raise TypeError('All tftype must be the same.')

            self.rs = tflist_arr[0].rs
            self.in_eng = tflist_arr[0].in_eng
            self.eng = tflist_arr[0].eng
            self.dlnz = tflist_arr[0].dlnz
            self.x = x_arr
            self.spec_type = tflist_arr[0].spec_type
            self.tflist_arr = tflist_arr

            grid_vals = np.array(np.stack(
                [tflist.grid_vals for tflist in tflist_arr]),
                                 ndmin=4)
            if tflist_arr[0].tftype == 'eng':
                # grid_vals should have indices corresponding to
                # (xH, rs, in_eng, eng).
                grid_vals = np.transpose(grid_vals, (0, 2, 1, 3))

            # grid_vals are now (xH, rs, in_eng, eng).

            self._grid_vals = grid_vals

            if self.rs[0] - self.rs[1] > 0:
                # data points have been stored in decreasing rs.
                self.rs = np.flipud(self.rs)
                self._grid_vals = np.flip(self._grid_vals, 1)

            # Now, data is stored in *increasing* rs.

        elif ndim == 3:
            if not all(
                    arrays_equal([tflist.rs for tflist in tflist_xHe_arr])
                    for tflist_xHe_arr in tflist_arr):
                raise TypeError('All redshift bins must be identical.')
            if not all(
                    arrays_equal([tflist.in_eng for tflist in tflist_xHe_arr])
                    for tflist_xHe_arr in tflist_arr):
                raise TypeError('All in_eng bins must be identical.')
            if not all(
                    arrays_equal([tflist.eng for tflist in tflist_xHe_arr])
                    for tflist_xHe_arr in tflist_arr):
                raise TypeError('All eng bins must be identical.')
            if len(
                    set(tflist.dlnz for tflist_xHe_arr in tflist_arr
                        for tflist in tflist_xHe_arr)) != 1:
                raise TypeError('All dlnz must be identical.')
            if len(
                    set(tflist.spec_type for tflist_xHe_arr in tflist_arr
                        for tflist in tflist_xHe_arr)) != 1:
                raise TypeError('All spec_type must be identical.')
            if len(
                    set(tflist.tftype for tflist_xHe_arr in tflist_arr
                        for tflist in tflist_xHe_arr)) != 1:
                raise TypeError('All tftype must be identical.')

            self.rs = tflist_arr[0][0].rs
            self.in_eng = tflist_arr[0][0].in_eng
            self.eng = tflist_arr[0][0].eng
            self.dlnz = tflist_arr[0][0].dlnz
            self.x = x_arr
            self.spec_type = tflist_arr[0][0].spec_type
            self.tflist_arr = tflist_arr

            grid_vals = np.array(np.stack([
                np.stack([tflist.grid_vals for tflist in tflist_xHe_arr])
                for tflist_xHe_arr in tflist_arr
            ]),
                                 ndmin=5)

            if tflist_arr[0][0].tftype == 'eng':
                # grid_vals should have indices corresponding to
                # (xH, xHe, rs, in_eng, eng).
                grid_vals = np.transpose(grid_vals, (0, 1, 3, 2, 4))

            # grid_vals are now (xH, xHe, rs, in_eng, eng)

            self._grid_vals = grid_vals

            if self.rs[0] - self.rs[1] > 0:
                # data points have been stored in decreasing rs.
                self.rs = np.flipud(self.rs)
                self._grid_vals = np.flip(self._grid_vals, 2)

            # Now, data is stored in *increasing* rs.

        else:
            raise TypeError(
                'x_arr dimensions is anomalous (and not in the good QFT way).')

        def __iter__(self):
            return iter(self.tflist_arr)

        def __getitem__(self, key):
            return self.tflist_arr[key]

        def __setitem__(self, key, value):
            self.tflist_arr[key] = value
def data_loader(data_name):
    if data_name == "starplus":
        """
        starplus = sio.loadmat("data-starplus-04847-v7.mat")
        metadata = starplus['meta'][0,0]
        #meta.study gives the name of the fMRI study
        #meta.subject gives the identifier for the human subject
        #meta.ntrials gives the number of trials in this dataset
        #meta.nsnapshots gives the total number of images in the dataset
        #meta.nvoxels gives the number of voxels (3D pixels) in each image
        #meta.dimx gives the maximum x coordinate in the brain image. The minimum x coordinate is x=1. meta.dimy and meta.dimz give the same information for the y and z coordinates.
        #meta.colToCoord(v,:) gives the geometric coordinate (x,y,z) of the voxel corresponding to column v in the data
        #meta.coordToCol(x,y,z) gives the column index (within the data) of the voxel whose coordinate is (x,y,z)
        #meta.rois is a struct array defining a few dozen anatomically defined Regions Of Interest (ROIs) in the brain. Each element of the struct array defines on of the ROIs, and has three fields: "name" which gives the ROI name (e.g., 'LIFG'), "coords" which gives the xyz coordinates of each voxel in that ROI, and "columns" which gives the column index of each voxel in that ROI.
        #meta.colToROI{v} gives the ROI of the voxel corresponding to column v in the data.
        study      = metadata['study']
        subject    = metadata['subject']
        ntrials    = metadata['ntrials'][0][0]
        nsnapshots = metadata['nsnapshots'][0][0]
        dimx       = metadata['dimx'][0][0]
        colToCoord = metadata['colToCoord']
        coordToCol = metadata['coordToCol']
        rois       = metadata['rois']
        colToROI   = metadata['colToROI']
        info = starplus['info'][0]
        #info: This variable defines the experiment in terms of a sequence of 'trials'. 'info' is a 1x54 struct array, describing the 54 time intervals, or trials. Most of these time intervals correspond to trials during which the subject views a single picture and a single sentence, and presses a button to indicate whether the sentence correctly describes the picture. Other time intervals correspond to rest periods. The relevant fields of info are illustrated in the following example:
        #info(18) mint: 894 maxt: 948 cond: 2 firstStimulus: 'P' sentence: ''It is true that the star is below the plus.'' sentenceRel: 'below' sentenceSym1: 'star' sentenceSym2: 'plus' img: sap actionAnswer: 0 actionRT: 3613
        #info.mint gives the time of the first image in the interval (the minimum time)
        #info.maxt gives the time of the last image in the interval (the maximum time)
        #info.cond has possible values 0,1,2,3. Cond=0 indicates the data in this segment should be ignored. Cond=1 indicates the segment is a rest, or fixation interval. Cond=2 indicates the interval is a sentence/picture trial in which the sentence is not negated. Cond=3 indicates the interval is a sentence/picture trial in which the sentence is negated.
        #info.firstStimulus: is either 'P' or 'S' indicating whether this trail was obtained during the session is which Pictures were presented before sentences, or during the session in which Sentences were presented before pictures. The first 27 trials have firstStimulus='P', the remained have firstStimulus='S'. Note this value is present even for trials that are rest trials. You can pick out the trials for which sentences and pictures were presented by selecting just the trials trials with info.cond=2 or info.cond=3.
        #info.sentence gives the sentence presented during this trial. If none, the value is '' (the empty string). The fields info.sentenceSym1, info.sentenceSym2, and info.sentenceRel describe the two symbols mentioned in the sentence, and the relation between them.
        #info.img describes the image presented during this trial. For example, 'sap' means the image contained a 'star above plus'. Each image has two tokens, where one is above the other. The possible tokens are star (s), plus (p), and dollar (d).
        #info.actionAnswer: has values -1 or 0. A value of 0 indicates the subject is expected to press the answer button during this trial (either the 'yes' or 'no' button to indicate whether the sentence correctly describes the picture). A value of -1 indicates it is inappropriate for the subject to press the answer button during this trial (i.e., it is a rest, or fixation trial).
        #info.actionRT: gives the reaction time of the subject, measured as the time at which they pressed the answer button, minus the time at which the second stimulus was presented. Time is in milliseconds. If the subject did not press the button at all, the value is 0.
        data = starplus['data']
        #data: This variable contains the raw observed data. The fMRI data is a sequence of images collected over time, one image each 500 msec. The data structure 'data' is a [54x1] cell array, with one cell per 'trial' in the experiment. Each element in this cell array is an NxV array of observed fMRI activations. The element data{x}(t,v) gives the fMRI observation at voxel v, at time t within trial x. Here t is the within-trial time, ranging from 1 to info(x).len. The full image at time t within trial x is given by data{x}(t,:).
        #Note the absolute time for the first image within trial x is given by info(x).mint.
        MIN_LOSS_CHANGE=0.0001
        maxFeatures = max([data[i][0].flatten().shape[0] for i in range(data.shape[0])])

        # Inputs
        X = np.zeros((ntrials, maxFeatures + 1))
        for i in range(data.shape[0]):
            f = data[i][0].flatten()
            X[i, :f.shape[0]] = f
            X[i, f.shape[0]] = 1  # Bias

        # Outputs (+1 = Picture, -1 = Sentence)
        Y = np.ones(ntrials)
        Y[np.array([info[i]['firstStimulus'][0] != 'P' for i in range(ntrials)])] = -1

        # Randomly permute the data
        np.random.seed(1)  # Seed the random number generator to preserve the dev/test split
        permutation = np.random.permutation(ntrials)
        permutation = np.random.permutation(X.shape[0])
        X = X[permutation,]
        Y = Y[permutation,]
        X_train = X[:40]
        X_test = X[40:]
        Y_train = Y[:40]
        Y_test = Y[40:]
        np.savez('Starplus.npz', X_train=X_train, X_test=X_test, Y_train=Y_train,Y_test=Y_test)
        """
        data = np.load('Starplus.npz')
        X_train = data['X_train'] / 10.0
        Y_train = data['Y_train']
        X_test = data['X_test'] / 10.0
        Y_test = data['Y_test']
        return np.transpose(X_train), Y_train.reshape(-1, 1), np.transpose(X_test), Y_test.reshape(-1, 1)

    else:
        X_train = [[1.0, 1.0, 1.0], [2.0, 2.0, 1.0], [4.0, 4.0, 1.0], [5.0, 5.0, 1.0], [1.0, 3.0, 1.0], [2.0, 4.0, 1.0], [4.0, 6.0, 1.0], [5.0, 7.0, 1.0]]
        X_test = [[3.0, 3.0, 1.0], [3.0, 5.0, 1.0]]
        Y_train = [[1.0], [1.0], [1.0], [1.0], [-1.0], [-1.0], [-1.0], [-1.0]]
        Y_test = [[1.0], [-1.0]]
        return np.transpose(np.asarray(X_train)), np.asarray(Y_train), np.transpose(np.asarray(X_test)), np.asarray(Y_test)
    def setupParams(self, root, f):
        args = parse_args(root, f + '.avi')
        params = args['params']
        # 50Hz gps data
        gps_name = args['gps_mark1']
        print gps_name
        gps_reader = GPSReader(gps_name)
        GPSData = gps_reader.getNumericData()
        if GPSData.shape[0] == 0:
            print 'empty gps1 log. skipping...'
            return False
        self.imuTransforms = IMUTransforms(GPSData)
        self.GPSTime = utc_from_gps_log_all(GPSData)
        # Video-synced gps data
        gps_name2 = args['gps_mark2']
        gps_reader2 = GPSReader(gps_name2)
        GPSData2 = gps_reader2.getNumericData()
        if GPSData2.shape[0] == 0:
            print 'empty gps2 log. skipping...'
            return False
        self.imuTransforms2 = IMUTransforms(GPSData2)

        # grab the initial time off the gps log and compute start and end times
        self.start_time = self.GPSTime[0] + 1 * 1e6
        self.end_time = self.GPSTime[0] + 600 * 1e6
        if self.end_time > self.GPSTime[-1]:
            self.end_time = self.GPSTime[-1]
        self.step_time = 0.5
        self.scan_window = 0.1
        self.lidar_loader = LDRLoader(args['frames'])
        self.T_from_l_to_i = params['lidar']['T_from_l_to_i']
        self.lidar_height = params['lidar']['height']

        print 'gps: ' + gps_name
        map_name = gps_name[0:-self.name_offset] + '.map'
        print 'map: ' + map_name
        self.map_outname = os.path.join(
            self.targetfolder,
            (gps_name[0:-self.name_offset] + '_lidarmap.pickle'))
        self.lane_outname = os.path.join(
            self.targetfolder,
            (gps_name[0:-self.name_offset] + '_interp_lanes.pickle'))
        print 'out: ' + self.lane_outname
        if os.path.isfile(self.lane_outname):
            print self.lane_outname + ' already exists, skipping...'
            return False
        total_num_frames = GPSData.shape[0]

        velocities = GPSData[:, 4:7]
        velocities[:, [0, 1]] = velocities[:, [1, 0]]
        vel_start = ENU2IMUQ50(np.transpose(velocities), GPSData[0, :])
        # sideways vector wrt starting imu frame
        sideways_start = np.cross(vel_start.transpose(),
                                  self.imuTransforms[:, 0:3, 2],
                                  axisa=1,
                                  axisb=1,
                                  axisc=1)
        self.sideways = sideways_start / (np.sqrt(
            (sideways_start**2).sum(1))[..., np.newaxis]
                                          )  # normalize to unit vector
        # initialize empty data holders
        self.left_data = []
        self.right_data = []
        self.left_time = []
        self.right_time = []
        self.all_data = dict()
        self.all_time = dict()
        return True
Example #51
0
    def get_path(self, req):
        start = req.start
        end = req.goal
        tolerance = req.tolerance

        resp = self.map_client()
        map_ = MyMap()
        map_.from_msg(resp.map)

        my_down = MyMap(grid=map_.downscale(map_.grid, 20), resolution=1 / 20)
        explored_map = my_down.binary
        cv2.imshow("Map BW", map_.to_img(True))

        voronoi_graph = generate_voronoi(explored_map)
        cv2.imshow("Voronoi", voronoi_graph)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

        free_points = np.transpose(np.where(voronoi_graph == 0))  # Points in white in the voronoi
        min_start = 100
        min_end = 100
        
        # closest_start = min(free_points, key=lambda p: abs(p[0] - start.pose.position.x) + abs(p[1] - start.pose.position.y))
        # closest_end = min(free_points, key=lambda p: abs(p[0] - end.pose.position.x) + abs(p[1] - end.pose.position.y))
        # Get the closest point in the graph for start and end
        for point in free_points:
            tmp_start = (abs(point[0] - start.pose.position.x*10) + abs(point[1] - start.pose.position.y*10))
            tmp_end = (abs(point[0] - end.pose.position.x*10) + abs(point[1] - end.pose.position.y*10))
            if tmp_start < min_start:
                min_start = tmp_start
                start_point = point
            if tmp_end < min_end:
                min_end = tmp_end
                end_point = point
        path_list = best_first_search(voronoi_graph,
                                      (start_point[0], start_point[1]),
                                      (end_point[0], end_point[1]), 0)
        path_list = list(map(lambda i: (float(i[0])/10, float(i[1])/10), path_list))
        path_list.append((end.pose.position.x, end.pose.position.y))
        path_list.insert(0, (start.pose.position.x, start.pose.position.y))
        
        self.is_path = True
        self.path = Path()
        self.path.header.frame_id = "map"
        filtered_path = []
        filtered_path = np.copy(path_list[::2])
        if path_list[-1][0] != filtered_path[-1][0] or path_list[-1][1] != filtered_path[-1][1]: 
            filtered_path = np.vstack([filtered_path, path_list[-1]])
        
        filtered_path = np.reshape(filtered_path, (-1,2))
        no_obstacles = True
        n = 1
        while no_obstacles and n<20:
            try_path = self.interpolate_path(filtered_path,0.2*n)
            n += 1
            no_obstacles = self.check_obstacles(try_path)
            
            if no_obstacles and len(try_path) >= 2: 
                smooth_path = np.copy(try_path)
                print('smoothened trajectory is valid')
            elif len(try_path) < 2:
                break
        if path_list[-1][0] != smooth_path[-1][0] or path_list[-1][1] != smooth_path[-1][1]: 
            smooth_path = np.vstack([smooth_path, path_list[-1]])

        for p in smooth_path:
            point = PoseStamped()
            point.pose.position.x = p[0]
            point.pose.position.y = p[1]
            self.path.poses.append(point)
        return self.path
Example #52
0
#!/usr/bin/env python2.7

"""
Extract the Cls from the chain CHAIN_FILE into a numpy file named CHAIN_FILE.cls.npy

Usage:
    extract_cls.py CHAIN_FILE
"""

import sys
from cosmoslik_plugins.samplers.metropolis_hastings import load_chain
import numpy as np

burnin = 1000
thin = 10

chain = load_chain(sys.argv[1]).burnin(burnin).join().thin(thin)
# chain = load_chain('data1/run1_lcdm.chain').burnin(burnin).join().thin(thin)
cls = np.array([x['cl_TT'] for x in chain['cl_TT']])
parnames = np.array(chain.keys())
parvls = np.transpose(np.array([chain[pn] for pn in parnames if pn!="cl_TT" ]))
parnames_iterable = np.array([pn for pn in parnames if pn!="cl_TT" ])


np.save(sys.argv[1]+'.cls',cls)
np.save(sys.argv[1]+'.parnames',parnames_iterable)
np.save(sys.argv[1]+'.parvls',parvls)
Example #53
0
# Plot data
file = filename[0]
with open(file) as f:
    line = f.readline()
    line = line.split(',')
    name = []
    for i in range(len(line)):
        name.append(line[i])

fig1 = plt.figure(figsize=(15,9))
ax1 = fig1.add_subplot(211)
ax2 = fig1.add_subplot(212)
for i,file in enumerate(filename):
    data = np.loadtxt(file,delimiter=',',comments='#',skiprows=1)
    data = np.transpose(data)
    x = data[0]
    for k in range(len(name)):
        if k in [4,6,8,9]:
            ax1.plot(x,data[k],label=name[k],ls='-')
    ax2.plot(x,data[1],label=name[1],ls='-',c='k')


# Fig
ax1.legend(loc=0)
ax2.legend(loc=0)
x_ticks = np.linspace(x[0], x[-1], 5)
ax1.set_xticks(x_ticks)
# ax1.set_ylim(-20, 60)
ax1.set_xlabel(r'$x$ (m)')
ax1.set_ylabel(r'$\dot{\omega}$ (kg/m3 s)')
Example #54
0
def make_outer_surf(
    orig_pial: Union[str, Path],
    image: Union[str, Path],
    output_fpath: Union[str, Path],
    outer_surface_sphere: float = 15,
):
    """Create outer surface of a pial volume.

    Make outer surface based on a pial volume and radius,
    write to surface in outfile.

    Parameters
    ----------
    orig_pial : str | pathlib.Path
        Pial surface (e.g. lh.pial)
    image : str | pathlib.Path
        Filled lh or rh pial image (e.g. lh.pial.filled.mgz)
    output_fpath : str | pathlib.Path
        surface file to write data to
    outer_surface_sphere : float | None
        radius for smoothing in mm (default=15). diameter of the sphere used
        by make_outer_surface to close the sulci using morphological operations.
        Ignored currently. Corresponds to ``se=strel('sphere',se_diameter);``
        in Matlab. See [1].

    References
    ----------
    .. [1] See FieldTrip Toolbox ``make_outer_surface`` function inside
    ``prepare_mesh_cortexhull.m`` file.

    .. [2] https://github.com/aestrivex/ielu
    """
    from scipy.signal import convolve
    from scipy.ndimage.morphology import grey_closing, generate_binary_structure
    from mne import write_surface
    from mcubes import marching_cubes

    # radius information is currently ignored
    # it is a little tougher to deal with the morphology in python

    # load original pial surface to get the volume information
    pial_surf = nb.freesurfer.read_geometry(orig_pial, read_metadata=True)
    volume_info = pial_surf[2]

    # load filled pial image
    fill = nb.load(image)
    filld = fill.get_data()
    filld[filld == 1] = 255

    # apply a very soft Gaussian filter with sigma = 1mm to
    # facilitate the closing
    gaussian = np.ones((2, 2)) * 0.25

    # initialize image cube array
    image_f = np.zeros((256, 256, 256))

    # initialize a thresholded image
    image2 = np.zeros((256, 256, 256))

    # for each slice, convolve the Gaussian filter on the
    # filled image
    for slice in range(256):
        temp = filld[:, :, slice]
        image_f[:, :, slice] = convolve(temp, gaussian, "same")

    # thresholded image based on value of 25
    image2[np.where(image_f <= 25)] = 0
    image2[np.where(image_f > 25)] = 255

    strel15 = generate_binary_structure(3, 1)

    # run multi-dimensional grayscale closing of the image
    BW2 = grey_closing(image2, structure=strel15)
    thresh = np.max(BW2) / 2
    BW2[np.where(BW2 <= thresh)] = 0
    BW2[np.where(BW2 > thresh)] = 255

    # apply marching cubes algorithm to get
    # vertices and faces
    v, f = marching_cubes(BW2, 100)

    # in order to cope with the different orientation
    v2 = np.transpose(
        np.vstack((
            128 - v[:, 0],
            v[:, 2] - 128,
            128 - v[:, 1],
        )))

    write_surface(output_fpath, v2, f, volume_info=volume_info)
Example #55
0
            # incrementar fila
            print('VERDADERO')
            truth_table[filas, columna] = 'VERDADERO'
            arreglo.append('VERDADERO')
            cont=cont-1
    bandera = False


x_out = []
for col in range(number_of_variables):
    print('+++++++++++++++')
    print('columna', col)
    for i in range(0,number_of_states,(2**(col+1))):
        print(i,':')
        imprimir_verdadero_falso(2**col, col, x_out)


print(x_out)

y  =  np.reshape(x_out,(number_of_variables,number_of_states))  # changes from array to matrix format
y2 = np.transpose(y)
#print('************\n',y2)

y3 = np.fliplr(y2)    #changes order of columns
print('************\n',y3)

np.savetxt('Tabla_Verdad.csv',y3, fmt = "%s", delimiter=',')
#np.savetxt('Tabla_Verdad.csv',y3, object, delimiter=',',header='v1,v2,v3,v4,v5,v6,v7,v8')


Example #56
0
    def __getitem__(self, index):

        #---------
        #  Image
        #---------

        img_path = self.img_files[index % len(self.img_files)].rstrip()
        img = np.array(Image.open(img_path))

        # Handles images with less than three channels
        while len(img.shape) != 3:
            index += 1
            img_path = self.img_files[index % len(self.img_files)].rstrip()
            img = np.array(Image.open(img_path))

        h, w, _ = img.shape
        dim_diff = np.abs(h - w)
        # Upper (left) and lower (right) padding
        pad1, pad2 = dim_diff // 2, dim_diff - dim_diff // 2
        # Determine padding
        pad = ((pad1, pad2), (0, 0), (0, 0)) if h <= w else ((0, 0), (pad1, pad2), (0, 0))
        # Add padding
        input_img = np.pad(img, pad, 'constant', constant_values=128) / 255.
        padded_h, padded_w, _ = input_img.shape
        # Resize and normalize
        input_img = resize(input_img, (*self.img_shape, 3), mode='reflect')
        # Channels-first
        input_img = np.transpose(input_img, (2, 0, 1))
        # As pytorch tensor
        input_img = torch.from_numpy(input_img).float()

        #---------
        #  Label
        #---------

        label_path = self.label_files[index % len(self.img_files)].rstrip()

        labels = None
        if os.path.exists(label_path):
            labels = np.loadtxt(label_path).reshape(-1, 5)
            # Extract coordinates for unpadded + unscaled image
            x1 = w * (labels[:, 1] - labels[:, 3]/2)
            y1 = h * (labels[:, 2] - labels[:, 4]/2)
            x2 = w * (labels[:, 1] + labels[:, 3]/2)
            y2 = h * (labels[:, 2] + labels[:, 4]/2)
            # Adjust for added padding
            x1 += pad[1][0]
            y1 += pad[0][0]
            x2 += pad[1][0]
            y2 += pad[0][0]
            # Calculate ratios from coordinates
            labels[:, 1] = ((x1 + x2) / 2) / padded_w
            labels[:, 2] = ((y1 + y2) / 2) / padded_h
            labels[:, 3] *= w / padded_w
            labels[:, 4] *= h / padded_h
        # Fill matrix
        filled_labels = np.zeros((self.max_objects, 5))
        if labels is not None:
            filled_labels[range(len(labels))[:self.max_objects]] = labels[:self.max_objects]
        filled_labels = torch.from_numpy(filled_labels)

        return img_path, input_img, filled_labels
Example #57
0
def rescaler(Xs, Ys, rescale_factor):
    Xs = np.asarray([tf.rescale(np.transpose(Xs[i], (2, 1, 0)), scale=rescale_factor) for i in range(0, Xs.shape[0])])
    Xs = Xs.transpose((0, 3, 2, 1))
    Ys = np.asarray([tf.rescale(np.transpose(Ys[i], (2, 1, 0)), scale=rescale_factor) for i in range(0, Ys.shape[0])])
    Ys = Ys.transpose((0, 3, 2, 1))
    return Ys, Xs
Example #58
0
                txt = plt.suptitle("-", fontsize=36, fontweight='bold')
                pred_text = str(all_labels[pred[0]])
                true_text = str(all_labels[true[0]])
                txt.set_text('PREDICTION: %s\nTRUTH: %s' %
                             (pred_text, true_text))

                plt.subplots_adjust(top=0.7)

                f_glimpse_images = np.reshape(
                    glimpse_images_fetched,
                    (params.num_glimpses, params.batch_size,
                     params.num_points_per_glimpse, params.loc_dim))

                # display the entire image along with the sampled locations
                sampled_locs_fetched = np.transpose(sampled_locs_fetched,
                                                    (1, 0, 2))
                plotWholePC(images[0, :], sampled_locs_fetched)

                # Display the glimpses
                plotGlimpses(f_glimpse_images, glimpse_axes)

                if display:
                    plt.ion()
                    plt.draw()
                    time.sleep(0.05)
                    plt.pause(pause_time)
                elif save:
                    plt.savefig('%s_%s_%s.png' %
                                (str(i), pred_text, true_text))
                    plt.close()
                else:
size_w    = integer, number of grid points in cake state space
w_grid    = vector, size_w x 1 vector of cake grid points
------------------------------------------------------------------------
'''

lb = 10
ub = 15

size_c = 50
size_z = 50

c_grid = np.linspace(lb, ub, size_c)

z_grid, pi = ar1.addacooper(size_z, mu, rho, sigma_z)
z_grid = np.exp(z_grid)
prob_m = np.transpose(pi)
'''
------------------------------------------------------------------------
Create grid of current utility values
------------------------------------------------------------------------
C        = matrix, current consumption (c=w-w')
U        = matrix, current period utility value for all possible
           choices of w and w' (rows are w, columns w')
------------------------------------------------------------------------
'''
C = np.zeros((size_c, size_c, size_c))
for i in range(size_c):
    for j in range(size_c):
        for k in range(size_c):
            C[i, j, k] = z_grid[k] * c_grid[i]**alpha + (
                1 - delta) * c_grid[i] - c_grid[j]
Example #60
0
    sys.stdout.write("\r{0}".format(index))
    sys.stdout.flush()

    x = X1_full[index]
    same_as_untreated = np.all(X0 == x,
                               axis=1)  # True if untreated is same as treated

    if any(same_as_untreated
           ):  # if same as treated, assign uniform weights to these untreated
        untreated_id = [i for i, x in enumerate(same_as_untreated) if x]
        all_w[index, untreated_id] = 1 / len(untreated_id)
    else:
        in_hull_flag = in_hull(x, X0)
        if in_hull_flag:
            X0_tilde, antiranks = incremental_pure_synth(X1_full[index], X0)
            w = pensynth_weights(np.transpose(X0_tilde),
                                 X1_full[index],
                                 pen=0.000001)
            all_w[index, antiranks] = np.transpose(w)
        else:
            w = pensynth_weights(np.transpose(X0), X1_full[index], pen=0)
            all_w[index, ] = np.transpose(w)

print(
    f"Temps d'exécution total : {(time.time() - start_time):.2f} secondes ---")

########## Save weights ##########
np.savetxt('/Users/jeremylhour/Documents/code/puresynth_solution.csv',
           all_w,
           delimiter=',')