Beispiel #1
0
def read_data(is_in_data_file):

    # READ DATE IF IS AVAILABLE
    if (is_in_data_file):
        with open('data/data', 'rb') as f:
            data = pickle.load(f)
            return data

    power_data = sp.delete(sp.genfromtxt("data/Power_history.csv", delimiter=","), 0, 1).flatten()

    weather_data = sp.stack([x.flatten() for x in sp.delete([sp.genfromtxt("data/" + filename, delimiter=",") for filename in filenames], [0, 1], 2)])
    weather_data = sp.delete(weather_data, sp.s_[:18], 1)

    # PREPROCESSING
    # REDUCE BROKEN DATA
    weather_data = sp.stack(x[~sp.isnan(x)] for x in weather_data)
    weather_data = sp.stack(x[~sp.isnan(power_data)] for x in weather_data)
    power_data = power_data[~sp.isnan(power_data)]
    data = sp.vstack([weather_data, power_data])
    data = data.transpose()
    # SELECT FILES TO BE INCLUDED IN COMPUTATION, POWER DATA ARE ALWAYS AT DATA[-1] !!!
    data = preproces(data, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

    # WRITING OUTPUT DATA TO FILE 'DATA'
    with open('data/data', 'wb') as f:
        pickle.dump(data, f)
    return data
Beispiel #2
0
def get_HRRR_data(filename):
    grbs = pygrib.open(filename)

    msgs = [str(grb) for grb in grbs]

    string = 'Geopotential Height:gpm'
    temp = [
        msg for msg in msgs
        if msg.find(string) > -1 and msg.find('isobaricInhPa') > -1
    ]
    pressure_levels_Pa = s.array([int(s.split(' ')[3]) for s in temp])

    geo_pot_height_grbs = grbs.select(name = 'Geopotential Height', \
                                      typeOfLevel='isobaricInhPa', level=lambda l: l > 0)
    temperature_grbs = grbs.select(name = 'Temperature', \
                                   typeOfLevel='isobaricInhPa', level=lambda l: l > 0)
    rh_grbs = grbs.select(name = 'Relative humidity', \
                          typeOfLevel='isobaricInhPa', level=lambda l: l > 0)

    lat, lon = geo_pot_height_grbs[0].latlons()

    geo_pot_height = s.stack([grb.values for grb in geo_pot_height_grbs])
    temperature = s.stack([grb.values for grb in temperature_grbs])
    rh = s.stack([grb.values for grb in rh_grbs])

    return lat, lon, geo_pot_height, temperature, rh, pressure_levels_Pa
Beispiel #3
0
def unpack_segment(Seg, num_lags=200, intercept=True):
    """ """

    lags = sp.arange(-num_lags / 2, num_lags / 2, 1, dtype='int32')

    Y = sp.stack([asig.magnitude.flatten() for asig in Seg.analogsignals],
                 axis=1)
    t_start = Seg.analogsignals[0].t_start.rescale('ms')
    t_stop = Seg.analogsignals[0].t_stop.rescale('ms')
    dt = Seg.analogsignals[0].sampling_period.rescale('ms')

    X = []
    for event in Seg.events:
        st = neo.core.SpikeTrain(event.times, t_stop=t_stop)
        bst = ele.conversion.BinnedSpikeTrain(st,
                                              binsize=dt,
                                              t_start=t_start,
                                              t_stop=t_stop)
        reg = bst.to_array().flatten()
        for lag in lags:
            X.append(sp.roll(reg, lag))

    X = sp.stack(X, axis=1)

    if intercept:
        X = sp.concatenate([sp.ones((X.shape[0], 1)), X], 1)

    t_lags = lags * dt.magnitude
    return Y, X, t_lags
Beispiel #4
0
def gen_sample(f, vnoise, xnoise):
    true_vx = f(t)
    true_x = cumtrapz(true_vx, t)
    true_x = sp.hstack([[0], true_x])

    noisy_vx = f(t + sp.random.randn(*t.shape) *
                 xnoise) + sp.random.randn(*t.shape) * vnoise
    noisy_x = true_x + sp.random.randn(*t.shape) * xnoise
    noisy_x = noisy_x

    return sp.stack([true_x, true_vx]).T, sp.stack([noisy_x, noisy_vx]).T
def gen_sample(v, vnoise_sigma, xnoise_mu1,xnoise_mu2, xnoise_sigma1,xnoise_sigma2):
    true_vx = v*sp.ones_like(t) #Velocity is taken as constant
    #Trapezoidal rule integration of velocity into position
    true_x  = cumtrapz(true_vx,t,initial=0) 
    
    #Velocity only has Gaussian noise (this might have to be changed)
    noisy_vx = true_vx+sp.random.randn(*t.shape)*vnoise_sigma
    
    #Position has bimodal noise
    noise_dist = bimodal_gaussian(xnoise_mu1,xnoise_mu2,xnoise_sigma1,xnoise_sigma2,-10,10,150)
    noisy_x  = true_x+noise_dist.sample(*t.shape)
    
    return sp.stack([true_x,true_vx]).T, sp.stack([noisy_x,noisy_vx]).T
Beispiel #6
0
def feros_velocity_correction(spec, create_fits=False, rv=False, out=''):
    """Radial velocity correction for a FEROS spectrum.

    Parameters
    ----------
    spec : file
        Fits file with FEROS spectra reduced with CERES.

    create_fits : bool, optional
        True to save a fits file with the result.

    Returns
    -------
    wavelength : array_like
        An array with the rest frame wavelengths.

    flux : array_like
        An array with the corresponding fluxes.

    """
    # Read fits file
    hdul = fits.open(spec)
    # Extract RV
    if not rv:
        rv = hdul[0].header['RV'] * u.km / u.s
        rv = rv.to(u.m / u.s)
    # Create gamma
    beta = rv / const.c
    gamma = 1 + beta.value
    # Extract wavelength per order
    wave = hdul[0].data[0, :, :]
    # Extract flux per order
    flux = hdul[0].data[9, :, :]
    orders = wave.shape[0]
    wave_rest = copy.deepcopy(wave)
    # Move spectra to rest frame
    for o in range(orders):
        wave_rest[o, :] /= gamma
    if create_fits:
        # Create new fits file
        if not out:
            out = spec.split('.fits')[0]
        else:
            date = hdul[0].header['HIERARCH SHUTTER START DATE'].split('-')
            ut = hdul[0].header['HIERARCH SHUTTER START UT'].split(':')
            out += hdul[0].header['HIERARCH TARGET NAME'] + '_'
            for d in date:
                out += d
            out += '_UT'
            for u in ut:
                out += u
        out += '_rest_frame.fits'
        hdu = fits.PrimaryHDU(sp.stack((wave_rest, flux)))
        try:
            hdu.writeto(out)
        except OSError:
            os.remove(out)
            hdu.writeto(out)
    hdul.close()
    return wave_rest, flux
Beispiel #7
0
def events2span(Data, entry_event, exit_event):
    entry_event = "TRIAL_ENTRY_EVENT"
    exit_event = "ITI_STATE"
    data_entry = Data.groupby("name").get_group(entry_event)
    data_exit = Data.groupby("name").get_group(exit_event)

    if data_entry.shape[0] == data_exit.shape[0]:
        # easy peasy
        Span = pd.DataFrame(sp.stack([data_entry['t'].values,data_exit['t'].values],axis=1),columns=['t_on','t_off'])
        Span['dt'] = Span['t_off'] - Span['t_on']
        return Span
    
    if data_entry.shape[0] != data_exit.shape[0]:
        print("problems occur: unequal number of entry and exits")
        ts = []
        for tup in data_entry.itertuples():
            t_on = tup.t
            t_max = data_exit.iloc[-1]['t']
            try:
                t_off = bhv.time_slice(data_exit, t_on, t_max, 't').iloc[0]['t']
                ts.append((t_on,t_off))
            except IndexError:
                # thrown when last is on
                pass

        Span = pd.DataFrame(ts,columns=['t_on','t_off'])
        Span['dt'] = Span['t_off'] - Span['t_on']
        return Span
def exportCV(x, y):
    global ncv
    curr = dataCube[y, x, :]
    cv = sp.stack((E, curr), axis=-1)
    path = filePath + '_' + str(ncv) + '_x' + str(x) + '_y' + str(y) + '.txt'
    sp.savetxt(path, cv, delimiter='\t')
    ncv += 1
Beispiel #9
0
def get_prf_data(pixel_values,
                 pixel_stddev,
                 pixel_offsets,
                 error_threshold=0.1):
    """
    Return the PRF measurements for (a subset of) the image.

    Args:
        pixel_values(2-D float array):    The calibrated pixel responses from
            the image to include in the plot.

        pixel_stddev(2-D float array):    The estimated standard deviation of
            `pixel_values`.

        pixel_offsets:    The slice of the return value of find_pixel_offsets()
            corresponding to `pixel_values`.

    Returns:
        (2-D float array, 2-D float array, 2-D float array, 2-D float array):

            * The x-offsets of the points at which PRF measurements are
              available.

            * The y-offsets of the points at which PRF measurements are
              available.

            * The measured normalized PRF at the available offsets

            * estimated errors of the PRF measurements.
    """

    prf_measurements = (
        (
            pixel_values
            -
            pixel_offsets['zero_point']
        )
        /
        pixel_offsets['norm']
    )

    prf_errors = (
        pixel_stddev
        /
        pixel_offsets['norm']
    )

    #False positive
    #pylint: disable=assignment-from-no-return
    include = scipy.logical_and(scipy.isfinite(prf_measurements),
                                scipy.isfinite(prf_errors))
    include = scipy.logical_and(include, prf_errors < error_threshold)
    #pylint: enable=assignment-from-no-return
    return scipy.stack((
        pixel_offsets['x_off'][include],
        pixel_offsets['y_off'][include],
        prf_measurements[include],
        prf_errors[include]
    ))
def exportCV(x,y):
        global ncv
        curr = dataCube[y,x,trim1:Nt-trim2]/Asc - y0
        darkCurr = darkCube[y,x,:]
        photoCurr = photoCube[y,x,:]
        cv = sp.stack((E[trim1:Nt-trim2],curr,darkCurr,photoCurr),axis=-1)
        path = cvsPath + baseName + '_' + str(ncv) + '_x' + str(x) + '_y' + str(y) + '.txt'
        sp.savetxt(path,cv,delimiter='\t')
        ncv += 1
Beispiel #11
0
def full_image(i=None):
    global _full
    if _full is None:
        path = Path('cache/full.npy')
        if not path.exists():
            ims = [_full_image(i) for i in range(1, COUNTS['full'] + 1)]
            sp.save(path, sp.stack(ims))
        _full = sp.load(path)
    ims = _full[i - 1] if i is not None else _full
    return ims
def generate_index_map(nonzero_locs, shape):
    r"""
    Determines the i,j,k indicies of the flattened array
    """
    #
    logger.info('creating index map of non-zero values...')
    x_c = sp.unravel_index(nonzero_locs, shape)[0].astype(sp.int16)
    y_c = sp.unravel_index(nonzero_locs, shape)[1].astype(sp.int16)
    z_c = sp.unravel_index(nonzero_locs, shape)[2].astype(sp.int16)
    index_map = sp.stack((x_c, y_c, z_c), axis=1)
    #
    return index_map
def generate_index_map(nonzero_locs, shape):
    r"""
    Determines the i,j,k indicies of the flattened array
    """
    #
    logger.info('creating index map of non-zero values...')
    x_c = sp.unravel_index(nonzero_locs, shape)[0].astype(sp.int16)
    y_c = sp.unravel_index(nonzero_locs, shape)[1].astype(sp.int16)
    z_c = sp.unravel_index(nonzero_locs, shape)[2].astype(sp.int16)
    index_map = sp.stack((x_c, y_c, z_c), axis=1)
    #
    return index_map
Beispiel #14
0
def quadrant_image(i=None):
    global _quadrant
    if _quadrant is None:
        path = Path('cache/quadrant.npy')
        if not path.exists():
            ims = [
                _quadrant_image(i) for i in range(1, COUNTS['quadrant'] + 1)
            ]
            sp.save(path, sp.stack(ims))
        _quadrant = sp.load(path)
    ims = _quadrant[i - 1] if i is not None else _quadrant
    return ims
Beispiel #15
0
def bumpmaps(category, number=None):
    """Uses a NN to generate a perfect image, and caches the result so it'll be fast to load next time"""

    if number is None:
        return sp.stack([
            bumpmaps(category, n)
            for n in tqdm(range(1, tools.COUNTS[category] + 1))
        ])

    path = Path(f'cache/nn/output/{category}/{number}.npy')
    if not path.exists():
        path.parent.mkdir(exist_ok=True, parents=True)
        losses, model = load(*MODEL)
        bumps = evaluate(category, number, model)[1]
        sp.save(path, bumps)
    return sp.load(path)
Beispiel #16
0
    def test_simple_mv(self):
        '''
        Test a simple multivariate example (to the best of our abilities given we don't have a ground truth for this
        example).
        :return:
        '''

        # load the second OX reference data set.
        with open(os.path.join(self.datapath, "2/params.json")) as f:
            params = json.loads(f.read())
        yy = sp.array(parse_ox_csv(
            os.path.join(self.datapath, "2/raw_data.csv")),
                      dtype=np.float64).transpose()
        y = sp.reshape(yy, (101, 25, 1))[1:]

        # now we'll make a few copies of the reference data and stack them along the third axis.
        y = sp.squeeze(sp.stack((y, y, y, y, y), axis=2))

        rows = []
        for i, group in enumerate(y):
            for entry in group:
                rows.append([i, "A"] + entry.tolist())
        panel_series = PanelSeries.from_list(rows)

        # run the filtering
        Z = sp.matrix(params["translation_matrix"])
        F = sp.matrix(params["transition_matrix"])
        a0 = sp.matrix(params["a0"]).reshape(-1, 1)
        Q0 = sp.matrix(params["Q0"])
        Q = sp.matrix(params["Q"])
        sigma = sp.eye(5)

        F = block_diag(F, F, F, F, F)
        Z = block_diag(Z, Z, Z, Z, Z)

        a0 = sp.vstack((a0, a0, a0, a0, a0))
        Q0 = block_diag(Q0, Q0, Q0, Q0, Q0)
        Q = block_diag(Q, Q, Q, Q, Q)

        rsk_filter = RSK(F, Z)

        fitted_means = rsk_filter.fit(panel_series, a0, Q0, Q, sigma=sigma)

        # check that all means are equal
        for row in fitted_means:
            assert np.allclose(row[0:2],
                               row[2:4]), "Measurements differ unexpectedly."
Beispiel #17
0
    def test_compare_ox_multi(self):
        '''
        Compare against multivariate ox reference implementation
        :return:
        '''
        with open(os.path.join(self.datapath, "3/params.json")) as f:
            params = json.loads(f.read())
        yy = sp.array(parse_ox_csv(
            os.path.join(self.datapath, "3/raw_data.csv")),
                      dtype=np.float64)

        # unstack yy
        subarrays = [yy[i * 15:i * 15 + 15, ] for i in range(10)]
        y = sp.stack(tuple(subarrays), axis=0)

        rows = []
        for i, group in enumerate(y):
            for entry in group:
                rows.append([i, "A"] + entry.tolist())
        panel_series = PanelSeries.from_list(rows)

        alpha = sp.matrix(parse_ox_csv(
            os.path.join(self.datapath, "3/alpha.csv")),
                          dtype=np.float64)[:, 1:]
        ox_means = sp.array(parse_ox_csv(
            os.path.join(self.datapath, "3/means.csv")),
                            dtype=np.float64).transpose()[1:]
        py_means, py_cov = panel_series.means(), panel_series.cov()

        # check means
        assert sp.allclose(
            ox_means,
            sp.vstack(py_means)), "Python means do not match OX means"

        # check alphas
        rsk_filter = RSK(sp.matrix(params["transition_matrix"]),
                         sp.matrix(params["translation_matrix"]))
        rsk_alpha, alpha_filter, alpha_smooth, V, V_filter, V_smooth, _ = rsk_filter._fit(
            panel_series,
            sp.matrix(params["a0"]),
            sp.matrix(params["Q0"]),
            sp.matrix(params["Q"]),
            sigma=sp.matrix(params["sigma"]))

        a1 = alpha.transpose()
        a2 = np.squeeze(rsk_alpha)[1:]
        assert sp.allclose(a1, a2), "Alpha does not match OX alpha"
Beispiel #18
0
def median_combine(spec_list, nord, targ_name, ra, dec, plx):
    """Median combine rest frame spectra.

    Parameters
    ----------
    spec_list : array_like
        Array with spectra files.
    nord : int
        Number of echelle orders.
    targ_name : str
        Target's name.
    ra : float
        Target's right ascension in degrees.
    dec : float
        Target's declination in degrees.
    plx : float
        Target's parallax.

    """
    wavelengths = []
    fluxes = []
    for o in tqdm(range(nord), desc='Order'):
        combiner = []
        for spec in spec_list:
            hdul = fits.open(spec)
            flux = hdul[0].data[1, o, :]
            wave = hdul[0].data[0, o, :]
            combiner.append(flux)
            hdul.close()
        combiner = sp.array(combiner)
        combined = sp.median(combiner, axis=0)
        fluxes.append(combined)
        wavelengths.append(wave)
    final_waves = sp.vstack(wavelengths)
    final_fluxes = sp.vstack(fluxes)
    final_out = sp.stack([final_waves, final_fluxes])
    out = targ_name + '_stacked.fits'
    hdr = fits.Header()
    hdr['NAME'] = targ_name
    hdr['PLX'] = plx
    hdr['RA (deg)'] = ra
    hdr['DEC (deg)'] = dec
    hdu = fits.PrimaryHDU(final_out, header=hdr)
    hdu.writeto(out)
    pass
def onclick(event):
    global X, Y, w1

    def xx():
        x = sp.sort(X)
        for i, s in enumerate(list(sp.asarray(X).argsort())):
            if x[i] == x[i - 1]:
                return X.remove(X[s]), Y.remove(Y[s]), xx()

    if event.button == 1 and event.inaxes:
        X.append(event.xdata)
        Y.append(event.ydata)
        w1.set_data(X, Y)
        fig.canvas.draw()
        plt.show()
    else:
        plt.disconnect(cid)

        xx()

        nx = len(X)
        x = sp.poly1d([1, 0])
        L = 0
        for i in sp.arange(nx):
            pom = 1
            for j in sp.hstack((sp.arange(i), sp.arange(i + 1, nx))):
                pom *= (x - X[j]) / (X[i] - X[j])
            L += pom * Y[i]
        print(
            "\nWielomian lagrange interpolujacy {} punkt ma postac:\n\tx\ty\n {}\n nL:\n {} "
            .format(nx, sp.stack((X, Y), axis=1), L))
        xw = sp.linspace(min(X), max(X), 1000)
        yw = L(xw)
        w2.set_data(xw, yw)
        ax.set_title("lagrange interpolacja {} pktow".format(nx))
        ax.set_xlabel('x')
        ax.set_ylabel('y')
        ax.axis('equal')
        ax.legend()
        fig.canvas.draw()
    def sample(self, *shape):
        bimodal_partial_cdf = cumtrapz(self.bimodal_pdf, initial=0, axis=0)
        #First sample in the x coordinate
        samplesx = self.ppfx(sp.random.rand(*shape))
        #Next sample in the ybin
        bin_index = self.ppfix(samplesx)

        #compute samples inside the ybin
        def compute_sample(ysample, xsample, binindex):
            upper_index = sp.int32(sp.ceil(binindex))
            lower_index = sp.int32(sp.floor(binindex))

            ppy_upper = interpolate.interp1d(
                bimodal_partial_cdf[:, upper_index], self.y_eval_space)
            ppy_lower = interpolate.interp1d(
                bimodal_partial_cdf[:, lower_index], self.y_eval_space)

            a = bimodal_partial_cdf[:, upper_index]
            b = bimodal_partial_cdf[:, lower_index]

            samples_upper = ppy_upper(ysample * (max(a) - min(a)) * 0.9999 +
                                      min(a) * 1.001)
            samples_lower = ppy_lower(ysample * (max(b) - min(b)) * 0.9999 +
                                      min(b) * 1.001)

            #Lerp over the lower and upper
            a = self.x_eval_space[upper_index]
            b = self.x_eval_space[lower_index]

            return samples_lower + (samples_upper -
                                    samples_lower) / (a - b) * (xsample - b)

        #Vectorize and sample in ybin
        samplesy = sp.random.rand(*shape)
        compute_samples = sp.vectorize(compute_sample)
        samplesy = compute_samples(samplesy, samplesx, bin_index)

        #Stack the values
        samples = sp.stack([samplesx, samplesy])
        return samples.T
def load_image_data(image_file, invert):
    r"""
    Loads an image from a *.tiff stack and creates an array from it. The
    fracture is assumed to be black and the solid is white.
    """
    logger.info('loading image...')
    img_data = Image.open(image_file)
    #
    # creating full image array
    logger.info('creating image array...')
    data_array = []
    for frame in range(img_data.n_frames):
        img_data.seek(frame)
        frame = sp.array(img_data, dtype=bool).transpose()
        if invert:
            frame = ~frame  # beacuse fracture is black, solid is white
        data_array.append(frame)
    #
    data_array = sp.stack(data_array, axis=2)
    logger.debug('    image dimensions: {} {} {}'.format(*data_array.shape))
    #
    return data_array
 def __new__(cls, image, dtype=bool, *args, **kwargs):
     r""" Reads image data, fracture pixels are assumed to be truthy
     (i.e. > 0) and rock pixels false (i.e. == 0)
     """
     #
     # either reads the image data from a file or image is a scipy array
     try:
         image = Image.open(image)
         logger.debug('loaded image from file or file like object')
         image_data = []
         for frame in range(image.n_frames):
             image.seek(frame)
             frame = sp.array(image, dtype=dtype).transpose()
             image_data.append(frame)
         #
         # stacking frames into a single 3 dimensional array
         image_data = sp.stack(image_data, axis=2)
     except AttributeError:
         logger.debug('initialized image from data array')
         image_data = sp.array(image, ndmin=3, dtype=dtype)
     #
     # returning a conversion of regular ndarray into my sybclass
     return sp.asarray(image_data).view(cls)
vnoise_mu    = (VNOISE_MU[1]-VNOISE_MU[0])*sp.random.rand(N_SAMPLES) + VNOISE_MU[0]
vnoise_sigma = (VNOISE_SCALE[1]-VNOISE_SCALE[0])*sp.random.rand(N_SAMPLES)+VNOISE_SCALE[0]
xnoise_mu1   = (XNOISE_MU1[1]-XNOISE_MU1[0])*sp.random.rand(N_SAMPLES) + XNOISE_MU1[0]
left_right = 2*((sp.random.rand(N_SAMPLES)>0.5)-0.5)
left_right[-1] = 1
left_right[-2] = -1
left_right[-3] = 1
left_right[-4] = -1
xnoise_mu2   = left_right*((XNOISE_MU2[1]-XNOISE_MU2[0])*sp.random.rand(N_SAMPLES) + XNOISE_MU2[0])
xnoise_scale1 = (XNOISE_SCALE1[1]-XNOISE_SCALE1[0])*sp.random.rand(N_SAMPLES) + XNOISE_SCALE1[0]
xnoise_scale2 = (XNOISE_SCALE2[1]-XNOISE_SCALE2[0])*sp.random.rand(N_SAMPLES) + XNOISE_SCALE2[0]

batch_generation_inputs = zip(vnoise_mu,vnoise_sigma,xnoise_mu1,xnoise_mu2,xnoise_scale1,xnoise_scale2)

y_batch, x_batch = list(zip(*[gen_sample(*generator) for generator in batch_generation_inputs]))
batch_y= sp.stack(y_batch)
batch_x= sp.stack(x_batch)
print(batch_y.shape,batch_x.shape)

if False:
    plt.figure(figsize=(14,16))
    for batch_idx in range(N_PLOTS):
        noisy_x = batch_x[batch_idx,:,0]
        noisy_vx = batch_x[batch_idx,:,1]
        true_x = batch_y[batch_idx,:,0]
        true_vx = batch_y[batch_idx,:,1]
        
        plt.subplot(20+(N_PLOTS)*100 + batch_idx*2+1)
        if batch_idx == 0: plt.title('Location x')
        plt.plot(t,true_x,lw=2,label='true')
        plt.plot(t,noisy_x,lw=1,label=r'measured ($\mu =$ [%3.2f, %3.2f], $\sigma =$ [%3.2f, %3.2f])'\
def generate_node_connectivity_array(index_map, data_array):
    r"""
    Generates a node connectivity array based on faces, edges and corner
    adjacency
    """
    #
    logger.info('generating network connections...')
    #
    # setting up some constants
    x_dim, y_dim, z_dim = data_array.shape
    conn_map = list(product([0, -1, 1], [0, -1, 1], [0, -1, 1]))
    conn_map = sp.array(conn_map, dtype=int)
    conn_map = conn_map[1:]
    #
    # creating slice list to process data chunks
    slice_list = [slice(0, 10000)]
    for i in range(slice_list[0].stop, index_map.shape[0], slice_list[0].stop):
        slice_list.append(slice(i, i+slice_list[0].stop))
    slice_list[-1] = slice(slice_list[-1].start, index_map.shape[0])
    #
    conns = sp.ones((0, 2), dtype=sp.uint32)
    logger.debug('    number of slices to process: {}'.format(len(slice_list)))
    for sect in slice_list:
        # getting coordinates of nodes and their neighbors
        nodes = index_map[sect]
        inds = sp.repeat(nodes, conn_map.shape[0], axis=0)
        inds += sp.tile(conn_map, (nodes.shape[0], 1))
        #
        # calculating the flattened index of the central nodes and storing
        nodes = sp.ravel_multi_index(sp.hsplit(nodes, 3), data_array.shape)
        inds = sp.hstack([inds, sp.repeat(nodes, conn_map.shape[0], axis=0)])
        #
        # removing neigbors with negative indicies
        mask = ~inds[:, 0:3] < 0
        inds = inds[sp.sum(mask, axis=1) == 3]
        # removing neighbors with indicies outside of bounds
        mask = (inds[:, 0] < x_dim, inds[:, 1] < y_dim, inds[:, 2] < z_dim)
        mask = sp.stack(mask, axis=1)
        inds = inds[sp.sum(mask, axis=1) == 3]
        # removing indices with zero-weight connection
        mask = data_array[inds[:, 0], inds[:, 1], inds[:, 2]]
        inds = inds[mask]
        if inds.size:
            # calculating flattened index of remaining nieghbor nodes
            nodes = sp.ravel_multi_index(sp.hsplit(inds[:, 0:3], 3),
                                         data_array.shape)
            inds = sp.hstack([sp.reshape(inds[:, -1], (-1, 1)), nodes])
            # ensuring conns[0] is always < conns[1] for duplicate removal
            mask = inds[:, 0] > inds[:, 1]
            inds[mask] = inds[mask][:, ::-1]
            # appending section connectivity data to conns array
            conns = sp.append(conns, inds.astype(sp.uint32), axis=0)
    #
    # using scipy magic from stackoverflow to remove dupilcate connections
    logger.info('removing duplicate connections...')
    dim0 = conns.shape[0]
    conns = sp.ascontiguousarray(conns)
    dtype = sp.dtype((sp.void, conns.dtype.itemsize*conns.shape[1]))
    dim1 = conns.shape[1]
    conns = sp.unique(conns.view(dtype)).view(conns.dtype).reshape(-1, dim1)
    logger.debug('    removed {} duplicates'.format(dim0 - conns.shape[0]))
    #
    return conns
def generate_node_connectivity_array(index_map, data_array):
    r"""
    Generates a node connectivity array based on faces, edges and corner
    adjacency
    """
    #
    logger.info('generating network connections...')
    #
    # setting up some constants
    x_dim, y_dim, z_dim = data_array.shape
    conn_map = list(product([0, -1, 1], [0, -1, 1], [0, -1, 1]))
    #
    conn_map = sp.array(conn_map, dtype=int)
    conn_map = conn_map[1:]
    #
    # creating slice list to process data chunks
    slice_list = [slice(0, 10000)]
    for i in range(slice_list[0].stop, index_map.shape[0], slice_list[0].stop):
        slice_list.append(slice(i, i + slice_list[0].stop))
    slice_list[-1] = slice(slice_list[-1].start, index_map.shape[0])
    #
    conns = sp.ones((0, 2), dtype=data_array.index_int_type)
    logger.debug('\tnumber of slices to process: {}'.format(len(slice_list)))
    percent = 10
    for n, sect in enumerate(slice_list):
        # getting coordinates of nodes and their neighbors
        nodes = index_map[sect]
        inds = sp.repeat(nodes, conn_map.shape[0], axis=0)
        inds += sp.tile(conn_map, (nodes.shape[0], 1))
        #
        # calculating the flattened index of the central nodes and storing
        nodes = sp.ravel_multi_index(sp.hsplit(nodes, 3), data_array.shape)
        inds = sp.hstack([inds, sp.repeat(nodes, conn_map.shape[0], axis=0)])
        #
        # removing neigbors with negative indicies
        mask = ~inds[:, 0:3] < 0
        inds = inds[sp.sum(mask, axis=1) == 3]
        # removing neighbors with indicies outside of bounds
        mask = (inds[:, 0] < x_dim, inds[:, 1] < y_dim, inds[:, 2] < z_dim)
        mask = sp.stack(mask, axis=1)
        inds = inds[sp.sum(mask, axis=1) == 3]
        # removing indices with zero-weight connection
        mask = data_array[inds[:, 0], inds[:, 1], inds[:, 2]]
        inds = inds[mask]
        if inds.size:
            # calculating flattened index of remaining nieghbor nodes
            nodes = sp.ravel_multi_index(sp.hsplit(inds[:, 0:3], 3),
                                         data_array.shape)
            inds = sp.hstack([sp.reshape(inds[:, -1], (-1, 1)), nodes])
            # ensuring conns[0] is always < conns[1] for duplicate removal
            mask = inds[:, 0] > inds[:, 1]
            inds[mask] = inds[mask][:, ::-1]
            # appending section connectivity data to conns array
            conns = sp.append(conns, inds.astype(sp.uint32), axis=0)
        if int(n / len(slice_list) * 100) == percent:
            logger.debug('\tprocessed slice {:5d}, {}% complete'.format(
                n, percent))
            percent += 10
    #
    # using scipy magic from stackoverflow to remove dupilcate connections
    logger.info('removing duplicate connections...')
    dim0 = conns.shape[0]
    conns = sp.ascontiguousarray(conns)
    dtype = sp.dtype((sp.void, conns.dtype.itemsize * conns.shape[1]))
    dim1 = conns.shape[1]
    conns = sp.unique(conns.view(dtype)).view(conns.dtype).reshape(-1, dim1)
    logger.debug('\tremoved {} duplicates'.format(dim0 - conns.shape[0]))
    #
    return conns
Beispiel #26
0
    shapes = [
        functools.reduce(lambda x, y: x * y, variable.get_shape())
        for variable in tf.trainable_variables()
    ]
    print('Nparams: ', functools.reduce(lambda x, y: x + y, shapes))
#%% TRAINING
data = []
#Add saver
saver = tf.train.Saver()
for i in range(8):
    file_name = 'Oval_circ1_N' + str(i + 1) + '.txt'
    data.append(read_file(file_name))
    file_name = 'Oval_circ2_N' + str(i + 1) + '.txt'
    data.append(read_file(file_name))
#data contains [t, t_diff, x,y,z,vx,vy,vz, x_t,y_t,z_t,vx_t,vy_t,vz_t]
all_data = sp.stack(data)
batch_x = all_data[:, :, 2:8]
batch_y = all_data[:, :, 8:]

train_batch_x = batch_x[:BATCH_SIZE, :, :]
train_batch_y = batch_y[:BATCH_SIZE, :, :]
test_batch_x = batch_x[BATCH_SIZE:, :, :]
test_batch_y = batch_y[BATCH_SIZE:, :, :]
#Save losses for plotting of progress
dev_loss_plot = []
tra_loss_plot = []
lr_plot = []

with tf.Session(graph=g1) as sess:
    if RESTORE_CHECKPOINT:
        saver.restore(sess, SAVE_DIR + "/model.ckpt")
Beispiel #27
0
def compute_contacts(dom, people, dmax):
    """
    This function uses a KDTree method to find the contacts \
    between individuals. Moreover the contacts with the walls \
    are also determined from the wall distance (obtained by the \
    fast-marching method).

    Parameters
    ----------
    dom: Domain
        contains everything for managing the domain
    people: numpy array
        people coordinates and radius : x,y,r
    dmax: float
        threshold value used to consider a contact as \
        active (dij<dmax)

    Returns
    -------
    contacts: numpy array
        all the contacts i,j,dij,eij_x,eij_y such that dij<dmax \
        and i<j (no duplication)
    """
    # lf : the number of points at which the algorithm
    # switches over to brute-force. Has to be positive.
    lf = 100
    if (lf > sys.getrecursionlimit()):
        sys.setrecursionlimit(lf)
    kd = cKDTree(people[:, :2], leafsize=lf)
    ## Find all pairs of points whose distance is at most dmax+2*rmax
    rmax = people[:, 2].max()
    neighbors = kd.query_ball_tree(kd, dmax + 2 * rmax)
    ## Create the contact array : i,j,dij,eij_x,eij_y
    first_elements = sp.arange(people.shape[0])  ## i.e. i
    other_elements = list(map(lambda x: x[1:],
                              neighbors))  ## i.e. all the j values for each i
    lengths = list(map(len, other_elements))
    tt = sp.stack([first_elements, lengths], axis=1)
    I = sp.concatenate(list(map(lambda x: sp.full((x[1], ), x[0]),
                                tt))).astype(int)
    J = sp.concatenate(other_elements).astype(int)
    ind = sp.where(I < J)[0]
    I = I[ind]
    J = J[ind]
    DP = people[J, :2] - people[I, :2]
    Norm = sp.linalg.norm(DP, axis=1, ord=2)
    Dij = Norm - people[I, 2] - people[J, 2]
    ind = sp.where(Dij < dmax)[0]
    Dij = Dij[ind]
    I = I[ind]
    J = J[ind]
    Norm = Norm[ind]
    DP = DP[ind]
    contacts = sp.stack([I, J, Dij, DP[:, 0] / Norm, DP[:, 1] / Norm], axis=1)
    # Add contacts with the walls
    II = sp.floor((people[:, 1] - dom.ymin - 0.5 * dom.pixel_size) /
                  dom.pixel_size).astype(int)
    JJ = sp.floor((people[:, 0] - dom.xmin - 0.5 * dom.pixel_size) /
                  dom.pixel_size).astype(int)
    DD = dom.wall_distance[II, JJ] - people[:, 2]
    ind = sp.where(DD < dmax)[0]
    wall_contacts = sp.stack([
        ind, -1 * sp.ones(ind.shape), DD[ind],
        dom.wall_grad_X[II[ind], JJ[ind]], dom.wall_grad_Y[II[ind], JJ[ind]]
    ],
                             axis=1)
    contacts = sp.vstack([contacts, wall_contacts])
    return sp.array(contacts)
Beispiel #28
0
    def ovf_to_hdf(self,
                   hdf_name,
                   ovf_files=[],
                   delete_ovfs=False,
                   overwrite=False):
        """Load the data from multiple .ovf files into a chunked .hdf5 file.
        This method is useful when the size of the simulation data is too large fit into RAM"""

        # check if destination file already exists and whether to overwrite
        if (os.path.isfile(hdf_name)) & (overwrite == False):
            print(
                'file already exists. Set kwarg "overwrite=True" to overwrite existing file. Aborting...'
            )
            return 0

        # calculate the size and shape of the data we are dealing with
        meta, header = self.read_ovf(ovf_files[0], target='meta')
        header_encoded = [n.encode("ascii", "ignore") for n in header]

        data_shape = sp.array([
            meta['znodes'], meta['ynodes'], meta['xnodes'], meta['valuedim'],
            len(ovf_files)
        ])
        data_size = sp.prod(data_shape * int(meta['Binary']))
        time = []

        with hd.File(hdf_name, 'w', libver='latest') as f:
            dset = f.create_dataset('mag',
                                    data_shape,
                                    dtype=sp.dtype('f' +
                                                   str(int(meta['Binary']))),
                                    chunks=True)

            # want to add the meta data in but cannot seem to get it to work
            # There is a problem with data format supported by hdf5

            # go through the ovf files and populate the hdf file with the data
            chunk_time_length = dset.chunks[-1]
            data_shape = dset.shape
            chunk_number = data_shape[-1] / chunk_time_length
            chunk_shape = dset.chunks

            # Close the hdf5 file at every opportunity
            # According to docs: https://support.hdfgroup.org/HDF5/faq/perfissues.html
            # a memory leak can occur when writing to the same file many times in a loop
            print("creating dataset")
            print("shape ", dset.shape)
            print("chunks ", dset.chunks)

            # prepare an array of all the data in one time chunk
            for c in tqdm(range(int(sp.ceil(chunk_number)))):
                temp_arr = sp.zeros(
                    (data_shape[0], data_shape[1], data_shape[2],
                     data_shape[3], chunk_shape[-1]))

                # fill the temp array with data from ovf files
                try:
                    for n in range(chunk_time_length):
                        temp_arr[:, :, :, :, n], meta, raw = self.read_ovf(
                            ovf_files[chunk_time_length * c + n])
                        time.append(meta['time'])
                except:
                    # This catches the unexpected case where the chunk length in time
                    # does not perfectly divide the length of ovf list

                    temp = list(
                        map(self.read_ovf, ovf_files[chunk_time_length * c:]))
                    temp_arr = sp.stack([a[0] for a in temp], axis=-1)

                # open hdf5 file, write the time chunk to disk, close hdf5 file
                with hd.File(hdf_name, 'r+', libver="latest") as f:
                    f['mag'][:, :, :, :, chunk_time_length *
                             c:chunk_time_length * (c + 1)] = temp_arr

                # optionally delete the ovf files as they are written to hdf5
                if delete_ovfs == True:
                    for n in range(chunk_time_length):
                        os.remove(ovf_files[chunk_time_length * c + n])

        # Append to the hdf5 file additional meta data
        with hd.File(hdf_name, 'a', libver='latest') as f:
            f.create_dataset('time', data=sp.array(time))
            f.create_dataset('header', (len(header_encoded), 1), 'S30',
                             header_encoded)
            try:
                f.create_dataset('meta', meta.data)
            except:
                print('failed to save metadata to disk')

        # change permissions data.hdf5 file to be read only
        os.chmod(hdf_name, 444)

        return 0
Beispiel #29
0
### COMMON
# the names of the things present in the log
span_names = [name.split('_ON')[0] for name in CodesDf['name'] if name.endswith('_ON')]
event_names = [name.split('_EVENT')[0] for name in CodesDf['name'] if name.endswith('_EVENT')]

Spans = bhv.log2Spans(Data, span_names)
Events = bhv.log2Events(Data, event_names)

### SOME PREPROCESSING
# filter unrealistic licks
bad_licks = sp.logical_or(Spans['LICK']['dt'] < 20,Spans['LICK']['dt'] > 100)
Spans['LICK'] = Spans['LICK'].loc[~bad_licks]

# add lick_event
Lick_Event = pd.DataFrame(sp.stack([['NA']*Spans['LICK'].shape[0],Spans['LICK']['t_on'].values,['LICK_EVENT']*Spans['LICK'].shape[0]]).T,columns=['code','t','name'])
Lick_Event['t'] = Lick_Event['t'].astype('float')
Data = Data.append(Lick_Event)
Data.sort_values('t')

event_names.append("LICK")
Events['LICK'] = bhv.log2Event(Data,'LICK')

Spans.pop("LICK")
span_names.remove("LICK")

colors = sns.color_palette('hls',n_colors=len(event_names)+len(span_names))[::-1]
cdict = dict(zip(event_names+span_names,colors))


"""
Beispiel #30
0
    def fit(self, steps=200, eps=1e-5, num_restarts=3):
        """Learn the parameters omega, sigma, eta from training data `train` via
        EM.

        Args:
            steps: integer, the maximum number of EM steps to take before
            terminating
            eps: integer, the tolerance threshold for terminating early from the
            optimization routine. Optimization stops when the change in each
            parameter is no greater than the value specified here.
            num_restarts: integers, the number of restarts to perform

        Returns: a pointer to `self`, where the parameter attributes omega,
        sigma, and eta have been learned.
        """
        train = self.train
        N = len(train)
        T = self.T

        # Allocate "p" matrices
        p_aftershock = sp.zeros((N, N))
        p_background = sp.zeros((N, N))

        # Massage data into proper format and compute reusable values for E-step
        # (only needs to be done once)

        # Provides iteration over the data (each pair of rows) without using a
        # `for` loop
        i, j = sp.ogrid[0:N, 0:N]
        t_i = train.iloc[i.reshape(N, )]['t'].values.reshape(N, 1)
        t_j = train.iloc[j.reshape(N, )]['t'].values.reshape(1, N)
        x_i = train.iloc[i.reshape(N, )]['x'].values.reshape(N, 1)
        x_j = train.iloc[j.reshape(N, )]['x'].values.reshape(1, N)
        y_i = train.iloc[i.reshape(N, )]['y'].values.reshape(N, 1)
        y_j = train.iloc[j.reshape(N, )]['y'].values.reshape(1, N)

        distance = (x_j - x_i)**2 + (y_j - y_i)**2
        time_check = t_i < t_j
        origin_check = sp.logical_and((x_i != x_j), (y_i != y_j))
        trigger_check = time_check * origin_check
        t_diff = t_j - t_i

        best_params = None
        best_likelihood = None
        for _ in range(num_restarts):
            # Initialize parameters to random values
            # Parameters for triggering kernel
            omega = sp.absolute(stats.norm.rvs(scale=0.10))  # time decay
            sigma = sp.absolute(sp.randn())  # spatial decay
            # Parameter for background rate
            eta = sp.absolute(sp.randn())
            # Loop until convergence
            for step in tqdm(range(steps)):
                old_parameters = sp.stack([omega, sigma, eta])

                # E-step: Calculate p matrices according to equations (9) and (10)
                # "[P and P^b] contain the probabilities that event i triggered
                # homicide j through either the triggering kernel g or the
                # background rate kernel"

                p_aftershock = trigger_check * omega * sp.exp(-omega * t_diff) \
                    * stats.norm.pdf(distance, scale=sigma)
                p_background = origin_check * stats.norm.pdf(distance,
                                                             scale=eta)

                # Normalize as necessary
                Z = p_aftershock + p_background  # also used for likelihood
                nonzero_Z = Z > 0
                p_aftershock[nonzero_Z] /= Z[nonzero_Z]
                p_background[nonzero_Z] /= T * Z[nonzero_Z]

                # M-step: Update parameters

                aftershock_sum = sp.sum(p_aftershock)
                time_gaps = T - t_i
                omega_denom = sp.sum(p_aftershock * (t_j - t_i)) \
                            + sp.sum(time_gaps * sp.exp(-omega * time_gaps))
                omega = aftershock_sum / omega_denom

                sigma = sp.sum(
                    p_aftershock * distance) / (2.0 * aftershock_sum)
                eta = sp.sum(
                    p_background * distance) / (2.0 * sp.sum(p_background))

                diff = sp.absolute(old_parameters -
                                   sp.stack([omega, sigma, eta])).max()
                if diff < eps:
                    print("Convergence met after {} iterations: {}".format(
                        step, sp.sum(Z)))
                    break
            # Check the likelihood given the discovered parameter values by
            # summing over Z
            likelihood = sp.sum(Z)
            if best_likelihood is None or likelihood > best_likelihood:
                best_params = omega, sigma, eta
                best_likelihood = likelihood

        self.p_aftershock, self.p_background = p_aftershock, p_background
        self.omega, self.sigma, self.eta = best_params
        return self
Beispiel #31
0
asig = Seg.analogsignals[0]
fig, axes = plt.subplots(ncols=nEvents,figsize=[6,2.5])
t_slice = (-2,2) * pq.s
for i, event in enumerate(Events[:-1]):
    asig_slices = []
    for t in event.times:
        try:
            asig_slice = asig.time_slice(t+t_slice[0],t+t_slice[1])
            tvec = asig_slice.times - t
            # axes[i].plot(tvec, asig_slice, 'k', alpha=0.25,lw=1)
            asig_slices.append(asig_slice)
        except ValueError:
            pass
    
    # average:
    avg = sp.stack([asig_slice.magnitude for asig_slice in asig_slices],axis=1).mean(axis=1)
    axes[i].plot(tvec,avg,'r')
    axes[i].plot(Kernels.times,Kernels[:,i],'C%i'%i)
    axes[i].set_xlabel('time (s)')

axes[0].set_ylabel('signal (au)')

fig.suptitle('event triggered average')
sns.despine(fig)
fig.tight_layout(rect=[0, 0.03, 1, 0.95])

# %%
"""
 
 ##     ## ##     ##    ###    ##       
  ##   ##  ##     ##   ## ##   ##       
Beispiel #32
0
    T = 20
    N = 100
    M = 200
    target_dir = os.path.join('data', 'bouncing_balls_ds0p1')
    os.makedirs(target_dir, exist_ok=True)
    nrof_balls = 1
    for j in range(M):
        print('.', end='')
        sys.stdout.flush()
        dat = scipy.empty((N), dtype=object)
        for i in range(N):
            dat[i] = bounce_vec(res=res,
                                n=nrof_balls,
                                T=T,
                                diffusion_stdev=diffusion_stdev)
        data = np.reshape(scipy.stack(dat), (N, T, res, res))
        utils.save_pickle(os.path.join(target_dir, 'train_%03d.pkl' % j), data)
    print('\nDone')

    N = 100
    M = 10
    dat = scipy.empty((N), dtype=object)
    for j in range(M):
        for i in range(N):
            dat[i] = bounce_vec(res=res,
                                n=nrof_balls,
                                T=T,
                                diffusion_stdev=diffusion_stdev)
        data = np.reshape(scipy.stack(dat), (N, T, res, res))
        utils.save_pickle(os.path.join(target_dir, 'test_%03d.pkl' % j), data)