예제 #1
0
def evaluate(gtdir, preddir, eval_pose=True, eval_track=True,
             eval_upper_bound=False):
    gtFramesAll, prFramesAll = load_data_dir(['', gtdir, preddir])

    print('# gt frames  :', len(gtFramesAll))
    print('# pred frames:', len(prFramesAll))

    apAll = np.full((Joint().count + 1, 1), np.nan)
    preAll = np.full((Joint().count + 1, 1), np.nan)
    recAll = np.full((Joint().count + 1, 1), np.nan)
    if eval_pose:
        apAll, preAll, recAll = evaluateAP(gtFramesAll, prFramesAll)
        print('Average Precision (AP) metric:')
        printTable(apAll)

    metrics = np.full((Joint().count + 4, 1), np.nan)
    if eval_track:
        metricsAll = evaluateTracking(
            gtFramesAll, prFramesAll, eval_upper_bound)

        for i in range(Joint().count + 1):
            metrics[i, 0] = metricsAll['mota'][0, i]
        metrics[Joint().count + 1, 0] = metricsAll['motp'][0, Joint().count]
        metrics[Joint().count + 2, 0] = metricsAll['pre'][0, Joint().count]
        metrics[Joint().count + 3, 0] = metricsAll['rec'][0, Joint().count]
        print('Multiple Object Tracking (MOT) metrics:')
        printTable(metrics, motHeader=True)
    return (apAll, preAll, recAll), metrics
예제 #2
0
  def testSequenceLoss(self):
    self.config_default_values()
    with self.cached_session(use_gpu=True):
      average_loss_per_example = loss.sequence_loss(
          self.logits, self.targets, self.weights,
          average_across_timesteps=True,
          average_across_batch=True)
      res = self.evaluate(average_loss_per_example)
      self.assertAllClose(self.expected_loss, res)

      average_loss_per_sequence = loss.sequence_loss(
          self.logits, self.targets, self.weights,
          average_across_timesteps=False,
          average_across_batch=True)
      res = self.evaluate(average_loss_per_sequence)
      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
      self.assertAllClose(compare_per_sequence, res)

      average_loss_per_batch = loss.sequence_loss(
          self.logits, self.targets, self.weights,
          average_across_timesteps=True,
          average_across_batch=False)
      res = self.evaluate(average_loss_per_batch)
      compare_per_batch = np.full((self.batch_size), self.expected_loss)
      self.assertAllClose(compare_per_batch, res)

      total_loss = loss.sequence_loss(
          self.logits, self.targets, self.weights,
          average_across_timesteps=False,
          average_across_batch=False)
      res = self.evaluate(total_loss)
      compare_total = np.full((self.batch_size, self.sequence_length),
                              self.expected_loss)
      self.assertAllClose(compare_total, res)
예제 #3
0
    def at(self, points, time, units=None, extrapolate=False, **kwargs):
        '''
        Interpolates this property to the given points at the given time with the units specified
        :param points: A Nx2 array of lon,lat points
        :param time: A datetime object. May be None; if this is so, the variable is assumed to be gridded
        but time-invariant
        :param units: The units that the result would be converted to
        '''
        value = None
        if len(self.time) == 1:
            # single time time series (constant)
            value = np.full((points.shape[0], 1), self.data, dtype=np.float64)
            if units is not None and units != self.units:
                value = unit_conversion.convert(self.units, units, value)
            return value

        if not extrapolate:
            self.time.valid_time(time)
        t_index = self.time.index_of(time, extrapolate)
        if time > self.time.max_time:
            value = self.data[-1]
        if time <= self.time.min_time:
            value = self.data[0]
        if value is None:
            t_alphas = self.time.interp_alpha(time, extrapolate)

            d0 = self.data[t_index - 1]
            d1 = self.data[t_index]
            value = d0 + (d1 - d0) * t_alphas
        if units is not None and units != self.units:
            value = unit_conversion.convert(self.units, units, value)

        return np.full((points.shape[0], 1), value, dtype=np.float64)
    def _clean_timeseries(self, timeseries, starttime, endtime):
        """Realigns timeseries data so the start and endtimes are the same
            as what was originally asked for, even if the data was during
            a gap.

        Parameters
        ----------
        timeseries: obspy.core.stream
            The timeseries stream as returned by the call to getWaveform
        starttime: obspy.core.UTCDateTime
            the starttime of the requested data
        endtime: obspy.core.UTCDateTime
            the endtime of the requested data

        Notes: the original timeseries object is changed.
        """
        for trace in timeseries:
            trace_starttime = UTCDateTime(trace.stats.starttime)
            trace_endtime = UTCDateTime(trace.stats.endtime)

            if trace.stats.starttime > starttime:
                cnt = int((trace_starttime - starttime) / trace.stats.delta)
                trace.data = numpy.concatenate([
                        numpy.full(cnt, numpy.nan, dtype=numpy.float64),
                        trace.data])
                trace.stats.starttime = starttime
            if trace_endtime < endtime:
                cnt = int((endtime - trace_endtime) / trace.stats.delta)
                trace.data = numpy.concatenate([
                        trace.data,
                        numpy.full(cnt, numpy.nan, dtype=numpy.float64)])
                trace.stats.endttime = endtime
예제 #5
0
def crossval_predict(predictor, X, y, prefix, n_cv=5):
    if not np.array_equal(predictor.classes_, [0, 1]):
        raise Exception("classes labels NOT match")

    can_pred_proba = common.can_predict_probability(predictor)

    n_samples = X.shape[0]
    print "totally {} samples, divided into {} folds".format(n_samples, n_cv)

    if can_pred_proba:
        datas = np.full((n_samples, 2), np.NaN)
        headers = ["{}_{}".format(prefix, t) for t in ["proba", "log_proba"]]
        yvalidates = pd.DataFrame(datas, columns=headers, index=y.index)
    else:
        datas = np.full((n_samples, 1), np.NaN)
        header = "{}_label".format(prefix)
        yvalidates = pd.DataFrame(datas, columns=[header], index=y.index)

    folds = StratifiedKFold(y, n_folds=n_cv, shuffle=True, random_state=seed)
    for index, (train_index, test_index) in enumerate(folds):
        Xtrain, Xtest = X[train_index], X[test_index]
        ytrain, ytest = y[train_index], y[test_index]

        predictor.fit(Xtrain, ytrain)
        if can_pred_proba:
            ytest_probas = predictor.predict_proba(Xtest)
            pos_proba = ytest_probas[:, 1]  # probability for label=1 (Positive)
            yvalidates.iloc[test_index, 0] = pos_proba
            yvalidates.iloc[test_index, 1] = np.log(pos_proba)
        else:
            yvalidates.iloc[test_index, 0] = predictor.predict(Xtest)

        print "====== cross-validated on {}-fold ======".format(index + 1)

    return yvalidates
예제 #6
0
    def calculate_landslide_probability(self, **kwds):
        """Main method of Landslide Probability class.

        Method creates arrays for output variables then loops through all
        the core nodes to run the method 'calculate_factor_of_safety.'
        Output parameters probability of failure, mean relative wetness,
        and probability of saturation are assigned as fields to nodes.
        """
        # Create arrays for data with -9999 as default to store output
        self.mean_Relative_Wetness = np.full(self.grid.number_of_nodes, -9999.)
        self.prob_fail = np.full(self.grid.number_of_nodes, -9999.)
        self.prob_sat = np.full(self.grid.number_of_nodes, -9999.)
        # Run factor of safety Monte Carlo for all core nodes in domain
        # i refers to each core node id
        for i in self.grid.core_nodes:
            self.calculate_factor_of_safety(i)
            # Populate storage arrays with calculated values
            self.mean_Relative_Wetness[i] = self._soil__mean_relative_wetness
            self.prob_fail[i] = self._landslide__probability_of_failure
            self.prob_sat[i] = self._soil__probability_of_saturation
        # Values can't be negative
        self.mean_Relative_Wetness[self.mean_Relative_Wetness < 0.] = 0.
        self.prob_fail[self.prob_fail < 0.] = 0.
        # assign output fields to nodes
        self.grid.at_node["soil__mean_relative_wetness"] = self.mean_Relative_Wetness
        self.grid.at_node["landslide__probability_of_failure"] = self.prob_fail
        self.grid.at_node["soil__probability_of_saturation"] = self.prob_sat
예제 #7
0
  def testMakeTableExceptions(self):
    # Verify that contents is being type-checked and shape-checked.
    with self.assertRaises(ValueError):
      text_plugin.make_table([])

    with self.assertRaises(ValueError):
      text_plugin.make_table('foo')

    with self.assertRaises(ValueError):
      invalid_shape = np.full((3, 3, 3), 'nope', dtype=np.dtype('S3'))
      text_plugin.make_table(invalid_shape)

    # Test headers exceptions in 2d array case.
    test_array = np.full((3, 3), 'foo', dtype=np.dtype('S3'))
    with self.assertRaises(ValueError):
      # Headers is wrong type.
      text_plugin.make_table(test_array, headers='foo')
    with self.assertRaises(ValueError):
      # Too many headers.
      text_plugin.make_table(test_array, headers=['foo', 'bar', 'zod', 'zoink'])
    with self.assertRaises(ValueError):
      # headers is 2d
      text_plugin.make_table(test_array, headers=test_array)

    # Also make sure the column counting logic works in the 1d array case.
    test_array = np.array(['foo', 'bar', 'zod'])
    with self.assertRaises(ValueError):
      # Too many headers.
      text_plugin.make_table(test_array, headers=test_array)
예제 #8
0
 def a_variations():
     yield np.arange(10)
     yield np.array([-1.1, np.nan, 2.2])
     yield np.array([-np.inf, 5])
     yield (4, 2, 5)
     yield (1,)
     yield np.full(5, 5)
     yield [2.2, -2.3, 0.1]
     a = np.linspace(-10, 10, 16).reshape(4, 2, 2)
     yield a
     yield np.asfortranarray(a)
     yield a[::-1]
     np.random.RandomState(0).shuffle(a)
     yield a
     yield 6
     yield 6.5
     yield -np.inf
     yield 1 + 4j
     yield [2.2, np.nan]
     yield [2.2, np.inf]
     yield ((4.1, 2.0, -7.6), (4.3, 2.7, 5.2))
     yield np.full(5, np.nan)
     yield 1 + np.nan * 1j
     yield np.nan + np.nan * 1j
     yield np.nan
예제 #9
0
    def test_window_safe(self, factor_len):
        # all true data set of (days, securities)
        data = full(self.default_shape, True, dtype=bool)

        class InputFilter(Filter):
            inputs = ()
            window_length = 0

        class TestFactor(CustomFactor):
            dtype = float64_dtype
            inputs = (InputFilter(), )
            window_length = factor_len

            def compute(self, today, assets, out, filter_):
                # sum for each column
                out[:] = np_sum(filter_, axis=0)

        results = self.run_graph(
            TermGraph({'windowsafe': TestFactor()}),
            initial_workspace={InputFilter(): data},
        )

        # number of days in default_shape
        n = self.default_shape[0]

        # shape of output array
        output_shape = ((n - factor_len + 1), self.default_shape[1])
        check_arrays(
            results['windowsafe'],
            full(output_shape, factor_len, dtype=float64)
        )
예제 #10
0
def show_output(net,testing=False):
    global error
    global error_divisor

    if testing:
        time.sleep(TRAINING_DURATION/2)
        output = net.get_output()
        output_init = output # Only different line
        output = [round(x*255) for x in output]
        #print "Red: " + str(output[2]) + "\t" + "Green: " + str(output[1]) + "\t" + "Blue: " + str(output[0]) + "\r",
        #sys.stdout.flush()
        output = np.full((32, 32, 3), output, dtype='uint8')
        cv2.imshow("Output", output)
        cv2.waitKey(int(1000 * TRAINING_DURATION / 2))

        error += abs(testing[2] - output_init[2])
        error += abs(testing[0] - output_init[0])
        error_divisor += 2
    else:
        time.sleep(TRAINING_DURATION/2)
        output = net.get_output()
        output = [round(x*255) for x in output]
        #print "Red: " + str(output[2]) + "\t" + "Green: " + str(output[1]) + "\t" + "Blue: " + str(output[0]) + "\r",
        #sys.stdout.flush()
        output = np.full((32, 32, 3), output, dtype='uint8')
        cv2.imshow("Output", output)
        cv2.waitKey(int(1000 * TRAINING_DURATION / 2))
예제 #11
0
def run(oiter):
    # ----- Variable for this run -----
    log_alpha_0 = all_log_alpha_0[oiter]

    print "Running job {0} on {1}".format(oiter + 1, socket.gethostname())
    train_images, train_labels, _, _, _ = load_data()
    train_images = train_images[:N_data, :]
    train_labels = train_labels[:N_data, :]
    batch_idxs = BatchList(N_data, batch_size)
    iter_per_epoch = len(batch_idxs)
    N_weights, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg)
    def indexed_loss_fun(w, idxs):
        return loss_fun(w, X=train_images[idxs], T=train_labels[idxs])

    V0 = npr.randn(N_weights) * velocity_scale
    losses = []
    d_losses = []
    alpha_0 = np.exp(log_alpha_0)
    for N_iters in all_N_iters:
        alphas = np.full(N_iters, alpha_0)
        betas = np.full(N_iters, beta_0)
        npr.seed(1)
        W0 = npr.randn(N_weights) * np.exp(log_param_scale)
        results = sgd(indexed_loss_fun, batch_idxs, N_iters, W0, V0, alphas, betas)
        losses.append(results['loss_final'])
        d_losses.append(d_log_loss(alpha_0, results['d_alphas']))

    return losses, d_losses
예제 #12
0
def test_uninterpolated_nan_regions(boundary, normalize_kernel):
    #8086
    # Test NaN interpolation of contiguous NaN regions with kernels of size
    # identical and greater than that of the region of NaN values.

    # Test case: kernel.shape == NaN_region.shape
    kernel = Gaussian2DKernel(1, 5, 5)
    nan_centroid = np.full(kernel.shape, np.nan)
    image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
                   constant_values=1)
    with pytest.warns(AstropyUserWarning,
                      match="nan_treatment='interpolate', however, NaN values detected "
                      "post convolution. A contiguous region of NaN values, larger "
                      "than the kernel size, are present in the input array. "
                      "Increase the kernel size to avoid this."):
        result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
                          normalize_kernel=normalize_kernel)
        assert(np.any(np.isnan(result)))

    # Test case: kernel.shape > NaN_region.shape
    nan_centroid = np.full((kernel.shape[0]-1, kernel.shape[1]-1), np.nan) # 1 smaller than kerenel
    image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
                   constant_values=1)
    result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
                      normalize_kernel=normalize_kernel)
    assert(~np.any(np.isnan(result))) # Note: negation
예제 #13
0
def complete_obs_table(obs_table, used_columns, filter_list, tolerance,
                       lim_flag, default_error=0.1, systematic_deviation=0.1):
    """Complete the observation table

    For each filter:
    * If the corresponding error is not present in the used column list or in
      the table columns, add (or replace) an error column with the default
      error.
    * Adjust the error value.

    Parameters
    ----------
    obs_table: astropy.table.Table
        The observation table.
    used_columns: list of strings
        The list of columns to use in the observation table.
    filter_list: list of strings
        The list of filters used in the analysis.
    tolerance: float
        Tolerance threshold under flux error is considered as 0.
    lim_flag: boolean
        Do we process upper limits (True) or treat them as no-data (False)?
    default_error: float
        Default error factor used when the provided error in under the
        tolerance threshold.
    systematic_deviation: float
        Systematic deviation added to the error.

    Returns
    -------
    obs_table = astropy.table.Table
        The completed observation table

    Raises
    ------
    Exception: When a filter is not present in the observation table.

    """
    # TODO Print or log a warning when an error column is in the used column
    # list but is not present in the observation table.
    for name in filter_list:
        if name not in obs_table.columns:
            raise Exception("The filter <{}> (at least) is not present in "
                                "the observation table.".format(name))

        name_err = name + "_err"
        if name_err not in obs_table.columns:
            obs_table.add_column(Column(name=name_err,
                                        data=np.full(len(obs_table), np.nan)),
            index=obs_table.colnames.index(name)+1)
        elif name_err not in used_columns:
            obs_table[name_err] = np.full(len(obs_table), np.nan)

        obs_table[name], obs_table[name_err] = adjust_data(obs_table[name],
                                                           obs_table[name_err],
                                                           tolerance,
                                                           lim_flag,
                                                           default_error,
                                                           systematic_deviation)
    return obs_table
    def loop_over_zone(self, array):
        """Generate a masked array and retrieve average values.

        :param array: ndarray, representing zone.

        :returns: ndarray, average values for zone.
        """
        mesh_1 = self.model_data.model_mesh3D[1]
        rows = mesh_1.shape[0]
        arr_zoned = [np.full(mesh_1.shape[1:3], np.nan)] * int(np.max(mesh_1))

        for zone in range(int(np.max(mesh_1))):
            temp = np.array([np.full(mesh_1.shape[1:3], np.nan)] * rows)

            zone_1 = float(zone + 1)
            for layer in range(rows):
                mesh_1_layer = mesh_1[layer]
                temp[layer][mesh_1_layer == zone_1] = array[layer][mesh_1_layer == zone_1]
            # End for

            masked_temp = np.ma.masked_array(temp, np.isnan(temp))
            arr_zoned[zone] = np.mean(masked_temp, axis=0)
        # End for

        return arr_zoned
예제 #15
0
 def _init(self, X, lengths, params):
     init = 1. / self.n_components
     if 's' in params or not hasattr(self, "startprob_"):
         self.startprob_ = np.full(self.n_components, init)
     if 't' in params or not hasattr(self, "transmat_"):
         self.transmat_ = np.full((self.n_components, self.n_components),
                                  init)
def iterations(X, y, T):
	d,n=X.shape
	e0=1.0
	f0=1.0
	a0=np.full(d, 10e-16)
	b0=np.full(d, 10e-16)
	mu0=np.ones((d,1))
	sigma0=np.ones((d,d))
	L=np.zeros(T)
	Eln_qw=np.zeros(T)
	Eln_qlambda=np.zeros(T)
	Eln_qalpha=np.zeros(T)
	Eln_pw=np.zeros(T)
	Eln_plambda=np.zeros(T)
	Eln_palpha=np.zeros(T)
	Eln_py=np.zeros(T)
	for t in range(T):
		a,b,e,f,mu,sigma=updates(X,y,a0,b0,e0,f0,mu0,sigma0)
		L[t]=obj_function(X,y,a,b,e,f,mu,sigma) 
		if t<T:
			a0=a
			b0=b
			e0=e
			f0=f
			mu0=mu
			sigma0=sigma
	return a, b, e, f, L, mu
예제 #17
0
파일: CLSolve.py 프로젝트: ohlord/cimpress
    def initBuffers(self,puzzle):
        #define lengths buffer and copy to the GPU
        #as we will not read from this buffer later, mapping is not required
        self.lengths = np.full(self.simulations,np.iinfo(np.int16).max,dtype=np.int16)
        self.lengthsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.lengths)
         
        #define buffer for aggregated lengths for each workgroup
        self.groupLengths = np.full(self.workGroups,np.iinfo(np.int16).max,dtype=np.int16)
        self.groupLengthsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.USE_HOST_PTR, hostbuf=self.groupLengths)
        
        #map group lengths buffer
        cl.enqueue_map_buffer(self.queue,self.groupLengthsBuffer,cl.map_flags.READ,0,self.groupLengths.shape,self.groupLengths.dtype)
        
        #get the input puzzle ready for the kernel; convert to 8 bit int (char)
        p = np.array(puzzle['puzzle']).astype(np.int8)
        #subtract 1 so that -1 denotes a gap and 0 denotes a square to be filled
        p = p - np.ones_like(p,dtype=p.dtype)
        
        #copy the puzzle, one for each simulation
        self.puzzles = np.zeros((self.simulations,self.height,self.width),dtype=p.dtype)
        self.puzzles[:,0:self.height,0:self.width] = p
    
        #define puzzles buffer and copy data (we do not need to worry about getting data out of this buffer, so mapping isn't required)
        #this buffer contains the input puzzles, one for each invocation (the puzzle is too large to hold in local or shared memory)
        self.puzzlesFlattened = self.puzzles.ravel()
        self.puzzlesBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.puzzlesFlattened)
        
        #define output buffer for best solutions aggregated across workgroups
        self.solutions = self.puzzles[0:self.workGroups]
        self.solutionsFlattened = self.solutions.ravel()
        self.solutionsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.USE_HOST_PTR, hostbuf=self.solutionsFlattened)

        #map solutions buffer
        cl.enqueue_map_buffer(self.queue,self.solutionsBuffer,cl.map_flags.READ,0,self.solutionsFlattened.shape,self.solutions.dtype)
예제 #18
0
파일: threefilter.py 프로젝트: yuanxm5/dip
def filter2d(input_img, filter, size = 3):
    newimg = Image.new(input_img.mode, input_img.size)
    data = list(input_img.getdata())
    pixels = np.reshape(data, (input_img.size[1], input_img.size[0]))
    a = size/2
    b = size/2
    imgheight, imgwidth = input_img.size
    if filter == 'average':
        weight = np.full(size*size, float(1)/(size*size))
    if filter == 'laplacian':
        laplacian = np.array([[-1, -1, -1],
                              [-1,  8, -1],
                              [-1, -1, -1]])
        
        weight = laplacian.flatten()
    if filter == 'sobel1':
        sobel1 = np.array([[-1, -2, -1],
                           [0,  0,  0],
                           [1,  2,  1]])
        weight = sobel1.flatten()
    if filter == 'sobel2':
        sobel2 = np.array([[-1, 0, 1],
                           [-2, 0, 2],
                           [-1, 0, 1]])
        weight = sobel2.flatten()
    for y in range(input_img.size[1]):
        for x in range(input_img.size[0]):                
            z = np.full(size * size, pixels[y][x])
            for j in range(y - a, y + a + 1):
                for i in range(x - b, x + b + 1):
                    if i > 0 and i < imgheight and j > 0 and j < imgwidth:
                        z[(j - y + a) * size + i - x + b] = pixels[j][i]
            newimg.putpixel((x, y), np.dot(weight, z))
    return newimg
예제 #19
0
    def test_multiple_rolling_factors(self):

        loader = self.loader
        engine = SimpleFFCEngine(loader, self.dates, self.asset_finder)
        shape = num_dates, num_assets = (5, len(self.assets))
        dates = self.dates[10:10 + num_dates]

        short_factor = RollingSumDifference(window_length=3)
        long_factor = RollingSumDifference(window_length=5)
        high_factor = RollingSumDifference(
            window_length=3,
            inputs=[USEquityPricing.open, USEquityPricing.high],
        )

        results = engine.factor_matrix(
            {'short': short_factor, 'long': long_factor, 'high': high_factor},
            dates[0],
            dates[-1],
        )
        self.assertEqual(set(results.columns), {'short', 'high', 'long'})

        # row-wise sum over an array whose values are all (1 - 2)
        assert_array_equal(
            results['short'].unstack().values,
            full(shape, -short_factor.window_length),
        )
        assert_array_equal(
            results['long'].unstack().values,
            full(shape, -long_factor.window_length),
        )
        # row-wise sum over an array whose values are all (1 - 3)
        assert_array_equal(
            results['high'].unstack().values,
            full(shape, -2 * high_factor.window_length),
        )
def generateFileList(rootPath):
	iterNum = 0
	X_train_i = np.array([])
	X_test_i = np.array([])
	y_train_i = np.array([])
	y_test_i = np.array([])

	if reducedInput:
		print "WARN: not using all available input"
		X_train_path = ["004_M3", "012_F2"]
		y_train_data = [1, 0]
	else:
		global X_train_path, y_train_data

	for p in X_train_path:
		a = root + "/" + p + "/C" + p + "_INDE"
		b = root + "/" + p + "/C" + p + "_SENT"
		filesThisFolder = np.append([a + "/" + f for f in listdir(a) if isfile(join(a, f)) and f.endswith(".wav")],
									[b + "/" + f for f in listdir(b) if isfile(join(b, f)) and f.endswith(".wav")])
		y_train_i = np.append(y_train_i, np.full((filesThisFolder.shape[0]), y_train_data[iterNum], dtype='int8'))
		X_train_i = np.append(X_train_i, filesThisFolder)
		iterNum += 1
	iterNum = 0
	for p in X_test_path:
		a = root + "/" + p + "/C" + p + "_INDE"
		b = root + "/" + p + "/C" + p + "_SENT"
		filesThisFolder = np.append([a + "/" + f for f in listdir(a) if isfile(join(a, f)) and f.endswith(".wav")],
									[b + "/" + f for f in listdir(b) if isfile(join(b, f)) and f.endswith(".wav")])
		y_test_i = np.append(y_test_i, np.full((filesThisFolder.shape[0]), y_test_data[iterNum], dtype='int8'))
		X_test_i = np.append(X_test_i, filesThisFolder)
		iterNum += 1
	return ((X_train_i, y_train_i), (X_test_i, y_test_i))
예제 #21
0
def run():
    train_images, train_labels, _, _, _ = load_data()
    train_images = train_images[:N_data, :]
    train_labels = train_labels[:N_data, :]
    batch_idxs = BatchList(N_data, batch_size)
    iter_per_epoch = len(batch_idxs)
    N_weights, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg)
    def indexed_loss_fun(w, idxs):
        return loss_fun(w, X=train_images[idxs], T=train_labels[idxs])

    log_alphas = np.full(N_iters, log_alpha_0)
    betas      = np.full(N_iters, beta_0)
    npr.seed(1)
    V0 = npr.randn(N_weights) * velocity_scale
    W0 = npr.randn(N_weights) * np.exp(log_param_scale)
    output = []
    for i in range(N_meta_iter):
        print "Meta iteration {0}".format(i)
        results = sgd(indexed_loss_fun, batch_idxs, N_iters,
                      W0, V0, np.exp(log_alphas), betas, record_learning_curve=True)
        learning_curve = results['learning_curve']
        d_log_alphas = np.exp(log_alphas) * results['d_alphas']
        output.append((learning_curve, log_alphas, d_log_alphas))
        log_alphas = log_alphas - meta_alpha * d_log_alphas

    return output
예제 #22
0
    def test_many_inputs(self):
        """
        Test adding NumericalExpressions with >10 inputs.
        """
        # Create an initial NumericalExpression by adding two factors together.
        f = self.f
        expr = f + f

        self.fake_raw_data = {f: full((5, 5), 0, float)}
        expected = 0

        # Alternate between adding and subtracting factors. Because subtraction
        # is not commutative, this ensures that we are combining factors in the
        # correct order.
        ops = (add, sub)

        for i, name in enumerate(ascii_uppercase):
            op = ops[i % 2]
            NewFactor = type(
                name,
                (Factor,),
                dict(dtype=float64_dtype, inputs=(), window_length=0),
            )
            new_factor = NewFactor()

            # Again we need a NumericalExpression, so add two factors together.
            new_expr = new_factor + new_factor
            self.fake_raw_data[new_factor] = full((5, 5), i + 1, float)
            expr = op(expr, new_expr)

            # Double the expected output since each factor is counted twice.
            expected = op(expected, (i + 1) * 2)

        self.check_output(expr, full((5, 5), expected, float))
예제 #23
0
    def make_default_configuration(self):
        self.global_register = ccpdv4['CCPD_GLOBAL'].copy()
        self.pixel_register = {
            "threshold": np.full((48, 12), 7, dtype=np.uint8),  # 16 columns (triple col) x 6 rows (double row)
#             "monitor": value = np.full((48,12), 0, dtype=np.uint8),
            "injection": np.full((6, ), 0, dtype=np.uint8)
        }
예제 #24
0
    def test_rolling_and_nonrolling(self):
        open_ = USEquityPricing.open
        close = USEquityPricing.close
        volume = USEquityPricing.volume

        # Test for thirty days up to the last day that we think all
        # the assets existed.
        dates_to_test = self.dates[-30:]

        constants = {open_: 1, close: 2, volume: 3}
        loader = PrecomputedLoader(constants=constants, dates=self.dates, sids=self.asset_ids)
        engine = SimplePipelineEngine(lambda column: loader, self.dates, self.asset_finder)

        sumdiff = RollingSumDifference()

        result = engine.run_pipeline(
            Pipeline(
                columns={"sumdiff": sumdiff, "open": open_.latest, "close": close.latest, "volume": volume.latest}
            ),
            dates_to_test[0],
            dates_to_test[-1],
        )
        self.assertIsNotNone(result)
        self.assertEqual({"sumdiff", "open", "close", "volume"}, set(result.columns))

        result_index = self.asset_ids * len(dates_to_test)
        result_shape = (len(result_index),)
        check_arrays(result["sumdiff"], Series(index=result_index, data=full(result_shape, -3, dtype=float)))

        for name, const in [("open", 1), ("close", 2), ("volume", 3)]:
            check_arrays(result[name], Series(index=result_index, data=full(result_shape, const, dtype=float)))
예제 #25
0
  def _recognize3(self, scores, transitions):
    lengthT = scores.shape[0]
    lengthS = transitions.shape[1]

    cost = np.full((lengthT, lengthT), np.inf, 'float32')
    back = np.full((lengthT, lengthT), np.inf, 'int32')

    cost[0] = np.min(scores[0])
    back[0] = -1

    transcript = []
    attention = []

    for s in xrange(1, lengthT):
      for t in xrange(min(s * lengthS, lengthT)):
        #if s % self.nstates == 0: # end state

        cost[s, t] = np.min(scores[s])
        q = transitions[t].copy()
        q[:min(t,lengthS)] += cost[s - 1, t - min(t,lengthS) : t]
        back[s, t] = q.argmin() + 1
        cost[s, t] += q.min()

    t = lengthT - 1
    s = 1
    while t >= 0 and s < lengthT:
      if s % self.nstates == 0:
        attention.append(t)
        transcript.append(scores[t].argmin()  / self.nstates)
      t -= back[-s, t]
      s += 1
    return transcript[::-1], attention[::-1]
예제 #26
0
def run():
    train_images, train_labels, _, _, _ = load_data(normalize=True)
    train_images = train_images[:N_real_data, :]
    train_labels = train_labels[:N_real_data, :]
    batch_idxs = BatchList(N_fake_data, batch_size)
    parser, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg, return_parser=True)
    N_weights = parser.N

    fake_data = npr.randn(*(train_images[:N_fake_data, :].shape)) * init_fake_data_scale
    fake_labels = one_hot(np.array(range(N_fake_data)) % N_classes, N_classes)  # One of each.

    def indexed_loss_fun(x, meta_params, idxs):   # To be optimized by SGD.
        return loss_fun(x, X=meta_params[idxs], T=fake_labels[idxs])
    def meta_loss_fun(x):                         # To be optimized in the outer loop.
        return loss_fun(x, X=train_images, T=train_labels)
    log_alphas = np.full(N_iters, log_alpha_0)
    betas      = np.full(N_iters, beta_0)
    npr.seed(0)
    v0 = npr.randn(N_weights) * velocity_scale
    x0 = npr.randn(N_weights) * np.exp(log_param_scale)

    output = []
    for i in range(N_meta_iter):
        results = sgd2(indexed_loss_fun, meta_loss_fun, batch_idxs, N_iters,
                       x0, v0, np.exp(log_alphas), betas, fake_data)
        learning_curve = results['learning_curve']
        validation_loss = results['M_final']
        output.append((learning_curve, validation_loss, fake_data))
        fake_data -= results['dMd_meta'] * data_stepsize   # Update data with one gradient step.
        print "Meta iteration {0} Valiation loss {1}".format(i, validation_loss)
    return output
예제 #27
0
def load_model(nbins_sfh=7,sigma=0.3,df=2.,agelims=None,objname=None, **extras):

    # we'll need this to access specific model parameters
    n = [p['name'] for p in model_params]

    # replace nbins_sfh
    nbins_sfh = 4 + (int(objname)-1) / 9

    # create SFH bins
    zred = model_params[n.index('zred')]['init']
    tuniv = WMAP9.age(zred).value

    # now construct the nonparametric SFH
    # current scheme: six bins, four spaced equally in logarithmic 
    # last bin is 15% age of the Universe, first two are 0-30, 30-100
    tbinmax = (tuniv*0.85)*1e9
    agelims = agelims[:2] + np.linspace(agelims[2],np.log10(tbinmax),nbins_sfh-2).tolist() + [np.log10(tuniv*1e9)]
    agebins = np.array([agelims[:-1], agelims[1:]])

    # load nvariables and agebins
    model_params[n.index('agebins')]['N'] = nbins_sfh
    model_params[n.index('agebins')]['init'] = agebins.T
    model_params[n.index('mass')]['N'] = nbins_sfh
    model_params[n.index('logsfr_ratios')]['N'] = nbins_sfh-1
    model_params[n.index('logsfr_ratios')]['init'] = np.full(nbins_sfh-1,0.0) # constant SFH
    model_params[n.index('logsfr_ratios')]['prior'] = priors.StudentT(mean=np.full(nbins_sfh-1,0.0),
                                                                      scale=np.full(nbins_sfh-1,sigma),
                                                                      df=np.full(nbins_sfh-1,df))

    return sedmodel.SedModel(model_params)
예제 #28
0
def plot_risk(risk_local, risk_central, risk_dist, iters, name_file, label_x, label_y):
    size = iters.shape[0]

    # Create the data
    risk_local   = np.full(size, risk_local)
    risk_central = np.full(size, risk_central)
    iters        = np.array(iters)

    # Plot graphs
    sns.set_style("ticks")
    plt.figure()
    plt.rc('text', usetex = True)
    plt.rc('text.latex', unicode = True)
    tests = list(risk_dist.keys())
    with sns.color_palette("tab10", len(tests) + 2):
        for test in tests:
            label = "SVM distribuído com " + test
            plt.plot(iters, risk_dist[test], linewidth = 2, label = label)
        plt.plot(iters, risk_local,   linewidth = 2.2, linestyle = '-.', label = 'SVM Local')
        plt.plot(iters, risk_central, linewidth = 2.2, linestyle = '-.', label = 'SVM Central')
        plt.legend(loc = 'upper right')
        sns.despine()
        plt.xlabel(label_x)
        plt.ylabel(label_y)
        file = str(plots_path) + "/" + name_file + ".pdf"
        plt.savefig(file, transparent = True)
def gen(npslice, output_type, black_filename, imagenet_filename):
    sets = ['suzanne','tux', 'airplane']
    imagenet = []
    black = []
    for i, s in enumerate(sets):
        print("set", s)
        input_samples = db.load('../00_Blender/%s_s1.h5' % s)
        assert np.max(input_samples['images']) > 50
        size = input_samples['alphas'][npslice].shape[0]

        min, max = range_extend_min_max(input_samples['images'], input_samples['alphas'])
        print("range_extend: min", min, "max", max)
        range_extended_images = range_extend(input_samples['images'][npslice],
                                             input_samples['alphas'][npslice], min, max)
        range_extended_db = {
            'images': range_extended_images,
            'alphas': input_samples['alphas'][npslice],
            'params': input_samples['params'][npslice],
            'label': np.full((size), i, dtype='uint8'),
            'label_split': np.full((size, len(sets)), [x == i for x in range(len(sets))], dtype='uint8')
        }

        bg = bgp.ImageNet(output_type)

        newdb = generate_dataset(range_extended_db, bg.generator(), size)
        newdb['background_indexes'] = np.array(bg.provided_indexes)

        imagenet.append(newdb)
        black.append(range_extended_db)

    db.write(imagenet_filename, append(imagenet))
    db.write(black_filename, append(black))
def _expected_kid_and_std(real_imgs, gen_imgs, max_block_size=1024):
  n_r, dim = real_imgs.shape
  n_g = gen_imgs.shape[0]

  n_blocks = int(np.ceil(max(n_r, n_g) / max_block_size))

  sizes_r = np.full(n_blocks, n_r // n_blocks)
  to_patch = n_r - n_blocks * (n_r // n_blocks)
  if to_patch > 0:
    sizes_r[-to_patch:] += 1
  inds_r = np.r_[0, np.cumsum(sizes_r)]
  assert inds_r[-1] == n_r

  sizes_g = np.full(n_blocks, n_g // n_blocks)
  to_patch = n_g - n_blocks * (n_g // n_blocks)
  if to_patch > 0:
    sizes_g[-to_patch:] += 1
  inds_g = np.r_[0, np.cumsum(sizes_g)]
  assert inds_g[-1] == n_g

  ests = []
  for i in range(n_blocks):
    r = real_imgs[inds_r[i]:inds_r[i + 1]]
    g = gen_imgs[inds_g[i]:inds_g[i + 1]]

    k_rr = (np.dot(r, r.T) / dim + 1)**3
    k_rg = (np.dot(r, g.T) / dim + 1)**3
    k_gg = (np.dot(g, g.T) / dim + 1)**3
    ests.append(-2 * k_rg.mean() +
                k_rr[np.triu_indices_from(k_rr, k=1)].mean() +
                k_gg[np.triu_indices_from(k_gg, k=1)].mean())

  var = np.var(ests, ddof=1) if len(ests) > 1 else np.nan
  return np.mean(ests), np.sqrt(var / len(ests))
@author: Amarantine
"""
import numpy as np
import os 
import pickle
MeditationSessions=['par011_1','par020_2','par040_2','par050_2','par061_1',
                   'par071_1','par080_2','par090_2','par101_1','par111_1',
                   'par120_2','par130_2']


NoMeditationSessions=['par011_2','par020_1','par040_1','par050_1','par061_2',
                   'par071_2','par080_1','par090_1','par101_2','par111_2',
                   'par120_1','par130_1']

meta_roc_auc_mean=np.full((2,12,12),np.nan) # 2 sessions, 12 participatnts, 12 trials
for participant, location in enumerate(MeditationSessions):
    participant=participant
    path='E:\Biomedical.master\Data\\' + location
    os.chdir(path)
    print path
    with open ('analysisResults_NumOftrials', 'rb') as fp:
        analysisResults_NumOftrials = pickle.load(fp)
    meta_roc_auc_mean[1][participant][:]=analysisResults_NumOftrials['meta_roc_auc_mean']    
    
for participant, location in enumerate(NoMeditationSessions):
    participant=participant
    path='E:\Biomedical.master\Data\\' + location
    os.chdir(path)
    print path
    with open ('analysisResults_NumOftrials', 'rb') as fp:
예제 #32
0
data3 = {
    'column1' : [110, 111, 113, 115],
    'column2' : [200, 300, 400, 500],
    'column3' : [10, 20, 30, 40],
}

df3 = pd.DataFrame(data3, index=['p1', 'p2', 'p3', 'p4'])
print('การกำหนดข้อมูลแบบดิกชันนารี')
print(df3)


data4 = {
    'Mon' : np.arange(10, 14),
    'Tue' : np.random.randint(10, 100, 4),
    'Web' : np.full(4, 10)
}
df4 = pd.DataFrame(data4, index=['p1', 'p2', 'p3', 'p4'])
print('การกำหนดข้อมูลแบบดิกชันนารี โดยใช้ numpy')
print(df4)


### การ join dataframe คือการนำ  dataframe / dataframe มาต่อกัน
data5 = {
    'col 1' : [10, 20, 30],
    'col 2' : [40, 50, 60],
    'col 3' : [70, 80, 90]
} 
df5 = pd.DataFrame(data5, index=list('ABC'))

data6 = {
예제 #33
0
    def _extract(self):
        # Extract all trials...

        # Get all stim_sync events detected
        ttls = [raw.get_port_events(tr, 'BNC1') for tr in self.bpod_trials]

        # Report missing events
        n_missing = sum(len(pulses) != 3 for pulses in ttls)
        # Check if all stim syncs have failed to be detected
        if n_missing == len(ttls):
            _logger.error(f'{self.session_path}: Missing ALL BNC1 TTLs ({n_missing} trials)')
        elif n_missing > 0:  # Check if any stim_sync has failed be detected for every trial
            _logger.warning(f'{self.session_path}: Missing BNC1 TTLs on {n_missing} trial(s)')

        # Extract datasets common to trainingChoiceWorld
        training = [ContrastLR, FeedbackTimes, Intervals, GoCueTimes, StimOnTriggerTimes]
        out, _ = run_extractor_classes(training, session_path=self.session_path, save=False,
                                       bpod_trials=self.bpod_trials, settings=self.settings)

        # GoCueTriggerTimes is the same event as StimOnTriggerTimes
        out['goCueTrigger_times'] = out['stimOnTrigger_times'].copy()

        # StimCenterTrigger times
        # Get the stim_on_state that triggers the onset of the stim
        stim_center_state = np.array([tr['behavior_data']['States timestamps']
                                      ['stim_center'][0] for tr in self.bpod_trials])
        out['stimCenterTrigger_times'] = stim_center_state[:, 0].T

        # StimCenter times
        stim_center_times = np.full(out['stimCenterTrigger_times'].shape, np.nan)
        for i, (sync, last) in enumerate(zip(ttls, out['stimCenterTrigger_times'])):
            """We expect there to be 3 pulses per trial; if this is the case, stim center will
            be the third pulse. If any pulses are missing, we can only be confident of the correct
            one if exactly one pulse occurs after the stim center trigger"""
            if len(sync) == 3 or (len(sync) > 0 and sum(pulse > last for pulse in sync) == 1):
                stim_center_times[i] = sync[-1]
        out['stimCenter_times'] = stim_center_times

        # StimOn times
        stimOn_times = np.full(out['stimOnTrigger_times'].shape, np.nan)
        for i, (sync, last) in enumerate(zip(ttls, out['stimCenterTrigger_times'])):
            """We expect there to be 3 pulses per trial; if this is the case, stim on will be the
            second pulse. If 1 pulse is missing, we can only be confident of the correct one if
            both pulses occur before the stim center trigger"""
            if len(sync) == 3 or (len(sync) == 2 and sum(pulse < last for pulse in sync) == 2):
                stimOn_times[i] = sync[1]
        out['stimOn_times'] = stimOn_times

        # RewardVolume
        trial_volume = [x['reward_amount'] for x in self.bpod_trials]
        out['rewardVolume'] = np.array(trial_volume).astype(np.float64)

        # StimOffTrigger times
        # StimOff occurs at trial start (ignore the first trial's state update)
        out['stimOffTrigger_times'] = np.array(
            [tr["behavior_data"]["States timestamps"]
             ["trial_start"][0][0] for tr in self.bpod_trials[1:]]
        )

        # StimOff times
        """
        There should be exactly three TTLs per trial.  stimOff_times should be the first TTL pulse.
        If 1 or more pulses are missing, we can not be confident of assigning the correct one.
        """
        trigg = out['stimOffTrigger_times']
        out['stimOff_times'] = np.array([sync[0] if len(sync) == 3 else np.nan
                                         for sync, off in zip(ttls[1:], trigg)])

        # FeedbackType is always positive
        out['feedbackType'] = np.ones(len(out['feedback_times']), dtype=np.int8)

        # ItiIn times
        out['itiIn_times'] = np.array(
            [tr["behavior_data"]["States timestamps"]
             ["iti"][0][0] for tr in self.bpod_trials]
        )

        # NB: We lose the last trial because the stim off event occurs at trial_num + 1
        n_trials = out['stimOff_times'].size
        return [out[k][:n_trials] for k in self.var_names]
예제 #34
0
파일: q3.py 프로젝트: locfree/cse547

#data=smallset
#n_user = 100
#n_artist = 100

data=realset
n_user = 1882
n_artist = 3000

sparity_ratio = data.size / ((n_user*n_artist) - data.size)

f=3
l=0.01

pref = np.full(data.size, 1.)
user_mat = sp.csr_matrix(np.full((n_user, f), 0.5))
artist_mat = sp.csr_matrix((n_artist, f))
lambda_identity = l*sp.identity(f)

p_artist = sp.csr_matrix((pref, (data['artistID'], data['userID'])), shape=(n_artist, n_user)).toarray()
p_user = p_artist.transpose()
sparity_vec = 1 + data['interactions']*sparity_ratio

artist_conf = []
artist_conf_sparse = []
idx = 0
while idx < n_artist:
    c_vec = np.full(n_user, 0.)
    filter = data['artistID']==idx
    for i in data[filter]:
def Question_2_essential(STDP_Mode,Inuput_Rate):
    # Question_1()
    # Units
        ms = 0.001
        mv = 0.001
        MO = 1.0e6
        nA = 1.0e-9
        nS = 1.0e-9
    # Neuron Param
        tau_m = 10*ms
        EL = -65*mv
        V_reset = -65*mv
        Vth = -50*mv
        Rm = 100*MO
        Ie = 0
        N = 40
        Recent_Post_Spike = -1000
    # Synapses
        tau_s = 2*ms
        gi = 4*nS
        ga = 2.08*nS
        Es = 0
        Deta_s = 0.5
        # 40 incoming synapses
        S = np.zeros(N)
        # g = np.full(N,gi)
        # g = np.full((1200005,N),0)
        # g[0,:] = gi
        Spikes_Counts = np.zeros(N)
        Recent_Pre_Spikes = np.zeros(N)
        Deta_t = 0.25*ms
        r_ini = Inuput_Rate #15 Hz
    # SDTP par
        A_plus = 0.2*nS
        A_minus = 0.25*nS
        tau_plus = 20*ms
        tau_minus = 20*ms
        # Presynaptic Spike time:
        # t_pre = 0
        # Postsynaptic Spike time:
        # t_post = 0 
        # Set the SDTP flag:
        # STDP_Mode = True
        # STDP_Mode = Flase
        if STDP_Mode:
            g = np.full(N,gi)
        else:
            g = np.full(N,ga)
        # print("The g_i is : ",g)
        # print("The S is : ",S)
        # Calculate the Time interval
        Runtime = 300
        Spike_Average_bin = 10
        TimeSteps = math.ceil(Runtime/Deta_t)
        print("TimeSteps: ", TimeSteps)
        V = []
        V_old = V_reset
        print("The g_i is : ",g)

        # Pre-synaptic neurons N:
        Spike_Count = 0
        Spike_Average = []
        g_average = []
        for t in range(0,TimeSteps):
            I_s = 0
            # Apply a possion process use random function
            # I_s = Rm*(Es - V_old)*np.sum(S*g).all()
            for it in range(0,40):
                I_s += Rm*(Es - V_old)*(S[it]*g[it])
            for i in range(0,40):
                rand = random.uniform(0,1)
                # Spike occures
                if rand < Deta_t*r_ini:
                    S[i] = S[i] + 0.5
                    # print("Pre_Spike")
                    # Spikes_Counts[i] += 1
                    if STDP_Mode:
                        # Store the Pre Spike time
                        Recent_Pre_Spikes[i] = t
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t <= 0:
                        g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
                        if g[i] < 0:
                            g[i] = 0
                # Spike not occures
                else:
                    S[i] = S[i]-S[i]*Deta_t/tau_s
            # Update The neuron:
            # Post-Synaptic neurons
            # I_s = Rm*(Es - V_old)*np.sum(S*g)
            V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
            # If there is a spike and update the V value
            if V_new > Vth:
                V_new = V_reset
                Recent_Post_Spike = t
                Spike_Count +=1 
                # print("Post Spike!!")
                if STDP_Mode:
                    for p in range(0,40):
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t > 0:
                        g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
                        if g[p] > (4*nS):
                            g[p] = 4*nS
            V_old = V_new
            V.append(V_new)
            if (t+1)%40000 == 0:
                Spike_Average.append(Spike_Count/Spike_Average_bin)
                # print("Spike_Average",Spike_Average)
                Spike_Count = 0
            if t > 800000:
                g_average.append(np.mean(g))
        g_average_all = (np.array(g_average)).mean()
        Spike_Average = np.array(Spike_Average)
        return g,Spike_Average,g_average_all
예제 #36
0
def main():
    parser = argparse.ArgumentParser(
        description='Worker script for the case study.')

    # Experiment settings
    descriptors = [
        'Bakery', 'Sour', 'Intensity', 'Sweet', 'Burnt', 'Pleasantness',
        'Fish', 'Fruit', 'Garlic', 'Spices', 'Cold', 'Acid', 'Warm', 'Musky',
        'Sweaty', 'Ammonia', 'Decayed', 'Wood', 'Grass', 'Flower', 'Chemical'
    ]
    parser.add_argument('--descriptor',
                        choices=descriptors,
                        default='Bakery',
                        help='The descriptor type to get p-values for.')
    parser.add_argument('--fdr_threshold',
                        type=float,
                        default=0.2,
                        help='Target false discovery rate.')
    parser.add_argument(
        '--importance_threshold',
        type=float,
        default=1e-3,
        help=
        'Minimum heuristic feature importance to make a feature test-worthy.')

    # Get the arguments from the command line
    args = parser.parse_args()
    dargs = vars(args)

    torch.set_num_threads(1)  # bad torch, no biscuit

    # Load the data and the model
    print('Loading data')
    X, Y, descriptors, target_features = load_olfaction()
    features = X.columns

    print('Loading model')
    # Get the model and data specifically for this descriptor class
    x, y, forest_model = load_or_fit_model(args.descriptor, X, Y)

    # Handle an idiosyncracy of multiprocessing with sklearn random forests
    for m in forest_model.models:
        m.n_jobs = 1

    # Get the weights to use in feature ranking
    model_weights = get_model_weights(forest_model)

    print('Total: {}'.format(len(X.columns)))

    all_p_path = 'data/p_{}.npy'.format(args.descriptor)
    if os.path.exists(all_p_path):
        p_values = np.load(all_p_path)
    else:
        p_values = np.full(len(features), np.nan)
    for feat_idx in range(len(features)):
        if not np.isnan(p_values[feat_idx]):
            continue

        p_path = 'data/p_{}_{}.npy'.format(args.descriptor, feat_idx)

        # Check if we have already computed this p-value
        if os.path.exists(p_path):
            p_values[feat_idx] = np.load(p_path)
            # print('p-value for {}: {}'.format(features[feat_idx], p_values[feat_idx]))
            continue

        # Check if the model assigns zero weight to this feature such that it can be ignored
        if np.abs(model_weights[feat_idx]) < args.importance_threshold:
            # print('p-value for {}: 1 (0 weight in model)'.format(features[feat_idx]))
            p_values[feat_idx] = 1
            continue

        print('************ Missing p-value for {} ************'.format(
            feat_idx))

    # Save the aggregated results
    np.save(all_p_path, p_values)

    # Print the top-ranked features by their heuristic weight
    for rank, (target_feature,
               importance) in enumerate(target_features[args.descriptor]):
        p_value = p_values[features.get_loc(target_feature)]
        print('{} & {:.4f} & {:.4f} \\\\'.format(
            target_feature.replace('\'', ''), importance, p_value))

    if np.any(np.isnan(p_values)):
        print('{} NaN p-values!'.format(np.isnan(p_values).sum()))
        missing = np.where(np.isnan(p_values))[0]
        print(missing)
        print([features[idx] for idx in missing])
        print('Setting to 1')
        p_values[np.isnan(p_values)] = 1

    # Only consider features with substantial heuristic feature importance
    important = model_weights >= args.importance_threshold
    p_important = p_values[important]

    print('{} features above importance threshold'.format(important.sum()))

    # Multiple testing correction via Benjamini-Hochberg at a 20% FDR
    # discoveries = bh(p_values, args.fdr_threshold) # Test all features
    discoveries = np.arange(len(features))[important][bh(
        p_important, args.fdr_threshold)]  # Test important features
    discovery_features = features[discoveries]
    discovery_p = p_values[discoveries]
    discovery_weights = model_weights[discoveries]

    # Print the discoveries along with their model weights and p-values
    order = np.argsort(np.abs(discovery_weights))[::-1]
    print('')
    print('Molecular Feature & Model Weight & $p$-value \\\\')
    for f, w, p in zip(discovery_features[order], discovery_weights[order],
                       discovery_p[order]):
        print('{} & {:.4f} & {:.4f} \\\\'.format(f, w, p))
def COMSM2127(STDP_Mode):
    # Pre-Setting
        B = 0 # 0Hz
    # Units
        ms = 0.001
        mv = 0.001
        MO = 1.0e6
        nA = 1.0e-9
        nS = 1.0e-9
    # Neuron Param
        tau_m = 10*ms
        EL = -65*mv
        V_reset = -65*mv
        Vth = -50*mv
        Rm = 100*MO
        Ie = 0
        N = 40
        Recent_Post_Spike = 0
    # Synapses
        tau_s = 2*ms
        gi = 4*nS
        ga = 2.08*nS
        Es = 0
        Deta_s = 0.5
        # 40 incoming synapses
        S = np.zeros(N)
        Spikes_Counts = np.zeros(N)
        Recent_Pre_Spikes = np.zeros(N)
        Deta_t = 0.25*ms
        # r_ini = Inuput_Rate #15 Hz
    # SDTP par
        A_plus = 0.2*nS
        A_minus = 0.25*nS
        tau_plus = 20*ms
        tau_minus = 20*ms
        if STDP_Mode:
            g = np.full(N,gi)
        else:
            g = np.full(N,gi)
        # Calculate the Time interval
        Runtime = 300
        TimeSteps = math.ceil(Runtime/Deta_t)
        print("TimeSteps: ", TimeSteps)
        V = []
        V_old = V_reset
        print("The g_i is : ",g)
        # Question4_Par
        freq = 10 # 10Hz
        r_0 = 15 # 20Hz
        # Pre-synaptic neurons N:
        Spike_Count = 0
        # example_pre_index = 17
        Pre_Neuron_Spikes_20s = []
        Pre_Neuron_Spikes_200s = []
        for example in range(0,N,1):
            Pre_Neuron_Spikes_20s.append([])
            Pre_Neuron_Spikes_200s.append([])
        Post_Neuron_Spikes_20s = []
        Post_Neuron_Spikes_200s = []
        for t in range(0,TimeSteps):
            I_s = 0
            # <r>(t) = <r>0 + B*sin(2*pai*frequency*t)
            r_ini = r_0 + B * math.sin(2*(math.pi)*freq*((t+1)*Deta_t))
            # Apply a possion process use random function
            # I_s = Rm*(Es - V_old)*np.sum(S*g).all()
            for it in range(0,40):
                I_s += Rm*(Es - V_old)*(S[it]*g[it])
            for i in range(0,40):
                rand = random.uniform(0,1)
                # Spike occures
                if rand < Deta_t*r_ini:
                    S[i] = S[i] + 0.5
                    # print("Pre_Spike")
                    # Spikes_Counts[i] += 1
                    if t <80000: #first 20 seconds
                        (Pre_Neuron_Spikes_20s[i]).append(t+1)
                    if t >799999: #last 100 seconds
                        (Pre_Neuron_Spikes_200s[i]).append(t+1)
                    if STDP_Mode:
                        # Store the Pre Spike time
                        Recent_Pre_Spikes[i] = t
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t <= 0:
                        g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
                        if g[i] < 0:
                            g[i] = 0
                # Spike not occures
                else:
                    S[i] = S[i]-S[i]*Deta_t/tau_s
            # Update The neuron:
        # Post-Synaptic neurons
            # I_s = Rm*(Es - V_old)*np.sum(S*g)
            V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
            # If there is a spike and update the V value
            if V_new > Vth:
                V_new = V_reset
                Recent_Post_Spike = t
                Spike_Count +=1
                if t < 80000:
                    Post_Neuron_Spikes_20s.append(t+1)
                if t > 799999:
                    Post_Neuron_Spikes_200s.append(t+1)
                # print("Post Spike!!")
                if STDP_Mode:
                    for p in range(0,40):
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t > 0:
                        g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
                        if g[p] > (4*nS):
                            g[p] = 4*nS
            V_old = V_new
            V.append(V_new)
        
            # x, y = np.random.randn(2, 100)
        diff_20s = []
        diff_200s = []
        for index in range(0,40,1):
            find = Pre_Neuron_Spikes_20s[index]
            for v in find:
                for k in Post_Neuron_Spikes_20s:
                    deta = (v-k)/4.00
                    if deta < 50 or deta>-50: 
                        diff_20s.append(int(deta))
        for index in range(0,40,1):
            find = Pre_Neuron_Spikes_200s[index]
            for v in find:
                for k in Post_Neuron_Spikes_200s:
                    deta = (v-k)/4.00
                    if deta < 50 or deta>-50: 
                        diff_200s.append(int(deta))
        region = []

        for index2 in range(-50,51,1):
            region.append(index2)
        
        hist20s, bins = np.histogram(diff_20s,region)
        hist200s, bins2 = np.histogram(diff_200s,region)
        # for v in Pre_Neuron_Spikes:
        #         for k in Post_Neuron_Spikes:
        #             diff.append(int(v-k))
        print(len(diff_20s))
        print("his: ",(hist20s))
        print("BIns: ",len(bins))
        X = range(-49,51,1)
        title = "Cross-correlogram (STDP: on)"
        plt.figure()
        plt.bar([i - 0.4 for i in X], (hist20s/N)/hist20s[50],width=0.4,color = 'r',label = "Begin 20s")
        plt.bar([i + 0.4 for i in X], (hist200s/N)/hist200s[50],width=0.4,color = 'g',label = "last 100s")
        plt.title(title)
        plt.legend()
        plt.ylabel("coefficient")
        plt.xlabel("time/ms")
        # plt.savefig('q4_cross_corr_off')



        # plt.show()

        # plt.title(title)
        # plt.legend()
        # plt.ylabel("coefficient")
        # plt.xlabel("time/ms")
        # plt.savefig('q4_cross_corr_off')
        plt.show()    
예제 #38
0
def generate_image_from_matrix(puzzle_matrix):
    def text(image, digit, row, column):
        text_size = cv2.getTextSize(str(digit), font, font_scale,
                                    font_thickness)
        text_width = text_size[0][0]
        text_height = text_size[0][1]
        location = (column * cell_size + border_size + int(text_width * 0.4),
                    row * cell_size + border_size + int(text_height * 1.4))
        cv2.putText(image, str(digit), location, font, font_scale, black,
                    font_thickness, cv2.LINE_AA)

    font_scale = 1.1
    font = cv2.FONT_HERSHEY_SIMPLEX
    font_thickness = 2
    cell_size = 40
    border_size = 40
    black = (0, 0, 0)
    light_gray = (240, 240, 240)
    dark_gray = (180, 180, 180)
    white = (255, 255, 255)
    line_width = 1
    image_dimension = cell_size * 9 + border_size * 2

    matrix_size = (image_dimension, image_dimension, 3)
    matrix_image = np.full(matrix_size, 255, dtype=np.uint8)

    cv2.rectangle(matrix_image, (border_size, border_size),
                  (cell_size * 9 + border_size, cell_size * 9 + border_size),
                  light_gray, -1)

    cv2.rectangle(matrix_image, (cell_size * 3 + border_size, border_size),
                  (cell_size * 6 + border_size, cell_size * 3 + border_size),
                  dark_gray, -1)
    cv2.rectangle(matrix_image, (border_size, cell_size * 3 + border_size),
                  (cell_size * 3 + border_size, cell_size * 6 + border_size),
                  dark_gray, -1)
    cv2.rectangle(matrix_image,
                  (cell_size * 6 + border_size, cell_size * 3 + border_size),
                  (cell_size * 9 + border_size, cell_size * 6 + border_size),
                  dark_gray, -1)
    cv2.rectangle(matrix_image,
                  (cell_size * 3 + border_size, cell_size * 6 + border_size),
                  (cell_size * 6 + border_size, cell_size * 9 + border_size),
                  dark_gray, -1)
    # add the vertical lines
    for i in range(0, 10):
        if i == 0:
            cv2.line(
                matrix_image, (i * cell_size + border_size, border_size),
                (i * cell_size + border_size, cell_size * 9 + border_size),
                black, line_width * 2)
        elif i == 9:
            cv2.line(matrix_image,
                     (i * cell_size - line_width + border_size, border_size),
                     (i * cell_size - line_width + border_size,
                      cell_size * 9 + border_size), black, line_width * 2)

        else:
            cv2.line(
                matrix_image, (i * cell_size + border_size, border_size),
                (i * cell_size + border_size, cell_size * 9 + border_size),
                black, line_width * 1)

    # add the horizontal lines
    for i in range(0, 10):
        if i == 0:
            cv2.line(
                matrix_image, (border_size, i * cell_size + border_size),
                (cell_size * 9 + border_size, i * cell_size + border_size),
                black, line_width * 2)
        elif i == 9:
            cv2.line(matrix_image,
                     (border_size, i * cell_size - line_width + border_size),
                     (cell_size * 9 + border_size,
                      i * cell_size - line_width + border_size), black,
                     line_width * 2)
        else:
            cv2.line(
                matrix_image, (border_size, i * cell_size + border_size),
                (cell_size * 9 + border_size, i * cell_size + border_size),
                black, line_width * 1)

    for row in range(len(puzzle_matrix)):
        for column in range(len(puzzle_matrix)):
            if puzzle_matrix[row][column] != 0:
                text(matrix_image, puzzle_matrix[row][column], row, column)

    return matrix_image
예제 #39
0
 def sess_fn(sess):
     #sess.run(tf.global_variables_initializer())
     return sess.run(
         [a, b],
         feed_dict={i: np.full((2, 3), 1.0)
                    for i in [x, y, z]})
def COMSM2127_2(STDP_Mode):
    # Pre-Setting
        B = 0 # 0Hz
    # Units
        ms = 0.001
        mv = 0.001
        MO = 1.0e6
        nA = 1.0e-9
        nS = 1.0e-9
    # Neuron Param
        tau_m = 10*ms
        EL = -65*mv
        V_reset = -65*mv
        Vth = -50*mv
        Rm = 100*MO
        Ie = 0
        N = 40
        Recent_Post_Spike = -1000
    # Synapses
        tau_s = 2*ms
        gi = 4*nS
        ga = 2.08*nS
        Es = 0
        Deta_s = 0.5
        # 40 incoming synapses
        S = np.zeros(N)
        Spikes_Counts = np.zeros(N)
        Recent_Pre_Spikes = np.zeros(N)
        Deta_t = 0.25*ms
        # r_ini = Inuput_Rate #15 Hz
    # SDTP par
        A_plus = 0.2*nS
        A_minus = 0.25*nS
        tau_plus = 20*ms
        tau_minus = 20*ms
        if STDP_Mode:
            g = np.full(N,gi)
        else:
            g = np.full(N,gi)
        # Calculate the Time interval
        Runtime = 300
        TimeSteps = math.ceil(Runtime/Deta_t)
        print("TimeSteps: ", TimeSteps)
        V = []
        V_old = V_reset
        print("The g_i is : ",g)
        # Question4_Par
        freq = 10 # 10Hz
        r_0 = 15 # 20Hz
        # Pre-synaptic neurons N:
        Spike_Count = 0
        # example_pre_index = 17
        Pre_Neuron_Spikes_20s = []
        Pre_Neuron_Spikes_200s = []
        for example in range(0,N,1):
            Pre_Neuron_Spikes_20s.append([])
            Pre_Neuron_Spikes_200s.append([])
        Post_Neuron_Spikes_20s = []
        Post_Neuron_Spikes_200s = []
        for t in range(0,TimeSteps):
            I_s = 0
            # <r>(t) = <r>0 + B*sin(2*pai*frequency*t)
            r_ini = r_0 + B * math.sin(2*(math.pi)*freq*((t+1)*Deta_t))
            # Apply a possion process use random function
            # I_s = Rm*(Es - V_old)*np.sum(S*g).all()
            for it in range(0,40):
                I_s += Rm*(Es - V_old)*(S[it]*g[it])
            for i in range(0,40):
                rand = random.uniform(0,1)
                # Spike occures
                if rand < Deta_t*r_ini:
                    S[i] = S[i] + 0.5
                    # print("Pre_Spike")
                    # Spikes_Counts[i] += 1
                    if t <80000: #first 20 seconds
                        (Pre_Neuron_Spikes_20s[i]).append(t/4)
                    if t >799999: #last 100 seconds
                        (Pre_Neuron_Spikes_200s[i]).append(t/4)
                    if STDP_Mode:
                        # Store the Pre Spike time
                        Recent_Pre_Spikes[i] = t
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t <= 0:
                        g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
                        if g[i] < 0:
                            g[i] = 0
                # Spike not occures
                else:
                    S[i] = S[i]-S[i]*Deta_t/tau_s
            # Update The neuron:
        # Post-Synaptic neurons
            # I_s = Rm*(Es - V_old)*np.sum(S*g)
            V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
            # If there is a spike and update the V value
            if V_new > Vth:
                V_new = V_reset
                Recent_Post_Spike = t
                Spike_Count +=1
                if t < 80000:
                    Post_Neuron_Spikes_20s.append(t/4)
                if t > 799999:
                    Post_Neuron_Spikes_200s.append(t/4)
                # print("Post Spike!!")
                if STDP_Mode:
                    for p in range(0,40):
                        STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
                        # if STDP_Deta_t > 0:
                        g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
                        if g[p] > (4*nS):
                            g[p] = 4*nS
            V_old = V_new
            V.append(V_new)
        
            # x, y = np.random.randn(2, 100)
        diff_20s = [0]*101
        diff_200s = [0]*101
        Post_Neuron_Spikes_20s = np.array(Post_Neuron_Spikes_20s)
        Post_Neuron_Spikes_200s =  np.array(Post_Neuron_Spikes_200s)
        X_axis = np.arange(-50,51,1)
        for index in range(0,40,1):
            for Target_Spike20 in Pre_Neuron_Spikes_20s[index]:
                for X_index in range(1,len(X_axis),1):
                    shift20 = Target_Spike20 - Post_Neuron_Spikes_20s
                    count = len(shift20[(shift20>(X_axis[X_index-1])) & (shift20<(X_axis[X_index]))])
                    diff_20s[X_index-1] +=count
        diff_20s = np.array(diff_20s)/N
        # plt.figure()
        # plt.bar(X_axis,diff_20s,width=0.4,color = 'r',label = "Begin 20s")
        # plt.show()

        X_axis = np.arange(-50,51,1)
        for index in range(0,40,1):
            for Target_Spike200 in Pre_Neuron_Spikes_200s[index]:
                for X_index in range(1,len(X_axis),1):
                    shift200 = Target_Spike200 - Post_Neuron_Spikes_200s
                    count = len(shift200[(shift200>(X_axis[X_index-1])) & (shift200<(X_axis[X_index]))])
                    diff_200s[X_index-1] +=count
        diff_200s = np.array(diff_200s)/N
        # plt.figure()
        # plt.bar(X_axis,diff_200s,width=0.4,color = 'r',label = "Last 100s")
        # plt.show()
        return diff_20s,diff_200s
def getDirectedSpread(visibility_matrix: np.ndarray, environment: np.ndarray, resolution: int) -> np.ndarray:
    """
    Parameters
    ----------
    visibility_matrix
        Holds all visibility maps corresponding to each positions in the environment
    environment
        Used for finding non zero element when initializing distanceMatrix
    directionToTable
        0 if table to the right of agent
        1 if table down of agent
        2 if table to the left of agent
        3 if table is up from the agent
    resolution
        Resolution parameter.
        Changes the resolution of the environmental map.

    Returns
    -------
    cone : np.ndarray
        Represent a cone shaped vision in the direction to the table. 
        Has values in [0,1].
        This matrix is later multiplied with the particle spread matrix corresponding 
        to current time step in order to decrease backward spread. 
    """
    
    #non_empty_pos = next(zip(*np.where(environment != TileType.WALL & environment != TileType.TABLE)))

    #local_width, local_height = visibility_matrix[non_empty_pos].shape
    
    found = False
    while found == False:
        for j in range(5,np.shape(environment)[0]):
            for i in range(5,np.shape(environment)[1]): 
                if environment[j,i] != 0:
                    distances = np.full(np.shape(visibility_matrix[int(j),int(i)]), 1, dtype=float)
                    found = True

    local_width, local_height = distances.shape
    
    
    assert(local_width > 0)
    assert(local_height > 0)
        
    radius = 4*resolution
    radii = np.linspace(0, radius, 30)
    thetas = np.linspace(-np.pi, 0, 40)
    cone = np.zeros((radius*2 + 1, radius*2 + 1), dtype=float)
    
    for theta in thetas:
            for rad in radii:
                # Center of visibility calculation in local coordinates.
                local_x = radius
                local_y = radius

                # 0.5 since radius does not take in account that indexing starts at 0
                
                dx = int(0.5 + np.cos(theta) * rad)
                dy = int(np.floor(0.5 + np.sin(theta) * rad)) # np.floor since int() rounds up negative numbers
                assert(-radius <= dx <= radius)
                assert(-radius <= dy <= radius)

                if cone[local_x + dx, local_y + dy] < 0.01:
                    if np.cos(theta) < -0.75:
                        cone[local_x + dx, local_y + dy] = 0.1
                    else:    
                        cone[local_x + dx - 1, local_y + dy] = 5/8 + (3/8 * np.cos(theta))
                        
                dx = int(0.5 + np.cos(-theta) * rad)
                dy = int(0.5 + np.sin(-theta) * rad)
                assert(-radius <= dx <= radius)
                assert(-radius <= dy <= radius)
                
                if cone[local_x + dx, local_y + dy] < 0.01:
                    if np.cos(-theta) < -0.75:
                        cone[local_x + dx, local_y + dy] = 0.1
                    else:    
                        cone[local_x + dx - 1, local_y + dy] = 5/8 + (3/8 * np.cos(-theta))
    
    
    cones = np.zeros((1, 4), dtype=object)
    cones[0,0] = cone
    cones[0,1] = np.rot90(cone)
    cones[0,2] = np.rot90(cone,2)
    cones[0,3] = np.rot90(cone,3)
    return cones
    
def Question_1():
        # PartB Spike-timing-dependent plasticity
    # Units
        ms = 0.001
        mv = 0.001
        MO = 1.0e6
        nA = 1.0e-9
        nS = 1.0e-9
    # Neuron Param
        tau_m = 10*ms
        EL = -65*mv
        V_reset = -65*mv
        Vth = -50*mv
        Rm = 100*MO
        Ie = 0
        N = 40
    # Synapses
        tau_s = 2*ms
        gi = 4*nS
        Es = 0
        Deta_s = 0.5
        # 40 incoming synapses
        S = np.zeros((1,40)) 
        g = np.full((1,40),gi)
        Deta_t = 0.25*ms
        r_ini = 15 #15 Hz
        # Euler's method 
        print("The g_i is : ",g)
        # Calculate the Time interval
        TimeSteps = math.ceil(1/Deta_t)
        # print("TimeSteps: ", TimeSteps)
        V = []
        V_old = V_reset
        Count = 0
        S0 = S
        for _ in range(0,TimeSteps):
            I_s = Rm*(Es - V_old)*np.sum(S0*gi)
            # print(I_s)
            V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
            Possiond = Possion(Deta_t,r_ini)
            # S = S0+S0*Deta_t/tau_s
            # Apply a possion process use random function
            for i in range(0,40):
                rand = random.uniform(0,1)
                if rand < Deta_t*r_ini:
                    S[0,i] = S0[0,i] + 0.5
                else:
                    S[0,i] = S0[0,i] - S0[0,i]*Deta_t/tau_s
            # If there is a spike and update the V value
            if V_new > Vth:
                V_new = V_reset
                Count+=1
            V_old = V_new
            S0 = S
            V.append(V_new)
        print("The fir rate is: ",Count)
        plt.figure()
        plt.plot(range(0,TimeSteps),V,'g')
        plt.title("40 synapses with fixed $\overline {g_i}$ in 1ms")
        plt.xlabel("Time/0.25 ms total 1s wiht 0.25ms intervals")
        plt.ylabel("Voltage/mv")
        plt.savefig('./coursework3/graphs/PartB_Question1.png')
예제 #43
0
def test_incremental_variance_numerical_stability():
    # Test Youngs and Cramer incremental variance formulas.

    def np_var(A):
        return A.var(axis=0)

    # Naive one pass variance computation - not numerically stable
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
    def one_pass_var(X):
        n = X.shape[0]
        exp_x2 = (X**2).sum(axis=0) / n
        expx_2 = (X.sum(axis=0) / n)**2
        return exp_x2 - expx_2

    # Two-pass algorithm, stable.
    # We use it as a benchmark. It is not an online algorithm
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm
    def two_pass_var(X):
        mean = X.mean(axis=0)
        Y = X.copy()
        return np.mean((Y - mean)**2, axis=0)

    # Naive online implementation
    # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
    # This works only for chunks for size 1
    def naive_mean_variance_update(x, last_mean, last_variance,
                                   last_sample_count):
        updated_sample_count = (last_sample_count + 1)
        samples_ratio = last_sample_count / float(updated_sample_count)
        updated_mean = x / updated_sample_count + last_mean * samples_ratio
        updated_variance = last_variance * samples_ratio + \
            (x - last_mean) * (x - updated_mean) / updated_sample_count
        return updated_mean, updated_variance, updated_sample_count

    # We want to show a case when one_pass_var has error > 1e-3 while
    # _batch_mean_variance_update has less.
    tol = 200
    n_features = 2
    n_samples = 10000
    x1 = np.array(1e8, dtype=np.float64)
    x2 = np.log(1e-5, dtype=np.float64)
    A0 = np.full((n_samples // 2, n_features), x1, dtype=np.float64)
    A1 = np.full((n_samples // 2, n_features), x2, dtype=np.float64)
    A = np.vstack((A0, A1))

    # Older versions of numpy have different precision
    # In some old version, np.var is not stable
    if np.abs(np_var(A) - two_pass_var(A)).max() < 1e-6:
        stable_var = np_var
    else:
        stable_var = two_pass_var

    # Naive one pass var: >tol (=1063)
    assert_greater(np.abs(stable_var(A) - one_pass_var(A)).max(), tol)

    # Starting point for online algorithms: after A0

    # Naive implementation: >tol (436)
    mean, var, n = A0[0, :], np.zeros(n_features), n_samples // 2
    for i in range(A1.shape[0]):
        mean, var, n = \
            naive_mean_variance_update(A1[i, :], mean, var, n)
    assert_equal(n, A.shape[0])
    # the mean is also slightly unstable
    assert_greater(np.abs(A.mean(axis=0) - mean).max(), 1e-6)
    assert_greater(np.abs(stable_var(A) - var).max(), tol)

    # Robust implementation: <tol (177)
    mean, var = A0[0, :], np.zeros(n_features)
    n = np.full(n_features, n_samples // 2, dtype=np.int32)
    for i in range(A1.shape[0]):
        mean, var, n = \
            _incremental_mean_and_var(A1[i, :].reshape((1, A1.shape[1])),
                                      mean, var, n)
    assert_array_equal(n, A.shape[0])
    assert_array_almost_equal(A.mean(axis=0), mean)
    assert_greater(tol, np.abs(stable_var(A) - var).max())
예제 #44
0
    def check_with_place(self, place):
        scope = core.Scope()
        # create and initialize Grad Variable
        height = 10
        rows = [0, 4, 7]
        row_numel = 12
        mu = 1.0
        use_nesterov = self.use_nesterov
        regularization_method = self.regularization_method
        regularization_coeff = self.regularization_coeff

        # create and initialize Param Variable
        param_array = np.full((height, row_numel), 5.0).astype("float32")
        param_out_array = np.full((height, row_numel), 0.0).astype("float32")

        param = scope.var('Param').get_tensor()
        param.set(param_array.astype("float16"), place)
        param_out = scope.var("ParamOut").get_tensor()
        param_out.set(param_out_array.astype("float16"), place)

        master_param = scope.var('MasterParam').get_tensor()
        master_param.set(param_array, place)
        master_param_out = scope.var("MasterParamOut").get_tensor()
        master_param_out.set(param_out_array, place)

        grad_selected_rows = scope.var('Grad').get_selected_rows()
        grad_selected_rows.set_height(height)
        grad_selected_rows.set_rows(rows)
        grad_np_array = np.ones((len(rows), row_numel)).astype("float32")
        grad_np_array[0, 0] = 2.0
        grad_np_array[2, 8] = 4.0
        grad_tensor = grad_selected_rows.get_tensor()
        grad_tensor.set(grad_np_array.astype("float16"), place)

        velocity = scope.var('Velocity').get_tensor()
        velocity_np_array = np.ones((height, row_numel)).astype("float32")
        velocity.set(velocity_np_array, place)
        velocity_out = scope.var('VelocityOut').get_tensor()
        velocity_out_np_array = np.full((height, row_numel),
                                        0.0).astype("float32")
        velocity_out.set(velocity_out_np_array, place)

        # create and initialize LearningRate Variable
        lr = scope.var('LearningRate').get_tensor()
        lr_array = np.full((1), 2.0).astype("float32")
        lr.set(lr_array, place)

        # create and run operator
        op = Operator(
            "momentum",
            Param='Param',
            Grad='Grad',
            Velocity='Velocity',
            MasterParam='MasterParam',
            ParamOut='ParamOut',
            VelocityOut='VelocityOut',
            MasterParamOut='MasterParamOut',
            LearningRate='LearningRate',
            mu=mu,
            use_nesterov=use_nesterov,
            regularization_method=regularization_method,
            regularization_coeff=regularization_coeff,
            multi_precision=True,
            rescale_grad=1.0)
        op.run(scope, place)

        # get and compare result
        param_out_np_array = np.array(param_out)
        velocity_out_np_array = np.array(velocity_out)

        _grad_np_array = np.full((height, row_numel), 0.0).astype("float32")
        for i in range(len(rows)):
            _grad_np_array[rows[i]] = grad_np_array[i]

        _param = param_array

        _param_out, _velocity_out = calculate_momentum_by_numpy(
            param=_param,
            grad=_grad_np_array,
            mu=mu,
            velocity=velocity_np_array,
            use_nesterov=use_nesterov,
            learning_rate=lr_array,
            regularization_method=regularization_method,
            regularization_coeff=regularization_coeff)

        self.assertTrue((_velocity_out == velocity_out_np_array).all())
        self.assertTrue((_param_out == param_out_np_array).all())
예제 #45
0
    def get_matrix_data(
            self, label, buffer_manager, region, placements, graph_mapper,
            application_vertex, variable, n_machine_time_steps):
        """ Read a uint32 mapped to time and neuron IDs from the SpiNNaker\
            machine.

        :param label: vertex label
        :param buffer_manager: the manager for buffered data
        :param region: the DSG region ID used for this data
        :param placements: the placements object
        :param graph_mapper: \
            the mapping between application and machine vertices
        :param application_vertex:
        :param variable: PyNN name for the variable (V, gsy_inh etc.)
        :type variable: str
        :param n_machine_time_steps:
        :return:
        """
        if variable == SPIKES:
            msg = "Variable {} is not supported use get_spikes".format(SPIKES)
            raise ConfigurationException(msg)
        vertices = graph_mapper.get_machine_vertices(application_vertex)
        progress = ProgressBar(
            vertices, "Getting {} for {}".format(variable, label))
        sampling_rate = self._sampling_rates[variable]
        expected_rows = int(math.ceil(
            n_machine_time_steps / sampling_rate))
        missing_str = ""
        data = None
        indexes = []
        for vertex in progress.over(vertices):
            placement = placements.get_placement_of_vertex(vertex)
            vertex_slice = graph_mapper.get_slice(vertex)
            neurons = self._neurons_recording(variable, vertex_slice)
            n_neurons = len(neurons)
            if n_neurons == 0:
                continue
            indexes.extend(neurons)
            # for buffering output info is taken form the buffer manager
            neuron_param_region_data_pointer, missing_data = \
                buffer_manager.get_data_for_vertex(
                    placement, region)
            record_raw = neuron_param_region_data_pointer.read_all()
            record_length = len(record_raw)

            row_length = self.N_BYTES_FOR_TIMESTAMP + \
                n_neurons * self.N_BYTES_PER_VALUE

            # There is one column for time and one for each neuron recording
            n_rows = record_length // row_length
            # Converts bytes to ints and make a matrix
            record = (numpy.asarray(record_raw, dtype="uint8").
                      view(dtype="<i4")).reshape((n_rows, (n_neurons + 1)))
            # Check if you have the expected data
            if not missing_data and n_rows == expected_rows:
                # Just cut the timestamps off to get the fragment
                fragment = (record[:, 1:] / float(DataType.S1615.scale))
            else:
                missing_str += "({}, {}, {}); ".format(
                    placement.x, placement.y, placement.p)
                # Start the fragment for this slice empty
                fragment = numpy.empty((expected_rows, n_neurons))
                for i in xrange(0, expected_rows):
                    time = i * sampling_rate
                    # Check if there is data for this timestep
                    local_indexes = numpy.where(record[:, 0] == time)
                    if len(local_indexes[0]) > 0:
                        # Set row to data for that timestep
                        fragment[i] = (record[local_indexes[0], 1:] /
                                       float(DataType.S1615.scale))
                    else:
                        # Set row to nan
                        fragment[i] = numpy.full(n_neurons, numpy.nan)
            if data is None:
                data = fragment
            else:
                # Add the slice fragment on axis 1 which is IDs/channel_index
                data = numpy.append(data, fragment, axis=1)
        if len(missing_str) > 0:
            logger.warn(
                "Population {} is missing recorded data in region {} from the"
                " following cores: {}".format(label, region, missing_str))
        sampling_interval = self.get_neuron_sampling_interval(variable)
        return (data, indexes, sampling_interval)
예제 #46
0
def Stochastic_Process(j):
    Profile, num_profiles = Initialise_model()
    peak_enlarg, mu_peak, s_peak, Year_behaviour, User_list = Initialise_inputs(j)
    '''
    Calculation of the peak time range, which is used to discriminate between off-peak and on-peak coincident switch-on probability
    Calculates first the overall Peak Window (taking into account all User classes). 
    The peak window is just a time window in which coincident switch-on of multiple appliances assumes a higher probability than off-peak
    Within the peak window, a random peak time is calculated and then enlarged into a peak_time_range following again a random procedure
    '''
    windows_curve = np.zeros(1440) #creates an empty daily profile
    Tot_curve = np.zeros(1440) #creates another empty daily profile
    for Us in User_list:
        App_count = 0
        for App in Us.App_list:
            #Calculate windows curve, i.e. the theoretical maximum curve that can be obtained, for each app, by switching-on always all the 'n' apps altogether in any time-step of the functioning windows
            single_wcurve = Us.App_list[App_count].daily_use*np.mean(Us.App_list[App_count].POWER)*Us.App_list[App_count].number #this computes the curve for the specific App
            windows_curve = np.vstack([windows_curve, single_wcurve]) #this stacks the specific App curve in an overall curve comprising all the Apps within a User class
            App_count += 1
        Us.windows_curve = windows_curve #after having iterated for all the Apps within a User class, saves the overall User class theoretical maximum curve
        Us.windows_curve = np.transpose(np.sum(Us.windows_curve, axis = 0))*Us.num_users
        Tot_curve = Tot_curve + Us.windows_curve #adds the User's theoretical max profile to the total theoretical max comprising all classes
    peak_window = np.transpose(np.argwhere(Tot_curve == np.amax(Tot_curve))) #Find the peak window within the theoretical max profile
    peak_time = round(random.normalvariate(round(np.average(peak_window)),1/3*(peak_window[0,-1]-peak_window[0,0]))) #Within the peak_window, randomly calculate the peak_time using a gaussian distribution
    peak_time_range = np.arange((peak_time-round(math.fabs(peak_time-(random.gauss(peak_time,(peak_enlarg*peak_time)))))),(peak_time+round(math.fabs(peak_time-random.gauss(peak_time,(peak_enlarg*peak_time)))))) #the peak_time is randomly enlarged based on the calibration parameter peak_enlarg
    
    '''
    The core stochastic process starts here. For each profile requested by the software user, 
    each Appliance instance within each User instance is separately and stochastically generated
    '''
    for prof_i in range(num_profiles): #the whole code is repeated for each profile that needs to be generated
        Tot_Classes = np.zeros(1440) #initialise an empty daily profile that will be filled with the sum of the hourly profiles of each User instance
        for Us in User_list: #iterates for each User instance (i.e. for each user class)
            Us.load = np.zeros(1440) #initialise empty load for User instance
            for i in range(Us.num_users): #iterates for every single user within a User class. Each single user has its own separate randomisation
                if Us.user_preference == 0:
                    rand_daily_pref = 0
                    pass
                else:
                    rand_daily_pref = random.randint(1,Us.user_preference)
                for App in Us.App_list: #iterates for all the App types in the given User class
                    #initialises variables for the cycle
                    tot_time = 0
                    App.daily_use = np.zeros(1440)
                    if random.uniform(0,1) > App.occasional_use: #evaluates if occasional use happens or not
                        continue
                    else:
                        pass
                    
                    if App.Pref_index == 0:
                        pass
                    else:
                        if rand_daily_pref == App.Pref_index: #evaluates if daily preference coincides with the randomised daily preference number
                            pass
                        else:
                            continue
                    if App.wd_we == Year_behaviour[prof_i] or App.wd_we == 2 : #checks if the app is allowed in the given yearly behaviour pattern
                        pass
                    else:
                        continue

                    #recalculate windows start and ending times randomly, based on the inputs
                    rand_window_1 = np.array([int(random.uniform((App.window_1[0]-App.random_var_1),(App.window_1[0]+App.random_var_1))),int(random.uniform((App.window_1[1]-App.random_var_1),(App.window_1[1]+App.random_var_1)))])
                    if rand_window_1[0] < 0:
                        rand_window_1[0] = 0
                    if rand_window_1[1] > 1440:
                        rand_window_1[1] = 1440
    
                    rand_window_2 = np.array([int(random.uniform((App.window_2[0]-App.random_var_2),(App.window_2[0]+App.random_var_2))),int(random.uniform((App.window_2[1]-App.random_var_2),(App.window_2[1]+App.random_var_2)))])
                    if rand_window_2[0] < 0:
                        rand_window_2[0] = 0
                    if rand_window_2[1] > 1440:
                        rand_window_2[1] = 1440
                            
                    rand_window_3 = np.array([int(random.uniform((App.window_3[0]-App.random_var_3),(App.window_3[0]+App.random_var_3))),int(random.uniform((App.window_3[1]-App.random_var_3),(App.window_3[1]+App.random_var_3)))])
                    if rand_window_3[0] < 0:
                        rand_window_3[0] = 0
                    if rand_window_3[1] > 1440:
                        rand_window_3[1] = 1440
    
                    #redefines functioning windows based on the previous randomisation of the boundaries
                    if App.flat == 'yes': #if the app is "flat" the code stops right after filling the newly created windows without applying any further stochasticity
                        App.daily_use[rand_window_1[0]:rand_window_1[1]] = np.full(np.diff(rand_window_1),App.POWER[prof_i]*App.number)
                        App.daily_use[rand_window_2[0]:rand_window_2[1]] = np.full(np.diff(rand_window_2),App.POWER[prof_i]*App.number)
                        App.daily_use[rand_window_3[0]:rand_window_3[1]] = np.full(np.diff(rand_window_3),App.POWER[prof_i]*App.number)
                        Us.load = Us.load + App.daily_use
                        continue
                    else: #otherwise, for "non-flat" apps it puts a mask on the newly defined windows and continues    
                        App.daily_use[rand_window_1[0]:rand_window_1[1]] = np.full(np.diff(rand_window_1),0.001)
                        App.daily_use[rand_window_2[0]:rand_window_2[1]] = np.full(np.diff(rand_window_2),0.001)
                        App.daily_use[rand_window_3[0]:rand_window_3[1]] = np.full(np.diff(rand_window_3),0.001)
                    App.daily_use_masked = np.zeros_like(ma.masked_not_equal(App.daily_use,0.001))
                  
                    App.power = App.POWER[prof_i]
                    
                    #random variability is applied to the total functioning time and to the duration of the duty cycles, if they have been specified
                    random_var_t = random.uniform((1-App.r_t),(1+App.r_t))
                    if App.activate == 1:
                        App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        random_cycle1 = np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))) #randomise also the fixed cycle
                        random_cycle2 = random_cycle1
                        random_cycle3 = random_cycle1
                    elif App.activate == 2:
                        App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_21 = App.P_21*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_22 = App.P_22*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        random_cycle1 = np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))) #randomise also the fixed cycle
                        random_cycle2 = np.concatenate(((np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21),(np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22))) #randomise also the fixed cycle
                        random_cycle3 = random_cycle1
                    elif App.activate == 3:
                        App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_21 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_22 = App.P_22*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_31 = App.P_31*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        App.p_32 = App.P_32*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
                        random_cycle1 = random.choice([np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))),np.concatenate(((np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12),(np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11)))]) #randomise also the fixed cycle
                        random_cycle2 = random.choice([np.concatenate(((np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21),(np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22))),np.concatenate(((np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22),(np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21)))])                    
                        random_cycle3 = random.choice([np.concatenate(((np.ones(int(App.t_31*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_31),(np.ones(int(App.t_32*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_32))),np.concatenate(((np.ones(int(App.t_32*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_32),(np.ones(int(App.t_31*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_31)))])#this is to avoid that all cycles are sincronous                      
                    else:
                        pass
                    rand_time = round(random.uniform(App.func_time,int(App.func_time*random_var_t)))
                    #control to check that the total randomised time of use does not exceed the total space available in the windows
                    if rand_time > 0.99*(np.diff(rand_window_1)+np.diff(rand_window_2)+np.diff(rand_window_3)):
                        rand_time = int(0.99*(np.diff(rand_window_1)+np.diff(rand_window_2)+np.diff(rand_window_3)))
                    max_free_spot = rand_time #free spots are used to detect if there's still space for switch_ons. Before calculating actual free spots, the max free spot is set equal to the entire randomised func_time
                           
                    while tot_time <= rand_time: #this is the key cycle, which runs for each App until the switch_ons and their duration equals the randomised total time of use of the App
                            #check how many windows to consider
                            if App.num_windows == 1:
                                switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1]))]))
                            elif App.num_windows == 2:
                                switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1])),random.uniform(rand_window_2[0],(rand_window_2[1]))]))
                            else: 
                                switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1])),random.uniform(rand_window_2[0],(rand_window_2[1])),random.uniform(rand_window_3[0],(rand_window_3[1]))]))
                            #Identifies a random switch on time within the available functioning windows
                            if App.daily_use[switch_on] == 0.001: #control to check if the app is not already on at the randomly selected switch-on time
                                if switch_on in range(rand_window_1[0],rand_window_1[1]):
                                    if np.any(App.daily_use[switch_on:rand_window_1[1]]!=0.001): #control to check if there are any other switch on times after the current one    
                                        next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)] #identifies the position of next switch on time and sets it as a limit for the duration of the current switch on
                                        if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
                                            upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_1[1]-switch_on))
                                        elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle: #if next switch_on event does not allow for a minimum functioning cycle without overlapping, but there are other larger free spots, the cycle tries again from the beginning
                                            continue
                                        else:
                                            upper_limit = next_switch[0]-switch_on #if there are no other options to reach the total time of use, empty spaces are filled without minimum cycle restrictions until reaching the limit                                              
                                    else:
                                        upper_limit = min(rand_time,rand_window_1[1]-switch_on) #if there are no other switch-on events after the current one, the upper duration limit is set this way
                                    
                                    if upper_limit >= App.func_cycle: #if the upper limit is higher than minimum functioning time, an array of indexes is created to be later put in the profile
                                        indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit)))) #a random duration is chosen between the upper limit and the minimum cycle
                                    else:
                                        indexes = np.arange(switch_on,switch_on+upper_limit) #this is the case in which empty spaces need to be filled without constraints to reach the total time goal
                                        
                                elif switch_on in range(rand_window_2[0],rand_window_2[1]): #if random switch_on happens in windows2, same code as above is repeated for windows2
                                    if np.any(App.daily_use[switch_on:rand_window_2[1]]!=0.001):
                                        next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)]
                                        if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
                                            upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_2[1]-switch_on))
                                        elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle:
                                            continue
                                        else:
                                            upper_limit = next_switch[0]-switch_on
                                    
                                    else:
                                        upper_limit = min(rand_time,rand_window_2[1]-switch_on)
                                    
                                    if upper_limit >= App.func_cycle:
                                        indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit))))
                                    else:    
                                        indexes = np.arange(switch_on,switch_on+upper_limit)
                                        
                                else: #if switch_on is not in window1 nor in window2, it shall be in window3. Same code is repreated
                                    if np.any(App.daily_use[switch_on:rand_window_3[1]]!=0.001):
                                        next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)]
                                        if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
                                            upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_3[1]-switch_on))
                                        elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle:
                                            continue
                                        else:
                                            upper_limit = next_switch[0]-switch_on
                                    
                                    else:
                                        upper_limit = min(rand_time,rand_window_3[1]-switch_on)
                                    
                                    if upper_limit >= App.func_cycle:
                                        indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit))))
                                    else:    
                                        indexes = np.arange(switch_on,switch_on+upper_limit)
                                        
                                tot_time = tot_time + indexes.size #the count of total time is updated with the size of the indexes array
                                
                                if tot_time > rand_time: #control to check when the total functioning time is reached. It will be typically overcome, so a correction is applied to avoid this
                                    indexes_adj = indexes[:-(tot_time-rand_time)] #correctes indexes size to avoid overcoming total time
                                    if np.in1d(peak_time_range,indexes_adj).any() and App.fixed == 'no': #check if indexes are in peak window and if the coincident behaviour is locked by the "fixed" attribute
                                        coincidence = min(App.number,max(1,math.ceil(random.gauss(math.ceil(App.number*mu_peak),(s_peak*App.number*mu_peak))))) #calculates coincident behaviour within the peak time range
                                    elif np.in1d(peak_time_range,indexes_adj).any()== False and App.fixed == 'no': #check if indexes are off-peak and if coincident behaviour is locked or not
                                        Prob = random.uniform(0,(App.number-1)/App.number) #calculates probability of coincident switch_ons off-peak
                                        array = np.arange(0,App.number)/App.number
                                        try:
                                            on_number = np.max(np.where(Prob>=array))+1
                                        except ValueError:
                                            on_number = 1 
                                        coincidence = on_number #randomly selects how many apps are on at the same time for each app type based on the above probabilistic algorithm
                                    else:
                                        coincidence = App.number #this is the case when App.fixed is activated. All 'n' apps of an App instance are switched_on altogether
                                    if App.activate > 0: #evaluates if the app has some duty cycles to be considered
                                        if indexes_adj.size > 0:
                                            evaluate = round(np.mean(indexes_adj)) #calculates the mean time position of the current switch_on event, to later select the proper duty cycle
                                        else:
                                            evaluate = 0 
                                        #based on the evaluate value, selects the proper duty cycle and puts the corresponding power values in the indexes range
                                        if evaluate in range(App.cw11[0],App.cw11[1]) or evaluate in range(App.cw12[0],App.cw12[1]):
                                            np.put(App.daily_use,indexes_adj,(random_cycle1*coincidence))
                                            np.put(App.daily_use_masked,indexes_adj,(random_cycle1*coincidence),mode='clip')
                                        elif evaluate in range(App.cw21[0],App.cw21[1]) or evaluate in range(App.cw22[0],App.cw22[1]):
                                            np.put(App.daily_use,indexes_adj,(random_cycle2*coincidence))
                                            np.put(App.daily_use_masked,indexes_adj,(random_cycle2*coincidence),mode='clip')
                                        else:
                                            np.put(App.daily_use,indexes_adj,(random_cycle3*coincidence))
                                            np.put(App.daily_use_masked,indexes_adj,(random_cycle3*coincidence),mode='clip')
                                    else: #if no duty cycles are specififed, a regular switch_on event is modelled
                                        np.put(App.daily_use,indexes_adj,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence)) #randomises also the App Power if Thermal_P_var is on
                                        np.put(App.daily_use_masked,indexes_adj,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence),mode='clip')
                                    App.daily_use_masked = np.zeros_like(ma.masked_greater_equal(App.daily_use_masked,0.001)) #updates the mask excluding the current switch_on event to identify the free_spots for the next iteration
                                    tot_time = (tot_time - indexes.size) + indexes_adj.size #updates the total time correcting the previous value
                                    break #exit cycle and go to next App
                                else: #if the tot_time has not yet exceeded the App total functioning time, the cycle does the same without applying corrections to indexes size
                                    if np.in1d(peak_time_range,indexes).any() and App.fixed == 'no':
                                        coincidence = min(App.number,max(1,math.ceil(random.gauss(math.ceil(App.number*mu_peak),(s_peak*App.number*mu_peak)))))
                                    elif np.in1d(peak_time_range,indexes).any() == False and App.fixed == 'no':
                                        Prob = random.uniform(0,(App.number-1)/App.number)
                                        array = np.arange(0,App.number)/App.number
                                        try:
                                            on_number = np.max(np.where(Prob>=array))+1
                                        except ValueError:
                                            on_number = 1
                                        coincidence = on_number
                                    else:
                                        coincidence = App.number
                                    if App.activate > 0:
                                        if indexes.size > 0:
                                            evaluate = round(np.mean(indexes))
                                        else:
                                            evaluate = 0
                                        if evaluate in range(App.cw11[0],App.cw11[1]) or evaluate in range(App.cw12[0],App.cw12[1]):
                                            np.put(App.daily_use,indexes,(random_cycle1*coincidence))
                                            np.put(App.daily_use_masked,indexes,(random_cycle1*coincidence),mode='clip')
                                        elif evaluate in range(App.cw21[0],App.cw21[1]) or evaluate in range(App.cw22[0],App.cw22[1]):
                                            np.put(App.daily_use,indexes,(random_cycle2*coincidence))
                                            np.put(App.daily_use_masked,indexes,(random_cycle2*coincidence),mode='clip')
                                        else:
                                            np.put(App.daily_use,indexes,(random_cycle3*coincidence))
                                            np.put(App.daily_use_masked,indexes,(random_cycle3*coincidence),mode='clip')
                                    else:
                                        np.put(App.daily_use,indexes,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence))
                                        np.put(App.daily_use_masked,indexes,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence),mode='clip')
                                    App.daily_use_masked = np.zeros_like(ma.masked_greater_equal(App.daily_use_masked,0.001))
                                    tot_time = tot_time #no correction applied to previously calculated value
                                                    
                                free_spots = [] #calculate how many free spots remain for further switch_ons
                                try:
                                    for j in ma.notmasked_contiguous(App.daily_use_masked):
                                        free_spots.append(j.stop-j.start)
                                except TypeError:
                                    free_spots = [0]
                                max_free_spot = max(free_spots) 
    
                            else:
                                continue #if the random switch_on falls somewhere where the App has been already turned on, tries again from beginning of the while cycle
                    Us.load = Us.load + App.daily_use #adds the App profile to the User load
            Tot_Classes = Tot_Classes + Us.load #adds the User load to the total load of all User classes
        Profile.append(Tot_Classes) #appends the total load to the list that will contain all the generated profiles
        print('Profile',prof_i+1,'/',num_profiles,'completed') #screen update about progress of computation
    return(Profile)
예제 #47
0
def align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list):
    r"""
    takes name scores and gives them to the best annotation

    Returns:
        score_list: list of scores aligned with cm.daid_list and cm.dnid_list

    Args:
        annot_score_list (list): score associated with each annot
        name_groupxs (list): groups annot_score lists into groups compatible with name_score_list
        name_score_list (list): score assocated with name
        nid2_nidx (dict): mapping from nids to index in name score list

    CommandLine:
        python -m ibeis.algo.hots.name_scoring --test-align_name_scores_with_annots
        python -m ibeis.algo.hots.name_scoring --test-align_name_scores_with_annots --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.hots.name_scoring import *  # NOQA
        >>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[18])
        >>> cm = cm_list[0]
        >>> cm.evaluate_csum_annot_score(qreq_)
        >>> cm.evaluate_nsum_name_score(qreq_)
        >>> # Annot aligned lists
        >>> annot_score_list = cm.algo_annot_scores['csum']
        >>> annot_aid_list   = cm.daid_list
        >>> daid2_idx        = cm.daid2_idx
        >>> # Name aligned lists
        >>> name_score_list  = cm.algo_name_scores['nsum']
        >>> name_groupxs     = cm.name_groupxs
        >>> # Execute Function
        >>> score_list = align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list)
        >>> # Check that the correct name gets the highest score
        >>> target = name_score_list[cm.nid2_nidx[cm.qnid]]
        >>> test_index = np.where(score_list == target)[0][0]
        >>> cm.score_list = score_list
        >>> ut.assert_eq(ibs.get_annot_name_rowids(cm.daid_list[test_index]), cm.qnid)
        >>> assert ut.isunique(cm.dnid_list[score_list > 0]), 'bad name score'
        >>> top_idx = cm.algo_name_scores['nsum'].argmax()
        >>> assert cm.get_top_nids()[0] == cm.unique_nids[top_idx], 'bug in alignment'
        >>> ut.quit_if_noshow()
        >>> cm.show_ranked_matches(qreq_)
        >>> ut.show_if_requested()

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.name_scoring import *  # NOQA
        >>> annot_score_list = []
        >>> annot_aid_list   = []
        >>> daid2_idx        = {}
        >>> # Name aligned lists
        >>> name_score_list  = np.array([], dtype=np.float32)
        >>> name_groupxs     = []
        >>> # Execute Function
        >>> score_list = align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list)
    """
    if len(name_groupxs) == 0:
        score_list = np.empty(0, dtype=name_score_list.dtype)
        return score_list
    else:
        # Group annot aligned indicies by nid
        annot_aid_list = np.array(annot_aid_list)
        #nid_list, groupxs  = vt.group_indices(annot_nid_list)
        grouped_scores     = vt.apply_grouping(annot_score_list, name_groupxs)
        grouped_annot_aids = vt.apply_grouping(annot_aid_list, name_groupxs)
        flat_grouped_aids  = np.hstack(grouped_annot_aids)
        #flat_groupxs  = np.hstack(name_groupxs)
        #if __debug__:
        #    sum_scores = np.array([scores.sum() for scores in grouped_scores])
        #    max_scores = np.array([scores.max() for scores in grouped_scores])
        #    assert np.all(name_score_list <= sum_scores)
        #    assert np.all(name_score_list > max_scores)
        # +------------
        # Find the position of the highest name_scoring annotation for each name
        # IN THE FLATTENED GROUPED ANNOT_AID_LIST (this was the bug)
        offset_list = np.array([annot_scores.argmax() for annot_scores in grouped_scores])
        # Find the starting position of eatch group use chain to start offsets with 0
        _padded_scores  = itertools.chain([[]], grouped_scores[:-1])
        sizeoffset_list = np.array([len(annot_scores) for annot_scores in _padded_scores])
        baseindex_list  = sizeoffset_list.cumsum()
        # Augment starting position with offset index
        annot_idx_list = np.add(baseindex_list, offset_list)
        # L______________
        best_aid_list = flat_grouped_aids[annot_idx_list]
        best_idx_list = ut.dict_take(daid2_idx, best_aid_list)
        # give the annotation domain a name score
        #score_list = np.zeros(len(annot_score_list), dtype=name_score_list.dtype)
        score_list = np.full(len(annot_score_list), fill_value=-np.inf,
                             dtype=name_score_list.dtype)
        #score_list = np.full(len(annot_score_list), fill_value=np.nan, dtype=name_score_list.dtype)
        #score_list = np.nan(len(annot_score_list), dtype=name_score_list.dtype)
        # HACK: we need to set these to 'low' values and we also have to respect negatives
        #score_list[:] = -np.inf
        # make sure that the nid_list from group_indicies and the nids belonging to
        # name_score_list (cm.unique_nids) are in alignment
        #nidx_list = np.array(ut.dict_take(nid2_nidx, nid_list))

        # THIS ASSUMES name_score_list IS IN ALIGNMENT WITH BOTH cm.unique_nids and
        # nid_list (which should be == cm.unique_nids)
        score_list[best_idx_list] = name_score_list
        return score_list
예제 #48
0
    def _calc_transport_wilcock_crowe(self):
        """Method to determine the transport time for each parcel in the active
        layer using a sediment transport equation.

        Note: could have options here (e.g. Wilcock and Crowe, FLVB, MPM, etc)
        """
        # Initialize _pvelocity, the virtual velocity of each parcel (link length / link travel time)
        self._pvelocity = np.zeros(self._num_parcels)

        # parcel attribute arrays from DataRecord

        Darray = self._parcels.dataset.D[:, self._time_idx]
        Activearray = self._parcels.dataset.active_layer[:, self._time_idx].values
        Rhoarray = self._parcels.dataset.density.values
        Volarray = self._parcels.dataset.volume[:, self._time_idx].values
        Linkarray = self._parcels.dataset.element_id[
            :, self._time_idx
        ].values  # link that the parcel is currently in

        R = (Rhoarray - self._fluid_density) / self._fluid_density

        # parcel attribute arrays to populate below
        frac_sand_array = np.zeros(self._num_parcels)
        vol_act_array = np.zeros(self._num_parcels)
        Sarray = np.zeros(self._num_parcels)
        Harray = np.zeros(self._num_parcels)
        Larray = np.zeros(self._num_parcels)
        # D_mean_activearray = np.zeros(self._num_parcels) * (np.nan)
        # active_layer_thickness_array = np.zeros(self._num_parcels) * np.nan
        D_mean_activearray = np.full(self._num_parcels, np.nan)
        active_layer_thickness_array = np.full(self._num_parcels, np.nan)

        #        rhos_mean_active = np.zeros(self._num_parcels)
        #        rhos_mean_active.fill(np.nan)

        # find active sand
        findactivesand = (
            self._parcels.dataset.D < _SAND_SIZE
        ) * self._active_parcel_records  # since find active already sets all prior timesteps to False, we can use D for all timesteps here.

        vol_act_sand = self._parcels.calc_aggregate_value(
            xr.Dataset.sum,
            "volume",
            at="link",
            filter_array=findactivesand,
            fill_value=0.0,
        )

        frac_sand = np.zeros_like(self._vol_act)
        frac_sand[self._vol_act != 0.0] = (
            vol_act_sand[self._vol_act != 0.0] / self._vol_act[self._vol_act != 0.0]
        )
        frac_sand[np.isnan(frac_sand)] = 0.0

        # Calc attributes for each link, map to parcel arrays
        for i in range(self._grid.number_of_links):

            active_here = np.nonzero(
                np.logical_and(Linkarray == i, Activearray == _ACTIVE)
            )[0]
            d_act_i = Darray[active_here]
            vol_act_i = Volarray[active_here]
            rhos_act_i = Rhoarray[active_here]
            vol_act_tot_i = np.sum(vol_act_i)

            self._d_mean_active[i] = np.sum(d_act_i * vol_act_i) / (vol_act_tot_i)
            if vol_act_tot_i > 0:
                self._rhos_mean_active[i] = np.sum(rhos_act_i * vol_act_i) / (
                    vol_act_tot_i
                )
            else:
                self._rhos_mean_active[i] = np.nan
            D_mean_activearray[Linkarray == i] = self._d_mean_active[i]
            frac_sand_array[Linkarray == i] = frac_sand[i]
            vol_act_array[Linkarray == i] = self._vol_act[i]
            Sarray[Linkarray == i] = self._grid.at_link["channel_slope"][i]
            Harray[Linkarray == i] = self._grid.at_link["flow_depth"][i]
            Larray[Linkarray == i] = self._grid.at_link["reach_length"][i]
            active_layer_thickness_array[Linkarray == i] = self._active_layer_thickness[
                i
            ]

        # Wilcock and Crowe calculate transport for all parcels (active and inactive)
        taursg = _calculate_reference_shear_stress(
            self._fluid_density, R, self._g, D_mean_activearray, frac_sand_array
        )

        frac_parcel = np.nan * np.zeros_like(Volarray)
        frac_parcel[vol_act_array != 0.0] = (
            Volarray[vol_act_array != 0.0] / vol_act_array[vol_act_array != 0.0]
        )

        b = 0.67 / (1.0 + np.exp(1.5 - Darray / D_mean_activearray))

        tau = self._fluid_density * self._g * Harray * Sarray
        tau = np.atleast_1d(tau)

        taur = taursg * (Darray / D_mean_activearray) ** b
        tautaur = tau / taur
        tautaur_cplx = tautaur.astype(np.complex128)
        # ^ work around needed b/c np fails with non-integer powers of negative numbers

        W = 0.002 * np.power(tautaur_cplx.real, 7.5)
        W[tautaur >= 1.35] = 14 * np.power(
            (1 - (0.894 / np.sqrt(tautaur_cplx.real[tautaur >= 1.35]))), 4.5
        )

        active_parcel_idx = Activearray == _ACTIVE

        # compute parcel virtual velocity, m/s
        self._pvelocity[active_parcel_idx] = (
            W.real[active_parcel_idx]
            * (tau[active_parcel_idx] ** (3.0 / 2.0))
            * frac_parcel[active_parcel_idx]
            / (self._fluid_density ** (3.0 / 2.0))
            / self._g
            / R[active_parcel_idx]
            / active_layer_thickness_array[active_parcel_idx]
        )

        self._pvelocity[np.isnan(self._pvelocity)] = 0.0

        if np.max(self._pvelocity) > 1:
            warnings.warn(
                "NetworkSedimentTransporter: Maximum parcel virtual velocity exceeds 1 m/s"
            )

        # Assign those things to the grid -- might be useful for plotting
        self._grid.at_link["sediment_total_volume"] = self._vol_tot
        self._grid.at_link["sediment__active__volume"] = self._vol_act
        self._grid.at_link["sediment__active__sand_fraction"] = frac_sand
예제 #49
0
def pool3d_ncdhw_python(
    np_data,
    kernel,
    strides,
    padding,
    out_shape,
    pool_type,
    count_include_pad=True,
    ceil_mode=False,
    dtype="float32",
):
    """baseline for max_pool3d and avg_pool3d, default layout is "NCDHW"""
    # fmt: off
    in_n, in_c, in_d, in_h, in_w = in_shape = np_data.shape
    if isinstance(kernel, int):
        k_d = k_h = k_w = kernel
    else:
        k_d, k_h, k_w = kernel
    if isinstance(strides, int):
        s_d = s_h = s_w = strides
    else:
        s_d, s_h, s_w = strides
    if isinstance(padding, int):
        pf = pt = pl = pk = pb = pr = padding
    else:
        pf, pt, pl, pk, pb, pr = padding

    if ceil_mode:
        assert out_shape[2] == int(
            math.ceil(float(in_shape[2] - k_d + pf + pk) / s_d) + 1)
        assert out_shape[3] == int(
            math.ceil(float(in_shape[3] - k_h + pt + pb) / s_h) + 1)
        assert out_shape[4] == int(
            math.ceil(float(in_shape[4] - k_w + pl + pr) / s_w) + 1)
    else:
        assert out_shape[2] == int(
            math.floor(float(in_shape[2] - k_d + pf + pk) / s_d) + 1)
        assert out_shape[3] == int(
            math.floor(float(in_shape[3] - k_h + pt + pb) / s_h) + 1)
        assert out_shape[4] == int(
            math.floor(float(in_shape[4] - k_w + pl + pr) / s_w) + 1)

    fill_value = tvm.tir.const(0.0, dtype).value
    if not (count_include_pad) and pool_type == 'max':
        fill_value = tvm.te.min_value(dtype).value

    pad_np = np.full(shape=(in_n, in_c, in_d + pf + pk, in_h + pt + pb,
                            in_w + pl + pr),
                     fill_value=fill_value,
                     dtype=dtype)

    no_zero = (range(in_n), range(in_c), (range(pf, in_d + pf)),
               (range(pt, in_h + pt)), (range(pl, in_w + pl)))
    pad_np[np.ix_(*no_zero)] = np_data
    ret_np = np.zeros(shape=out_shape).astype(dtype)

    if pool_type == 'avg':
        for k in range(out_shape[2]):
            for i in range(out_shape[3]):
                for j in range(out_shape[4]):
                    if count_include_pad:
                        ret_np[:, :, k, i, j] = \
                            np.mean(pad_np[:, :, k * s_d: k * s_d + k_d,
                                           i * s_h: i * s_h + k_h,
                                           j * s_w: j * s_w + k_w], axis=(2, 3, 4))
                    else:
                        pad_count = np.sum(pad_np[:, :, k * s_d:k * s_d + k_d,
                                                  i * s_h:i * s_h + k_h,
                                                  j * s_w:j * s_w + k_w] > 0,
                                           axis=(2, 3, 4))
                        ret_np[:, :, k, i, j] = np.sum(
                            pad_np[:, :, k * s_d:k * s_d + k_d, i *
                                   s_h:i * s_h + k_h, j * s_w:j * s_w + k_w],
                            axis=(2, 3, 4)) / np.maximum(pad_count, 1)
    elif pool_type == 'max':
        for k in range(out_shape[2]):
            for i in range(out_shape[3]):
                for j in range(out_shape[4]):
                    ret_np[:, :, k, i,
                           j] = np.max(pad_np[:, :, k * s_d:k * s_d + k_d,
                                              i * s_h:i * s_h + k_h,
                                              j * s_w:j * s_w + k_w],
                                       axis=(2, 3, 4))
    else:
        raise ValueError("pool type {} is not supported".format(pool_type))

    ret_np = np.maximum(ret_np, fill_value)
    # fmt: on
    return ret_np
    def transform(self, X: dt.Frame, **kwargs):
        """
        Uses fitted models (1 per time group) to predict the target
        :param X: Datatable Frame containing the features
        :return: FB Prophet predictions
        """
        # Get the logger if it exists
        logger = self.get_experiment_logger()

        tmp_folder = self._create_tmp_folder(logger)

        n_jobs = self._get_n_jobs(logger, **kwargs)

        # Reduce X to TGC
        tgc_wo_time = list(np.setdiff1d(self.tgc, self.time_column))

        # Change date feature name to match Prophet requirements
        X = self.convert_to_prophet(X)

        y_predictions = self.predict_with_average_model(X, tgc_wo_time)
        y_predictions.columns = ['average_pred']

        # Go through groups
        for grp_col in tgc_wo_time:
            # Get the unique dates to be predicted
            X_groups = X[['ds', grp_col]].groupby(grp_col)

            # Go though the groups and predict only top
            XX_paths = []
            model_paths = []

            def processor(out, res):
                out.append(res)

            num_tasks = len(X_groups)
            pool_to_use = small_job_pool
            pool = pool_to_use(logger=None,
                               processor=processor,
                               num_tasks=num_tasks,
                               max_workers=n_jobs)

            for _i_g, (key, X_grp) in enumerate(X_groups):

                # Just log where we are in the fitting process
                if (_i_g + 1) % max(1, num_tasks // 20) == 0:
                    loggerinfo(
                        logger, "FB Prophet : %d%% of groups predicted" %
                        (100 * (_i_g + 1) // num_tasks))

                # Create dict key to store the min max scaler
                grp_hash = self.get_hash(key)
                X_path = os.path.join(tmp_folder,
                                      "fbprophet_Xt" + str(uuid.uuid4()))

                if grp_hash not in self.grp_models[grp_col]:
                    # unseen groups
                    XX = pd.DataFrame(np.full((X_grp.shape[0], 1), np.nan),
                                      columns=['yhat'])
                    XX.index = X_grp.index
                    save_obj(XX, X_path)
                    XX_paths.append(X_path)
                    continue

                if self.grp_models[grp_col][grp_hash] is None:
                    # known groups but not enough train data
                    XX = pd.DataFrame(np.full((X_grp.shape[0], 1), np.nan),
                                      columns=['yhat'])
                    XX.index = X_grp.index
                    save_obj(XX, X_path)
                    XX_paths.append(X_path)
                    continue

                model = self.grp_models[grp_col][grp_hash]
                model_path = os.path.join(
                    tmp_folder, "fbprophet_modelt" + str(uuid.uuid4()))
                save_obj(model, model_path)
                save_obj(X_grp, X_path)
                model_paths.append(model_path)

                args = (model_path, X_path, self.priors[grp_col][grp_hash],
                        tmp_folder)
                kwargs = {}
                pool.submit_tryget(
                    None,
                    MyProphetOnSingleGroupsTransformer_transform_async,
                    args=args,
                    kwargs=kwargs,
                    out=XX_paths)

            pool.finish()
            y_predictions[f'{grp_col}_pred'] = pd.concat(
                (load_obj(XX_path) for XX_path in XX_paths),
                axis=0).sort_index()
            for p in XX_paths + model_paths:
                remove(p)

        # Now we can invert scale
        # But first get rid of NaNs
        for grp_col in tgc_wo_time:
            # Add time group to the predictions, will be used to invert scaling
            y_predictions[grp_col] = X[grp_col]
            # Fill NaN
            y_predictions[f'{grp_col}_pred'] = y_predictions[
                f'{grp_col}_pred'].fillna(y_predictions['average_pred'])

        # Go through groups and recover the scaled target for knowed groups
        if len(tgc_wo_time) > 0:
            X_groups = y_predictions.groupby(tgc_wo_time)
        else:
            X_groups = [([None], y_predictions)]

        for _f in [f'{grp_col}_pred'
                   for grp_col in tgc_wo_time] + ['average_pred']:
            inverted_ys = []
            for key, X_grp in X_groups:
                grp_hash = self.get_hash(key)
                # Scale target for current group
                if grp_hash in self.scalers.keys():
                    inverted_y = self.scalers[grp_hash].inverse_transform(
                        X_grp[[_f]])
                else:
                    inverted_y = self.general_scaler.inverse_transform(
                        X_grp[[_f]])

                # Put back in a DataFrame to keep track of original index
                inverted_df = pd.DataFrame(inverted_y, columns=[_f])
                inverted_df.index = X_grp.index
                inverted_ys.append(inverted_df)
            y_predictions[_f] = pd.concat(tuple(inverted_ys),
                                          axis=0).sort_index()[_f]

        self._clean_tmp_folder(logger, tmp_folder)

        y_predictions.drop(tgc_wo_time, axis=1, inplace=True)

        self._output_feature_names = [
            f'{self.display_name}{orig_feat_prefix}{self.time_column}{extra_prefix}{_f}'
            for _f in y_predictions
        ]
        self._feature_desc = self._output_feature_names
        return y_predictions
예제 #51
0
a = np.array([1, 2, 3, 4, 5])  # Create a rank 1 array
print(type(a))  # Prints "<class 'numpy.ndarray'>"
print(a.shape)  # Prints "(3,)"
print(a[0], a[1], a[2])  # Prints "1 2 3"
a[0] = 5  # Change an element of the array
print(a)  # Prints "[5, 2, 3]"

b = np.array([[1, 2, 3], [4, 5, 6]])  # Create a rank 2 array
print(b.shape)  # Prints "(2, 3)"
print(b[0, 0], b[0, 1], b[1, 0])  # Prints "1 2 4"

a = np.zeros((2, 2))  # Create an array of all zeros
print(a)  # Prints "[[ 0.  0.]
#          [ 0.  0.]]"

b = np.ones((1, 2))  # Create an array of all ones
print(b)  # Prints "[[ 1.  1.]]"

c = np.full((2, 2), 7)  # Create a constant array
print(c)  # Prints "[[ 7.  7.]
#          [ 7.  7.]]"

d = np.eye(2)  # Create a 2x2 identity matrix
print(d)  # Prints "[[ 1.  0.]
#          [ 0.  1.]]"

e = np.random.random((2, 2))  # Create an array filled with random values
print(e)  # Might print "[[ 0.91940167  0.08143941]
#               [ 0.68744134  0.87236687]]"
예제 #52
0
        if count1 >= count2 and count1 >= count3:
            prediction = 1
        if count2 >= count1 and count2 >= count3:
            prediction = 2
        if count3 >= count1 and count3 >= count2:
            prediction = 3  # 取出现次数最多的为预测值
        if prediction == test[0, 4]:
            accuracy += 1
    Accuracy = accuracy / 150
    print("Iris数据集的最近邻准确率为:", Accuracy)
    return Accuracy


x = iris.iloc[:, 0:4]
x = np.mat(x)
a = np.full((50, 1), 1)
b = np.full((50, 1), 2)
c = np.full((50, 1), 3)
Res = np.zeros(50)

d = np.append(a, b, axis=0)
d = np.append(d, c, axis=0)
X = np.append(x, d, axis=1)  # 将数据集中的标签更换为1,2,3

for m in range(10):
    k = m + 1
    Res[m] = k_nn(X)
'''
# 绘制 k与分类准确率的图像
import matplotlib.pyplot as plt