def test_edge_cases_normalize(self): """`stats.normalize`: Edge Case Validator. Tests the behavior of `normalize` with edge cases. Raises: Exception: If at least one `Exception` raised is not of the expected kind. """ with self.assertRaises(_InvalidFeatureSetError): # Empty matrix `X`. normalize(_np.matrix([[]]))
def test_invalid_args_normalize(self): """`stats.normalize`: Argument Validator. Tests the behavior of `normalize` with invalid argument counts and values. Raises: Exception: If at least one `Exception` raised is not of the expected kind. """ n, d = self.data_shape """(int, int): Number of data points and number of features.""" with self.assertRaises(TypeError): # No arguments. normalize() with self.assertRaises(TypeError): # More than one argument. normalize(123, 123, 123) with self.assertRaises(_InvalidFeatureSetError): # `None` instead of matrix `X`. normalize(None) with self.assertRaises(_InvalidFeatureSetError): # List of an empty list instead of matrix `X`. normalize([[]])
def genes_jsd(self, gene1, gene2, log=True): gene1_i = self.name_or_index(gene1) gene2_i = self.name_or_index(gene2) gexpr1 = self.expr[gene1_i,:] if log: gexpr1 = [math.log(e+1) for e in gexpr1] gexpr1 = stats.normalize(gexpr1) gexpr2 = self.expr[gene2_i,:] if log: gexpr2 = [math.log(e+1) for e in gexpr2] gexpr2 = stats.normalize(gexpr2) return stats.jsd(gexpr1, gexpr2)
def genes_jsd(self, gene1, gene2, log=True): gene1_i = self.name_or_index(gene1) gene2_i = self.name_or_index(gene2) gexpr1 = self.expr[gene1_i, :] if log: gexpr1 = [math.log(e + 1) for e in gexpr1] gexpr1 = stats.normalize(gexpr1) gexpr2 = self.expr[gene2_i, :] if log: gexpr2 = [math.log(e + 1) for e in gexpr2] gexpr2 = stats.normalize(gexpr2) return stats.jsd(gexpr1, gexpr2)
def gene_specificity(self, gene, log=True): gene_i = self.name_or_index(gene) if gene_i == None: spec = 0 else: gexpr = self.expr[gene_i,:] if log: gexpr = [math.log(e+1) for e in gexpr] if sum(gexpr) == 0: spec = 0 else: gexpr = stats.normalize(gexpr) min_jsd = 1.0 for j in range(len(self.experiments)): q_j = [0]*len(self.experiments) q_j[j] = 1.0 min_jsd = min(min_jsd, math.sqrt(stats.jsd(gexpr, q_j))) spec = 1.0 - min_jsd return spec
def clamp_and_scale(img, bands, p, AOI): """ clip the upper range of an image based on percentile This function is similar to ee.Image().clip() and ee.Image().unitScale(), but operates on multiple bands with potentially different upper limits. Parameters: img (ee.Image): the image to modify bands (ee.List<str>): p (int): upper percentile above which to truncate values AOI (ee.Geometry): area within which to calculate percentile Returns: ee.Image: rescaled image with band values [0, 1] """ #create a list of the 99th percentile value for all bands percentiles = img.select(bands).reduceRegion( reducer = ee.Reducer.percentile([99]).repeat(ee.List(bands).size()), geometry = AOI, scale = 100, maxPixels = 1e13, tileScale = 12 ).get('p99') #turn list of 99th percentiles into constant image upperImg = ee.Image.constant(percentiles).rename(bands) #clip the upper range of extreme values where sensors get washed out normImage = img.where(img.gte(upperImg), upperImg) # rescale the truncated image to [0, 1] rescaled = normalize(normImage, upperImg, ee.Image.constant(0)) return ee.Image(rescaled)
def test_random_normalize(self): """`stats.normalize`: Randomized Validator. Tests the behavior of `normalize` by feeding it randomly generated arguments. Raises: AssertionError: If `normalize` needs debugging. """ n, d = self.data_shape """(int, int): Number of data points and number of features.""" for i in range(self.n_tests): X = normalize(_np.matrix(_np.random.rand(n, d))) """np.matrix: Randomized test input.""" # `X` should be a matrix. self.assertIsInstance(X, _np.matrix) # The means of all rows in `X` should be zero-valued. self.assertLessEqual( _np.linalg.norm(_np.mean(X, 1))**2, self.zero_cutoff) # The standard deviations of all rows in `X` should be unit-valued. self.assertEqual(_np.linalg.norm(_np.std(X, 1))**2, n)
def transform(self, x, norm=False): if norm: x = stats.normalize(x) x = to_tensor(x) return x
def gene_specificity(self, gene, log=True): gene_i = self.name_or_index(gene) if gene_i == None: spec = 0 else: gexpr = self.expr[gene_i, :] if log: gexpr = [math.log(e + 1) for e in gexpr] if sum(gexpr) == 0: spec = 0 else: gexpr = stats.normalize(gexpr) min_jsd = 1.0 for j in range(len(self.experiments)): q_j = [0] * len(self.experiments) q_j[j] = 1.0 min_jsd = min(min_jsd, math.sqrt(stats.jsd(gexpr, q_j))) spec = 1.0 - min_jsd return spec
def gene_entropy(self, gene, log=False): gene_i = self.name_or_index(gene) gexpr = self.expr[gene_i,:] if log: gexpr = [math.log(e+1) for e in gexpr] gexpr = stats.normalize(gexpr) return stats.entropy(gexpr)
def gene_entropy(self, gene, log=False): gene_i = self.name_or_index(gene) gexpr = self.expr[gene_i, :] if log: gexpr = [math.log(e + 1) for e in gexpr] gexpr = stats.normalize(gexpr) return stats.entropy(gexpr)
def keyword_trend_png(k): """PNG from ``stats`` for historical score of keyword ``k`` """ db_resp = db.keyword_trend(k) if len(db_resp) < 2: # not enough data for a plot abort(404) x = map(lambda x: x['Performed'], db_resp) y = map(lambda x: x['EntropyScore'], db_resp) if 'nonorm' not in request.args: y = stats.normalize(y) response = make_response(stats.keyword_trend(k, x, y).getvalue()) response.mimetype = 'image/png' return response
print("Ninja combined {0:.2f} {1:.2f}".format(annual_cf_ninja_nat, max_cf_ninja_nat)) annual_cf_ninja_on = ninja['onshore'].sum() / 365 max_cf_ninja_on = ninja['onshore'].max() print("Ninja onshore {0:.2f} {1:.2f}".format(annual_cf_ninja_on, max_cf_ninja_on)) annual_cf_ninja_off = ninja['offshore'].sum() / 365 max_cf_ninja_off = ninja['offshore'].max() print("Ninja offshore {0:.2f} {1:.2f}".format(annual_cf_ninja_off, max_cf_ninja_off)) annual_cf_ng = national_grid_cf.sum() / 365 max_cf_ng = national_grid_cf.max() print("National Grid {0:.2f} {1:.2f}".format(annual_cf_ng, max_cf_ng)) annual_cf_ex = elexon_cf.sum() / 365 max_cf_ex = elexon_cf.max() print("Elexon Generation {0:.2f} {1:.2f}".format(annual_cf_ex, max_cf_ex)) ninja_national = ninja['national'] # Normalise CF before stats elexon_cf = stats.normalize(elexon_cf) era_total = stats.normalize(era_total) midas_total = stats.normalize(midas_total) ninja_national = stats.normalize(ninja_national) national_grid_cf = stats.normalize(national_grid_cf) # do stats for all 4 print('Stats compared to elexon') stats.print_stats_header() stats.print_stats(era_total, elexon_cf, 'era elexon', 1, True, 'Capacity Factor ERA', 'Capacity Factor Elexon') stats.print_stats(midas_total, elexon_cf, 'midas', 1, True, 'Capacity Factor MIDAS', 'Capacity Factor Elexon') stats.print_stats(ninja['national'], elexon_cf, 'ninja', 1, True, 'Capacity Factor Ninja', 'Capacity Factor Elexon') stats.print_stats(national_grid_cf, elexon_cf, 'national grid', 1, True, 'Capacity Factor Embedded', 'Capacity Factor Elexon')
for i in range(a.shape[0]): b = a.loc[i].values if b[1] == to_test: cur.execute(sql.SQL("select {} from storm_student where baseuser_ptr_id=%s;").format(sql.Identifier(key_num_test[0])),[int(b[0])]) c = cur.fetchall() new_number = int(c[0][0]) new_number+=1 cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_num_test[0])),[new_number,int(b[0])]) cur.execute(sql.SQL("select {} from storm_student where baseuser_ptr_id=%s;").format(sql.Identifier(key_total_test[0])),[int(b[0])]) c = cur.fetchall() new_total = float(c[0][0]) bus_id = int(b[3]) distr = distribution_maker(bus_id) new_total+=stats.normalize(distr,int(b[2])) cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_total_test[0])),[new_total,int(b[0])]) if new_number != 0: new_average = new_total/float(new_number) else: new_average = 0 cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_average_test[0])),[new_average,int(b[0])]) # Committing changes cur.execute("commit;") print("Success populating student columns.") # Populating business columns
weather_year = '2018' met_temp_filename = '/home/malcolm/uclan/data/hadcet_mean_2018.csv' gas_filename = '/home/malcolm/uclan/data/GasLDZOfftakeEnergy' + weather_year + '.csv' met_temp = readers.read_hadcet(met_temp_filename) print('MET') print(met_temp.index) print(met_temp) gas = readers.read_gas(gas_filename) inverse_temp = met_temp * -1.0 # normalize gas = stats.normalize(gas) inverse_temp = stats.normalize(inverse_temp) # output plots gas.plot(label='Gas') inverse_temp.plot(label='Inverse Temperature') plt.title('2018 UK mean daily temperature inverse vs gas') plt.xlabel('Day of the year') plt.ylabel('Temperature (Degrees C)') plt.legend(loc='upper right') plt.show() stats.print_stats_header() stats.print_stats(gas, inverse_temp, 'Gas vs Temp')