def test_edge_cases_normalize(self):
"""`stats.normalize`: Edge Case Validator.
Tests the behavior of `normalize` with edge cases.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
with self.assertRaises(_InvalidFeatureSetError):
# Empty matrix `X`.
normalize(_np.matrix([[]]))
def test_invalid_args_normalize(self):
"""`stats.normalize`: Argument Validator.
Tests the behavior of `normalize` with invalid argument counts and
values.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
n, d = self.data_shape
"""(int, int): Number of data points and number of features."""
with self.assertRaises(TypeError):
# No arguments.
normalize()
with self.assertRaises(TypeError):
# More than one argument.
normalize(123, 123, 123)
with self.assertRaises(_InvalidFeatureSetError):
# `None` instead of matrix `X`.
normalize(None)
with self.assertRaises(_InvalidFeatureSetError):
# List of an empty list instead of matrix `X`.
normalize([[]])
def genes_jsd(self, gene1, gene2, log=True):
gene1_i = self.name_or_index(gene1)
gene2_i = self.name_or_index(gene2)
gexpr1 = self.expr[gene1_i,:]
if log:
gexpr1 = [math.log(e+1) for e in gexpr1]
gexpr1 = stats.normalize(gexpr1)
gexpr2 = self.expr[gene2_i,:]
if log:
gexpr2 = [math.log(e+1) for e in gexpr2]
gexpr2 = stats.normalize(gexpr2)
return stats.jsd(gexpr1, gexpr2)
def genes_jsd(self, gene1, gene2, log=True):
gene1_i = self.name_or_index(gene1)
gene2_i = self.name_or_index(gene2)
gexpr1 = self.expr[gene1_i, :]
if log:
gexpr1 = [math.log(e + 1) for e in gexpr1]
gexpr1 = stats.normalize(gexpr1)
gexpr2 = self.expr[gene2_i, :]
if log:
gexpr2 = [math.log(e + 1) for e in gexpr2]
gexpr2 = stats.normalize(gexpr2)
return stats.jsd(gexpr1, gexpr2)
def gene_specificity(self, gene, log=True):
gene_i = self.name_or_index(gene)
if gene_i == None:
spec = 0
else:
gexpr = self.expr[gene_i,:]
if log:
gexpr = [math.log(e+1) for e in gexpr]
if sum(gexpr) == 0:
spec = 0
else:
gexpr = stats.normalize(gexpr)
min_jsd = 1.0
for j in range(len(self.experiments)):
q_j = [0]*len(self.experiments)
q_j[j] = 1.0
min_jsd = min(min_jsd, math.sqrt(stats.jsd(gexpr, q_j)))
spec = 1.0 - min_jsd
return spec
def clamp_and_scale(img, bands, p, AOI):
"""
clip the upper range of an image based on percentile
This function is similar to ee.Image().clip() and ee.Image().unitScale(),
but operates on multiple bands with potentially different upper limits.
Parameters:
img (ee.Image): the image to modify
bands (ee.List<str>):
p (int): upper percentile above which to truncate values
AOI (ee.Geometry): area within which to calculate percentile
Returns:
ee.Image: rescaled image with band values [0, 1]
"""
#create a list of the 99th percentile value for all bands
percentiles = img.select(bands).reduceRegion(
reducer = ee.Reducer.percentile([99]).repeat(ee.List(bands).size()),
geometry = AOI,
scale = 100,
maxPixels = 1e13,
tileScale = 12
).get('p99')
#turn list of 99th percentiles into constant image
upperImg = ee.Image.constant(percentiles).rename(bands)
#clip the upper range of extreme values where sensors get washed out
normImage = img.where(img.gte(upperImg), upperImg)
# rescale the truncated image to [0, 1]
rescaled = normalize(normImage, upperImg, ee.Image.constant(0))
return ee.Image(rescaled)
def test_random_normalize(self):
"""`stats.normalize`: Randomized Validator.
Tests the behavior of `normalize` by feeding it randomly generated
arguments.
Raises:
AssertionError: If `normalize` needs debugging.
"""
n, d = self.data_shape
"""(int, int): Number of data points and number of features."""
for i in range(self.n_tests):
X = normalize(_np.matrix(_np.random.rand(n, d)))
"""np.matrix: Randomized test input."""
# `X` should be a matrix.
self.assertIsInstance(X, _np.matrix)
# The means of all rows in `X` should be zero-valued.
self.assertLessEqual(
_np.linalg.norm(_np.mean(X, 1))**2, self.zero_cutoff)
# The standard deviations of all rows in `X` should be unit-valued.
self.assertEqual(_np.linalg.norm(_np.std(X, 1))**2, n)
def transform(self, x, norm=False):
if norm:
x = stats.normalize(x)
x = to_tensor(x)
return x
def gene_specificity(self, gene, log=True):
gene_i = self.name_or_index(gene)
if gene_i == None:
spec = 0
else:
gexpr = self.expr[gene_i, :]
if log:
gexpr = [math.log(e + 1) for e in gexpr]
if sum(gexpr) == 0:
spec = 0
else:
gexpr = stats.normalize(gexpr)
min_jsd = 1.0
for j in range(len(self.experiments)):
q_j = [0] * len(self.experiments)
q_j[j] = 1.0
min_jsd = min(min_jsd, math.sqrt(stats.jsd(gexpr, q_j)))
spec = 1.0 - min_jsd
return spec
def gene_entropy(self, gene, log=False):
gene_i = self.name_or_index(gene)
gexpr = self.expr[gene_i,:]
if log:
gexpr = [math.log(e+1) for e in gexpr]
gexpr = stats.normalize(gexpr)
return stats.entropy(gexpr)
def gene_entropy(self, gene, log=False):
gene_i = self.name_or_index(gene)
gexpr = self.expr[gene_i, :]
if log:
gexpr = [math.log(e + 1) for e in gexpr]
gexpr = stats.normalize(gexpr)
return stats.entropy(gexpr)
def keyword_trend_png(k):
"""PNG from ``stats`` for historical score of keyword ``k``
"""
db_resp = db.keyword_trend(k)
if len(db_resp) < 2:
# not enough data for a plot
abort(404)
x = map(lambda x: x['Performed'], db_resp)
y = map(lambda x: x['EntropyScore'], db_resp)
if 'nonorm' not in request.args:
y = stats.normalize(y)
response = make_response(stats.keyword_trend(k, x, y).getvalue())
response.mimetype = 'image/png'
return response
print("Ninja combined {0:.2f} {1:.2f}".format(annual_cf_ninja_nat, max_cf_ninja_nat))
annual_cf_ninja_on = ninja['onshore'].sum() / 365
max_cf_ninja_on = ninja['onshore'].max()
print("Ninja onshore {0:.2f} {1:.2f}".format(annual_cf_ninja_on, max_cf_ninja_on))
annual_cf_ninja_off = ninja['offshore'].sum() / 365
max_cf_ninja_off = ninja['offshore'].max()
print("Ninja offshore {0:.2f} {1:.2f}".format(annual_cf_ninja_off, max_cf_ninja_off))
annual_cf_ng = national_grid_cf.sum() / 365
max_cf_ng = national_grid_cf.max()
print("National Grid {0:.2f} {1:.2f}".format(annual_cf_ng, max_cf_ng))
annual_cf_ex = elexon_cf.sum() / 365
max_cf_ex = elexon_cf.max()
print("Elexon Generation {0:.2f} {1:.2f}".format(annual_cf_ex, max_cf_ex))
ninja_national = ninja['national']
# Normalise CF before stats
elexon_cf = stats.normalize(elexon_cf)
era_total = stats.normalize(era_total)
midas_total = stats.normalize(midas_total)
ninja_national = stats.normalize(ninja_national)
national_grid_cf = stats.normalize(national_grid_cf)
# do stats for all 4
print('Stats compared to elexon')
stats.print_stats_header()
stats.print_stats(era_total, elexon_cf, 'era elexon', 1, True, 'Capacity Factor ERA', 'Capacity Factor Elexon')
stats.print_stats(midas_total, elexon_cf, 'midas', 1, True, 'Capacity Factor MIDAS', 'Capacity Factor Elexon')
stats.print_stats(ninja['national'], elexon_cf, 'ninja', 1, True, 'Capacity Factor Ninja', 'Capacity Factor Elexon')
stats.print_stats(national_grid_cf, elexon_cf, 'national grid', 1, True, 'Capacity Factor Embedded', 'Capacity Factor Elexon')
for i in range(a.shape[0]):
b = a.loc[i].values
if b[1] == to_test:
cur.execute(sql.SQL("select {} from storm_student where baseuser_ptr_id=%s;").format(sql.Identifier(key_num_test[0])),[int(b[0])])
c = cur.fetchall()
new_number = int(c[0][0])
new_number+=1
cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_num_test[0])),[new_number,int(b[0])])
cur.execute(sql.SQL("select {} from storm_student where baseuser_ptr_id=%s;").format(sql.Identifier(key_total_test[0])),[int(b[0])])
c = cur.fetchall()
new_total = float(c[0][0])
bus_id = int(b[3])
distr = distribution_maker(bus_id)
new_total+=stats.normalize(distr,int(b[2]))
cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_total_test[0])),[new_total,int(b[0])])
if new_number != 0:
new_average = new_total/float(new_number)
else:
new_average = 0
cur.execute(sql.SQL("update storm_student set {}=%s where baseuser_ptr_id=%s;").format(sql.Identifier(key_average_test[0])),[new_average,int(b[0])])
# Committing changes
cur.execute("commit;")
print("Success populating student columns.")
# Populating business columns
weather_year = '2018'
met_temp_filename = '/home/malcolm/uclan/data/hadcet_mean_2018.csv'
gas_filename = '/home/malcolm/uclan/data/GasLDZOfftakeEnergy' + weather_year + '.csv'
met_temp = readers.read_hadcet(met_temp_filename)
print('MET')
print(met_temp.index)
print(met_temp)
gas = readers.read_gas(gas_filename)
inverse_temp = met_temp * -1.0
# normalize
gas = stats.normalize(gas)
inverse_temp = stats.normalize(inverse_temp)
# output plots
gas.plot(label='Gas')
inverse_temp.plot(label='Inverse Temperature')
plt.title('2018 UK mean daily temperature inverse vs gas')
plt.xlabel('Day of the year')
plt.ylabel('Temperature (Degrees C)')
plt.legend(loc='upper right')
plt.show()
stats.print_stats_header()
stats.print_stats(gas, inverse_temp, 'Gas vs Temp')
