def area_and_parity_check_single_batch(data_mb,
num_constraints=None,
area=None,
polygons_number=None,
img_width=None,
img_height=None,
**kwargs):
# preallocate an output array that will contain the computed
# constraints values for the batch
mb_constraints_values = np_empty(shape=(data_mb.shape[0], num_constraints),
dtype=np_float32)
for i in prange(data_mb.shape[0]):
sample = data_mb[i]
# preallocate an output array that will contain the computed
# constraints value for the i-th element
constraints = np_empty(shape=(num_constraints, ), dtype=np_float32)
target_area = area * polygons_number
nonzero = np_sum(sample[1:-1, 1:-1])
norm = img_width * img_height - target_area
greater_area_inner = min(1, max(0, nonzero - target_area) / norm)
smaller_area_inner = min(1, max(0, target_area - nonzero) / norm)
constraints[0] = greater_area_inner
constraints[1] = smaller_area_inner
for x in prange(1, 10): # _parity_check_rl_X
sum_left = np_sum(sample[x, 1:int(img_width / 2)])
constraints[x + 1] = int(sample[x][0][0] != sum_left % 2)
for x in prange(1, 10): # _parity_check_ct_X
sum_top = np_sum(sample[1:int(img_height / 2), x])
constraints[x + 10] = int(sample[0][x][0] != sum_top % 2)
mb_constraints_values[i] = constraints
return mb_constraints_values
def area_and_convexity_single_batch(data_mb,
num_constraints=None,
area=None,
polygons_number=None,
img_width=None,
img_height=None,
**kwargs):
# preallocate an output array that will contain the computed
# constraints values for the batch
mb_constraints_values = np_empty(shape=(data_mb.shape[0], num_constraints),
dtype=np_float32)
for i in prange(data_mb.shape[0]):
sample = data_mb[i]
# preallocate an output array that will contain the computed
# constraints value for the i-th element
constraints = np_empty(shape=(num_constraints, ), dtype=np_float32)
target_area = area * polygons_number
nonzero = np_sum(sample)
norm = img_width * img_height - target_area
greater_area_inner = min(1, max(0, nonzero - target_area) / norm)
smaller_area_inner = min(1, max(0, target_area - nonzero) / norm)
constraints[0] = greater_area_inner
constraints[1] = smaller_area_inner
# convexity
constraints[2] = _convex(sample, img_width, img_height)
mb_constraints_values[i] = constraints
return mb_constraints_values
def _do_bkpt(self):
"""
The _do_bkpt function calculates the min and max terminals of the node on each dimension of the node.
:returns: an array containing the min terminals and one containing the max terminals
:rtype: numpy.array, numpy.array
"""
bkpt_list_min = np_empty(self.tree.size_word)
bkpt_list_max = np_empty(self.tree.size_word)
for i, iSAX_letter in enumerate(self.iSAX_word):
bkpt_tmp = self.tree.isax._card_to_bkpt(iSAX_letter[1])
# The case where there is no BKPT (root node)
if iSAX_letter[1] < 2:
bkpt_list_min[i] = self.tree.min_max[i][0]
bkpt_list_max[i] = self.tree.min_max[i][1]
# the case where there is no BKPT inf
elif iSAX_letter[0] == 0:
bkpt_list_min[i] = self.tree.min_max[i][0]
bkpt_list_max[i] = bkpt_tmp[iSAX_letter[0]]
# the case where there is no BKPT sup
elif iSAX_letter[0] == iSAX_letter[1]-1:
bkpt_list_min[i] = bkpt_tmp[iSAX_letter[0]-1]
bkpt_list_max[i] = self.tree.min_max[i][1]
# The general case
else:
bkpt_list_min[i] = bkpt_tmp[iSAX_letter[0]-1]
bkpt_list_max[i] = bkpt_tmp[iSAX_letter[0]]
return bkpt_list_min, bkpt_list_max
def area_and_all_parity_check_multi_batch(R_mb,
n_samples=None,
bs=None,
num_constraints=None,
area=None,
polygons_number=None,
img_width=None,
img_height=None,
**kwargs):
# we have (n_samples x bs) generated items; for each of them we have to
# compute the values of each constraint.
# R_mb has shape [n_samples, bs] + image_shape
# e.g. [n_samples, bs, 20, 20, 1]
# so we need to iterate over the first two dimensions
# preallocate an output array that will contain the computed
# constraints values for the group of batches
R_mb_constraints_values = np_empty(shape=(n_samples, bs, num_constraints),
dtype=np_float32)
for i in prange(R_mb.shape[0]):
for j in prange(R_mb.shape[1]):
sample = R_mb[i][j]
# preallocate an output array that will contain the computed
# constraints value for the i-th element
constraints = np_empty(shape=(num_constraints, ), dtype=np_float32)
target_area = area * polygons_number
nonzero = np_sum(sample[1:-1, 1:-1])
norm = img_width * img_height - target_area
greater_area_inner = min(1, max(0, nonzero - target_area) / norm)
smaller_area_inner = min(1, max(0, target_area - nonzero) / norm)
constraints[0] = greater_area_inner
constraints[1] = smaller_area_inner
curr_index = 1
for x in prange(1, img_width - 1): # _parity_check_rl_X
sum_left = np_sum(sample[x, 1:int(img_width / 2)])
constraints[x +
curr_index] = int(sample[x][0][0] != sum_left % 2)
curr_index += img_width - 2
for x in prange(1, img_height - 1): # _parity_check_ct_X
sum_top = np_sum(sample[1:int(img_height / 2), x])
constraints[x +
curr_index] = int(sample[0][x][0] != sum_top % 2)
curr_index += img_height - 2
for x in prange(1, img_width - 1): # _parity_check_rr_X
sum_right = np_sum(sample[x, int(img_width / 2):img_width - 1])
constraints[x + curr_index] = int(
sample[x][img_width - 1][0] != sum_right % 2)
curr_index += img_width - 2
for x in prange(1, img_height - 1): # _parity_check_cb_X
sum_bottom = np_sum(sample[int(img_height / 2):img_height - 1,
x])
constraints[x + curr_index] = int(
sample[img_height - 1][x][0] != sum_bottom % 2)
R_mb_constraints_values[i][j] = constraints
return R_mb_constraints_values
def calcSigma(self,func,*args,deconv=False):
if deconv:
self.sigmaDeconv=np_empty(self.transDeconv.shape,dtype=complex)
for i in range(self.tPumpDeconv.size):
self.sigmaDeconv[i,:]=func(self.transDeconv[i,:],*args)
else:
self.sigma=np_empty(self.trans.shape,dtype=complex)
for i in range(self.tPump.size):
self.sigma[i,:]=func(self.trans[i,:],*args)
return
def set_data(self):
data_len = self.col_mat.shape[0]*self.col_mat.shape[1]
self.pos = np_empty((data_len, 3))
self.size = np_ones((data_len))*.1
self.color = np_empty((data_len, 4))
jj = 0
for ii in range(len(self.y)):
for zz in range(len(self.z)):
self.pos[jj,:] = np_array((self.x[ii]/self.kx,self.y[ii]/self.ky,zz/self.kz))
self.color[jj,:] = self.col_mat[ii,zz]/255.
jj = jj + 1
def set_data(self):
data_len = self.col_mat.shape[0] * self.col_mat.shape[1]
self.pos = np_empty((data_len, 3))
self.size = np_ones((data_len)) * .1
self.color = np_empty((data_len, 4))
jj = 0
for ii in range(len(self.y)):
for zz in range(len(self.z)):
self.pos[jj, :] = np_array(
(self.x[ii] / self.kx, self.y[ii] / self.ky, zz / self.kz))
self.color[jj, :] = self.col_mat[ii, zz] / 255.
jj = jj + 1
def write_spectra_params(self):
""" Writes spectra_params file (input for latm executable) """
from numpy import empty as np_empty
from numpy import int as np_int
# Get variables from input variables:
n_component = len(self.components)
resp_name = response_dict[self.response]
# spectra.params file:
filename = self.dirname + "/" + self.spectra_params_fname
f = open(filename, "w")
f.write("{0} {1}\n".format(n_component, self.static))
for ii in range(n_component):
# Map component 'xyz' to digits '123'
cc = list(self.components[ii])
cci = np_empty(shape=(len(cc)), dtype=np_int)
for jj in range(len(cc)):
cci[jj] = component_dict[cc[jj]]
#
resp_filename = resp_name + "." + self.components[
ii] + ".dat_" + self.case
file_num = 501 + ii
f.write("%i %s %i T\n" % (self.response, resp_filename, file_num))
for jj in range(len(cc)):
f.write("%i " % (cci[jj]))
f.write("\n")
f.close()
def update_color_peak_image(self, color, index): img = np_empty((COLOR_PEAK_COLOR_PREVIEW_IMAGE_HEIGHT, COLOR_PEAK_COLOR_PREVIEW_IMAGE_WIDTH, 3), dtype=np_uint8) for i in range(3): img[:,:,i] = color.channels[i] tkimg = get_tkimg_from_nparr(img) if index == 1: self.color_peak_left_img = tkimg self.color_peak_left_lbl.configure(image=self.color_peak_left_img) elif index == 2: self.color_peak_right_img = tkimg self.color_peak_right_lbl.configure(image=self.color_peak_right_img)
def np_empty_mock(shape, dtype):
"""mock numpy empty to return an initialised array with all hours in 2020 if shape is 365 and dtype.kind is M.
The objective is to simulate a numpy.empty that returns an uninitialized array with random content that
can cause issues when tz_localize is applied with a timezone with DST"""
import numpy
dtype = numpy.dtype(dtype)
if shape == 8784 and dtype.kind == "M":
a = numpy.arange(start="2020-01-01", stop="2021-01-01",
dtype="M8[h]").astype(dtype)
else:
a = np_empty(shape, dtype)
return a
def __init__(self, tree, parent, sax, cardinality):
"""
Initialization function of the rootnode class
:returns: a root node
:rtype: RootNode
"""
self.iSAX_word = np_array([sax, cardinality]).T
Node.__init__(self, parent=parent, name=str(self.iSAX_word))
self.tree = tree
self.sax = sax
self.cardinality = cardinality
self.cardinality_next = np_copy(self.cardinality)
self.cardinality_next = np_array([x*2 for x in self.cardinality_next])
# Number of sequences contained in the node (or by its sons)
self.nb_sequences = 0
""" The incremental computing part for CFOF """
self.mean = np_empty(shape=self.tree.size_word)
# Allows the incremental calculation of self.mean
self.sum = np_empty(shape=self.tree.size_word)
self.std = np_empty(shape=self.tree.size_word)
# Allows the incremental calculation of self.std
self.sn = np_empty(shape=self.tree.size_word)
# Specific to internal nodes
self.nodes = []
self.key_nodes = {}
self.terminal = False
self.level = 0
self.id = RootNode.id_global
RootNode.id_global += 1
def area_and_convexity_multi_batch(R_mb,
n_samples=None,
bs=None,
num_constraints=None,
area=None,
polygons_number=None,
img_width=None,
img_height=None,
**kwargs):
# we have (n_samples x bs) generated items; for each of them we have to
# compute the values of each constraint.
# R_mb has shape [n_samples, bs] + image_shape
# e.g. [n_samples, bs, 20, 20, 1]
# so we need to iterate over the first two dimensions
# preallocate an output array that will contain the computed
# constraints values for the group of batches
R_mb_constraints_values = np_empty(shape=(n_samples, bs, num_constraints),
dtype=np_float32)
for i in prange(R_mb.shape[0]):
for j in prange(R_mb.shape[1]):
sample = R_mb[i][j]
# preallocate an output array that will contain the computed
# constraints value for the i-th element
constraints = np_empty(shape=(num_constraints, ), dtype=np_float32)
target_area = area * polygons_number
nonzero = np_sum(sample)
norm = img_width * img_height - target_area
greater_area_inner = min(1, max(0, nonzero - target_area) / norm)
smaller_area_inner = min(1, max(0, target_area - nonzero) / norm)
constraints[0] = greater_area_inner
constraints[1] = smaller_area_inner
# convexity
constraints[2] = _convex(sample, img_width, img_height)
R_mb_constraints_values[i][j] = constraints
return R_mb_constraints_values
def Skew(x,y,dat,noise=3):
interp=sp_interp2d(x,y,dat)
dx=x[1]-x[0]
ySkew=np_arange(np_amin(y)-dx*x.size,np_amax(y),dx)
DAT=np_empty((ySkew.size,x.size))
yMax=np_amax(y); yMin=np_amin(y)
for i in range(ySkew.size):
for j in range(x.size):
if ySkew[i]+j*dx > yMax or ySkew[i]+j*dx < yMin:
DAT[i,j]=(np_rand(1)-0.5)*noise
else:
DAT[i,j]=interp(x[j],ySkew[i]+j*dx)
return ySkew,DAT
def shrink(img, xTimes, yTimes):
'''
shrink gray scale
:param img:
:param xTimes:
:param yTimes:
:return:
'''
(h, w) = img.shape
tW = math.ceil(w / xTimes)
tH = math.ceil(h / yTimes)
# 1维寻址快
tImg = np_empty([tH, tW], dtype='uint8')
sBuf = img.reshape((-1))
tBuf = tImg.reshape((-1))
tCur = tW * tH
sCur = w * h
while (tCur > 0):
tX = tW
lineCur = sCur
if (xTimes > 1):
# undebug
while (tX > 0):
tCur -= 1
sCur -= xTimes
tBuf[tCur] = sBuf[sCur]
tX -= 1
# tCur = lineCur - tW * yTimes
sCur = lineCur - w * yTimes
else:
tBuf[tCur - tW:tCur] = sBuf[sCur - tW:sCur]
sCur -= tW * yTimes
tCur -= tW
return tImg
def load(satscene,
calibrate=True,
area_extent=None,
read_basic_or_detailed='both',
**kwargs):
"""Load MSG SEVIRI High Resolution Wind (HRW) data from hdf5 format.
"""
# Read config file content
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
values = {
"orbit": satscene.orbit,
"satname": satscene.satname,
"number": satscene.number,
"instrument": satscene.instrument_name,
"satellite": satscene.fullname
}
LOG.info("assume seviri-level5")
print "... assume seviri-level5"
satscene.add_to_history("hdf5 data read by mpop/nwcsaf_hrw_hdf.py")
# end of scan time 4 min after start
end_time = satscene.time_slot + datetime.timedelta(minutes=4)
# area !!! satscene.area
filename = os.path.join(
satscene.time_slot.strftime(conf.get("seviri-level5", "dir",
raw=True)),
satscene.time_slot.strftime(
conf.get("seviri-level5", "filename", raw=True)) % values)
# define classes before we search for files (in order to return empty class if no file is found)
HRW_basic = HRW_class()
HRW_basic.detailed = False
HRW_basic.date = satscene.time_slot
HRW_detailed = HRW_class()
HRW_detailed.detailed = True
HRW_detailed.date = satscene.time_slot
print "... search for file: ", filename
filenames = glob(str(filename))
if len(filenames) != 0:
if len(filenames) > 1:
print "*** Warning, more than 1 datafile found: ", filenames
filename = filenames[0]
print("... read data from %s" % str(filename))
# create an instant of the HRW_class
m_per_s_to_knots = 1.944
## limit channels to read
#hrw_channels=['HRV']
# limit basic or detailed or both
#read_basic_or_detailed='detailed'
#read_basic_or_detailed='basic'
with h5py.File(filename, 'r') as hf:
#print hf.attrs.keys()
#print hf.attrs.values()
region_name = hf.attrs['REGION_NAME'].replace("_", "")
print "... read HRW data for region ", region_name
LOG.info("... read HRW data for region " + region_name)
sat_ID = GP_IDs[int(hf.attrs["GP_SC_ID"])]
print "... derived from Meteosat ", sat_ID
LOG.info("... derived from Meteosat " + sat_ID)
# print('List of arrays in this file: \n', hf.keys()), len(hf.keys())
if len(hf.keys()) == 0:
print "*** Warning, empty file ", filename
print ""
else:
for key in hf.keys():
if key[4:9] == "BASIC":
if 'read_basic_or_detailed' in locals():
if read_basic_or_detailed.lower() == "detailed":
continue
HRW_data = HRW_basic # shallow copy
elif key[4:12] == "DETAILED":
if 'read_basic_or_detailed' in locals():
if read_basic_or_detailed.lower() == "basic":
continue
HRW_data = HRW_detailed # shallow copy
hrw_chn = dict_channel[key[len(key) - 9:]]
if 'hrw_channels' in locals():
if hrw_channels != None:
if hrw_chn not in hrw_channels:
print "... " + hrw_chn + " is not in hrw_channels", hrw_channels
print " skip reading this channel"
continue
# read all data
channel = hf.get(key)
# print '... read wind vectors of channel ', channel.name, hrw_chn
# print " i lon lat speed[kn] dir pressure"
#for i in range(channel.len()):
# print '%3d %10.7f %10.7f %7.2f %7.1f %8.1f' % (channel[i]['wind_id'], channel[i]['lon'], channel[i]['lat'], \
# channel[i]['wind_speed']*m_per_s_to_knots, \
# channel[i]['wind_direction'], channel[i]['pressure'])
# create string array with channel names
channel_chararray = np_empty(channel.len(), dtype='|S6')
channel_chararray[:] = hrw_chn
HRW_data.channel = np_append(HRW_data.channel,
channel_chararray)
HRW_data.wind_id = np_append(HRW_data.wind_id,
channel[:]['wind_id'])
HRW_data.prev_wind_id = np_append(
HRW_data.prev_wind_id, channel[:]['prev_wind_id'])
HRW_data.segment_X = np_append(HRW_data.segment_X,
channel[:]['segment_X'])
HRW_data.segment_Y = np_append(HRW_data.segment_Y,
channel[:]['segment_Y'])
HRW_data.t_corr_method = np_append(
HRW_data.t_corr_method, channel[:]['t_corr_method'])
HRW_data.lon = np_append(HRW_data.lon, channel[:]['lon'])
HRW_data.lat = np_append(HRW_data.lat, channel[:]['lat'])
HRW_data.dlon = np_append(HRW_data.dlon,
channel[:]['dlon'])
HRW_data.dlat = np_append(HRW_data.dlat,
channel[:]['dlat'])
HRW_data.pressure = np_append(HRW_data.pressure,
channel[:]['pressure'])
HRW_data.wind_speed = np_append(HRW_data.wind_speed,
channel[:]['wind_speed'])
HRW_data.wind_direction = np_append(
HRW_data.wind_direction, channel[:]['wind_direction'])
HRW_data.temperature = np_append(HRW_data.temperature,
channel[:]['temperature'])
HRW_data.conf_nwp = np_append(HRW_data.conf_nwp,
channel[:]['conf_nwp'])
HRW_data.conf_no_nwp = np_append(HRW_data.conf_no_nwp,
channel[:]['conf_no_nwp'])
HRW_data.t_type = np_append(HRW_data.t_type,
channel[:]['t_type'])
HRW_data.t_level_method = np_append(
HRW_data.t_level_method, channel[:]['t_level_method'])
HRW_data.t_winds = np_append(HRW_data.t_winds,
channel[:]['t_winds'])
HRW_data.t_corr_test = np_append(HRW_data.t_corr_test,
channel[:]['t_corr_test'])
HRW_data.applied_QI = np_append(HRW_data.applied_QI,
channel[:]['applied_QI'])
HRW_data.NWP_wind_levels = np_append(
HRW_data.NWP_wind_levels,
channel[:]['NWP_wind_levels'])
HRW_data.num_prev_winds = np_append(
HRW_data.num_prev_winds, channel[:]['num_prev_winds'])
HRW_data.orographic_index = np_append(
HRW_data.orographic_index,
channel[:]['orographic_index'])
HRW_data.cloud_type = np_append(HRW_data.cloud_type,
channel[:]['cloud_type'])
HRW_data.wind_channel = np_append(
HRW_data.wind_channel, channel[:]['wind_channel'])
HRW_data.correlation = np_append(HRW_data.correlation,
channel[:]['correlation'])
HRW_data.pressure_error = np_append(
HRW_data.pressure_error, channel[:]['pressure_error'])
# sort according to wind_id
inds = HRW_data.wind_id.argsort()
HRW_data.subset(inds) # changes HRW_data itself
# sorting without conversion to numpy arrays
#[e for (wid,pwid) in sorted(zip(HRW_data.wind_id,HRW_data.prev_wind_id))]
else:
print "*** Error, no file found"
print ""
sat_ID = "no file"
# but we continue the program in order to add an empty channel below
## filter data according to the given optional arguments
#n1 = str(HRW_data.channel.size)
#HRW_data = HRW_data.filter(**kwargs)
#print " apply filters "+' ('+n1+'->'+str(HRW_data.channel.size)+')'
chn_name = "HRW"
satscene[chn_name].HRW_basic = HRW_basic.filter(
**kwargs) # returns new object (deepcopied and filtered)
satscene[chn_name].HRW_detailed = HRW_detailed.filter(
**kwargs) # returns new object (deepcopied and filtered)
satscene[chn_name].info['units'] = 'm/s'
satscene[chn_name].info['satname'] = 'meteosat'
satscene[chn_name].info['satnumber'] = sat_ID
satscene[chn_name].info['instrument_name'] = 'seviri'
satscene[chn_name].info['time'] = satscene.time_slot
satscene[chn_name].info['is_calibrated'] = True
def load(satscene, calibrate=True, area_extent=None, read_basic_or_detailed='both', **kwargs):
"""Load MSG SEVIRI High Resolution Wind (HRW) data from hdf5 format.
"""
# Read config file content
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
values = {"orbit": satscene.orbit,
"satname": satscene.satname,
"number": satscene.number,
"instrument": satscene.instrument_name,
"satellite": satscene.fullname
}
LOG.info("assume seviri-level5")
print "... assume seviri-level5"
satscene.add_to_history("hdf5 data read by mpop/nwcsaf_hrw_hdf.py")
# end of scan time 4 min after start
end_time = satscene.time_slot + datetime.timedelta(minutes=4)
# area !!! satscene.area
filename = os.path.join( satscene.time_slot.strftime(conf.get("seviri-level5", "dir", raw=True)),
satscene.time_slot.strftime(conf.get("seviri-level5", "filename", raw=True)) % values )
# define classes before we search for files (in order to return empty class if no file is found)
HRW_basic = HRW_class()
HRW_basic.detailed = False
HRW_basic.date = satscene.time_slot
HRW_detailed = HRW_class()
HRW_detailed.detailed = True
HRW_detailed.date = satscene.time_slot
print "... search for file: ", filename
filenames=glob(str(filename))
if len(filenames) != 0:
if len(filenames) > 1:
print "*** Warning, more than 1 datafile found: ", filenames
filename = filenames[0]
print("... read data from %s" % str(filename))
# create an instant of the HRW_class
m_per_s_to_knots = 1.944
## limit channels to read
#hrw_channels=['HRV']
# limit basic or detailed or both
#read_basic_or_detailed='detailed'
#read_basic_or_detailed='basic'
with h5py.File(filename,'r') as hf:
#print hf.attrs.keys()
#print hf.attrs.values()
region_name = hf.attrs['REGION_NAME'].replace("_", "")
print "... read HRW data for region ", region_name
LOG.info("... read HRW data for region "+region_name)
sat_ID = GP_IDs[int(hf.attrs["GP_SC_ID"])]
print "... derived from Meteosat ", sat_ID
LOG.info("... derived from Meteosat "+sat_ID)
# print('List of arrays in this file: \n', hf.keys()), len(hf.keys())
if len(hf.keys()) == 0:
print "*** Warning, empty file ", filename
print ""
else:
for key in hf.keys():
if key[4:9] == "BASIC":
if 'read_basic_or_detailed' in locals():
if read_basic_or_detailed.lower() == "detailed":
continue
HRW_data = HRW_basic # shallow copy
elif key[4:12] == "DETAILED":
if 'read_basic_or_detailed' in locals():
if read_basic_or_detailed.lower() == "basic":
continue
HRW_data = HRW_detailed # shallow copy
hrw_chn = dict_channel[key[len(key)-9:]]
if 'hrw_channels' in locals():
if hrw_channels != None:
if hrw_chn not in hrw_channels:
print "... "+hrw_chn+" is not in hrw_channels", hrw_channels
print " skip reading this channel"
continue
# read all data
channel = hf.get(key)
# print '... read wind vectors of channel ', channel.name, hrw_chn
# print " i lon lat speed[kn] dir pressure"
#for i in range(channel.len()):
# print '%3d %10.7f %10.7f %7.2f %7.1f %8.1f' % (channel[i]['wind_id'], channel[i]['lon'], channel[i]['lat'], \
# channel[i]['wind_speed']*m_per_s_to_knots, \
# channel[i]['wind_direction'], channel[i]['pressure'])
# create string array with channel names
channel_chararray = np_empty(channel.len(), dtype='|S6')
channel_chararray[:] = hrw_chn
HRW_data.channel = np_append(HRW_data.channel , channel_chararray )
HRW_data.wind_id = np_append(HRW_data.wind_id , channel[:]['wind_id'] )
HRW_data.prev_wind_id = np_append(HRW_data.prev_wind_id , channel[:]['prev_wind_id'] )
HRW_data.segment_X = np_append(HRW_data.segment_X , channel[:]['segment_X'] )
HRW_data.segment_Y = np_append(HRW_data.segment_Y , channel[:]['segment_Y'] )
HRW_data.t_corr_method = np_append(HRW_data.t_corr_method , channel[:]['t_corr_method'] )
HRW_data.lon = np_append(HRW_data.lon , channel[:]['lon'] )
HRW_data.lat = np_append(HRW_data.lat , channel[:]['lat'] )
HRW_data.dlon = np_append(HRW_data.dlon , channel[:]['dlon'] )
HRW_data.dlat = np_append(HRW_data.dlat , channel[:]['dlat'] )
HRW_data.pressure = np_append(HRW_data.pressure , channel[:]['pressure'] )
HRW_data.wind_speed = np_append(HRW_data.wind_speed , channel[:]['wind_speed'] )
HRW_data.wind_direction = np_append(HRW_data.wind_direction , channel[:]['wind_direction'] )
HRW_data.temperature = np_append(HRW_data.temperature , channel[:]['temperature'] )
HRW_data.conf_nwp = np_append(HRW_data.conf_nwp , channel[:]['conf_nwp'] )
HRW_data.conf_no_nwp = np_append(HRW_data.conf_no_nwp , channel[:]['conf_no_nwp'] )
HRW_data.t_type = np_append(HRW_data.t_type , channel[:]['t_type'] )
HRW_data.t_level_method = np_append(HRW_data.t_level_method , channel[:]['t_level_method'] )
HRW_data.t_winds = np_append(HRW_data.t_winds , channel[:]['t_winds'] )
HRW_data.t_corr_test = np_append(HRW_data.t_corr_test , channel[:]['t_corr_test'] )
HRW_data.applied_QI = np_append(HRW_data.applied_QI , channel[:]['applied_QI'] )
HRW_data.NWP_wind_levels = np_append(HRW_data.NWP_wind_levels , channel[:]['NWP_wind_levels'] )
HRW_data.num_prev_winds = np_append(HRW_data.num_prev_winds , channel[:]['num_prev_winds'] )
HRW_data.orographic_index = np_append(HRW_data.orographic_index, channel[:]['orographic_index'] )
HRW_data.cloud_type = np_append(HRW_data.cloud_type , channel[:]['cloud_type'] )
HRW_data.wind_channel = np_append(HRW_data.wind_channel , channel[:]['wind_channel'] )
HRW_data.correlation = np_append(HRW_data.correlation , channel[:]['correlation'] )
HRW_data.pressure_error = np_append(HRW_data.pressure_error , channel[:]['pressure_error'] )
# sort according to wind_id
inds = HRW_data.wind_id.argsort()
HRW_data.subset(inds) # changes HRW_data itself
# sorting without conversion to numpy arrays
#[e for (wid,pwid) in sorted(zip(HRW_data.wind_id,HRW_data.prev_wind_id))]
else:
print "*** Error, no file found"
print ""
sat_ID = "no file"
# but we continue the program in order to add an empty channel below
## filter data according to the given optional arguments
#n1 = str(HRW_data.channel.size)
#HRW_data = HRW_data.filter(**kwargs)
#print " apply filters "+' ('+n1+'->'+str(HRW_data.channel.size)+')'
chn_name="HRW"
satscene[chn_name].HRW_basic = HRW_basic.filter(**kwargs) # returns new object (deepcopied and filtered)
satscene[chn_name].HRW_detailed = HRW_detailed.filter(**kwargs) # returns new object (deepcopied and filtered)
satscene[chn_name].info['units'] = 'm/s'
satscene[chn_name].info['satname'] = 'meteosat'
satscene[chn_name].info['satnumber'] = sat_ID
satscene[chn_name].info['instrument_name'] = 'seviri'
satscene[chn_name].info['time'] = satscene.time_slot
satscene[chn_name].info['is_calibrated'] = True
def __init__(self,
size_word,
threshold,
data_ts,
base_cardinality=2,
max_card_alphabet=128,
boolean_card_max=True):
"""
Class initialization function TreeISAX
:returns: a tree iSAX
:rtype: TreeISAX
"""
# Number of letters contained in the SAX words indexed in the tree
self.size_word = size_word
# Threshold before the separation of a sheet into two leaf nodes
self.threshold = threshold
# Cardinality of each letter at level 1 of the tree
# is not very useful ...
self._base_cardinality = base_cardinality
# Current Max Cardinality
self.bigger_current_cardinality = base_cardinality
# If true, defined maximum cardinality for the alphabet of the tree
self.boolean_card_max = boolean_card_max
self.max_card_alphabet = max_card_alphabet
# mean, standard deviation of data_ts sequences
self.mu, self.sig = norm.fit(data_ts)
self.min_max = np_empty(shape=(self.size_word, 2))
for i, dim in enumerate(np_array(data_ts).T.tolist()):
self.min_max[i][0] = min(dim) - self.mu
self.min_max[i][1] = max(dim) + self.mu
self.isax = IndexableSymbolicAggregateApproximation(self.size_word,
mean=self.mu,
std=self.sig)
# verif if all values are properly indexable
tmp_max = np_max(abs(np_array(data_ts) - self.mu))
bkpt_max = abs(
self.isax._card_to_bkpt(self.max_card_alphabet)[-1] - self.mu)
if tmp_max > bkpt_max:
ratio = tmp_max / bkpt_max
self.isax = IndexableSymbolicAggregateApproximation(self.size_word,
mean=self.mu,
std=self.sig *
ratio)
# and we transmit all this at the root node
self.root = RootNode(
tree=self,
parent=None,
sax=[0] * self.size_word,
cardinality=np_array([int(self._base_cardinality / 2)] *
self.size_word))
self.num_nodes = 1
# Attributes for preprocessing
self._minmax_nodes_computed = False
self.node_list = None
self.node_list_leaf = None
self.node_leaf_ndarray_mean = None
# Attributes for *i*\ CFOF computation
self._preprocessing_computed = False
self.min_array = None
self.max_array = None
self.min_array_leaf = None
self.max_array_leaf = None
self.cdf_mean = None
self.cdf_std = None
# Boolean value passing True after an update of the tree
self._new_insertion_after_preproc = False
self._new_insertion_after_minmax_nodes = False
def kappa(y_true, y_pred, weights=None, allow_off_by_one=False,
min_rating=None,
max_rating=None):
"""
Calculates the kappa inter-rater agreement between two the gold standard
and the predicted ratings. Potential values range from -1 (representing
complete disagreement) to 1 (representing complete agreement). A kappa
value of 0 is expected if all agreement is due to chance.
In the course of calculating kappa, all items in `y_true` and `y_pred` will
first be converted to floats and then rounded to integers.
It is assumed that y_true and y_pred contain the complete range of possible
ratings.
This function contains a combination of code from yorchopolis's kappa-stats
and Ben Hamner's Metrics projects on Github.
:param y_true: The true/actual/gold labels for the data.
:type y_true: array-like of float
:param y_pred: The predicted/observed labels for the data.
:type y_pred: array-like of float
:param weights: Specifies the weight matrix for the calculation.
Options are:
- None = unweighted-kappa
- 'quadratic' = quadratic-weighted kappa
- 'linear' = linear-weighted kappa
- two-dimensional numpy array = a custom matrix of
weights. Each weight corresponds to the
:math:`w_{ij}` values in the wikipedia description
of how to calculate weighted Cohen's kappa.
:type weights: str or numpy array
:param allow_off_by_one: If true, ratings that are off by one are counted as
equal, and all other differences are reduced by
one. For example, 1 and 2 will be considered to be
equal, whereas 1 and 3 will have a difference of 1
for when building the weights matrix.
:type allow_off_by_one: bool
"""
from numpy import round as np_round
from numpy import empty as np_empty
from numpy import bincount,outer,count_nonzero
# Ensure that the lists are both the same length
assert(len(y_true) == len(y_pred))
y_true = np_round(y_true).astype(int)
y_pred = np_round(y_pred).astype(int)
# Figure out normalized expected values
if min_rating is None:
min_rating = min(min(y_true), min(y_pred))
if max_rating is None:
max_rating = max(max(y_true), max(y_pred))
# shift the values so that the lowest value is 0
# (to support scales that include negative values)
y_true = y_true - min_rating
y_pred = y_pred - min_rating
# Build the observed/confusion matrix
num_ratings = max_rating - min_rating + 1
#from sklearn.metrics import confusion_matrix
observed = confusion_matrix(y_true, y_pred,
labels=list(range(num_ratings)))
num_scored_items = float(len(y_true))
# Build weight array if weren't passed one
from six import string_types
if isinstance(weights, string_types):
wt_scheme = weights
weights = None
else:
wt_scheme = ''
if weights is None:
weights = np_empty((num_ratings, num_ratings))
for i in range(num_ratings):
for j in range(num_ratings):
diff = abs(i - j)
if allow_off_by_one and diff:
diff -= 1
if wt_scheme == 'linear':
weights[i, j] = diff
elif wt_scheme == 'quadratic':
weights[i, j] = diff ** 2
elif not wt_scheme: # unweighted
weights[i, j] = bool(diff)
else:
raise ValueError('Invalid weight scheme specified for '
'kappa: {}'.format(wt_scheme))
hist_true = bincount(y_true, minlength=num_ratings)
hist_true = hist_true[: num_ratings] / num_scored_items
hist_pred = bincount(y_pred, minlength=num_ratings)
hist_pred = hist_pred[: num_ratings] / num_scored_items
expected = outer(hist_true, hist_pred)
# Normalize observed array
observed = observed / num_scored_items
# If all weights are zero, that means no disagreements matter.
k = 1.0
if count_nonzero(weights):
k -= (sum(sum(weights * observed)) / sum(sum(weights * expected)))
return k
def transfer_learning(self, max_iterations, accepted_mean_square_error=0.1, batch_size=5000, learning_rate=1e-4):
# Adam optimizer optimizes the parameters (weights and biases) at the learning rate specified
output_network_optimizer = Adam(self.output_neural_net.parameters(), lr=learning_rate)
# This is the row count difference between that of input and output grids
row_difference = self.output_grid_dimension[0] - self.input_grid_dimension[0]
# This is the column count difference between that of input and output grids
column_difference = self.output_grid_dimension[1] - self.input_grid_dimension[1]
# We cycle through iterations of each batch of training the output neural network until the max iteration
for _ in range(0, max_iterations):
# Training list - each element in the list contains [input state for output neural net, target value]
training_list = []
# Shuffling through batches and then calculating the Mean square error for the entire batch
for batch in range(0, batch_size):
# Creating a matrix to hold the observation state of the input neural network map (0 or 1)
output_network_known_state = np_array([[randint(0, 1) for _ in range(0, self.output_grid_dimension[1])]
for _ in range(0, self.output_grid_dimension[0])])
# This is used to store the robot and target state of the input neural network
input_network_state = np_empty((self.network_robot_count+1)*2, dtype=np_uint8)
# Creating positions for all of the robots and target of the input neural network randomly
for i in range(0, (self.network_robot_count+1)*2, 2):
input_network_state[i] = randint(0, self.input_grid_dimension[0]-1)
input_network_state[i+1] = randint(0, self.input_grid_dimension[1]-1)
# Creates a backup copy of the input state
input_network_state_memory = input_network_state
# Sliding the input network state window over different sections of the output network state
for i in range(0, row_difference):
for j in range(0, column_difference):
# Creating a matrix to hold the observation state of the input neural network map (0 or 1)
input_network_known_state = output_network_known_state[i:(i+self.input_grid_dimension[0]),
j:(j+self.input_grid_dimension[1])]
# This is used to store the robot and target state of the output neural network
output_network_state = np_empty((self.network_robot_count+1)*2, dtype=np_uint8)
# Extending the input position states across the moving window within the output grid dimensions
for k in range(0, (self.network_robot_count+1)*2, 2):
output_network_state[k] = input_network_state[k]+i
output_network_state[k+1] = input_network_state[k+1]+j
# Now we flatten data in the input network state, a 2-D matrix to 1-D and append
input_network_state = np_append(input_network_state,
np_ndarray.flatten(input_network_known_state))
# Now we flatten data in the output network state, a 2-D matrix to 1-D and append
output_network_state = np_append(output_network_state,
np_ndarray.flatten(output_network_known_state))
# Looping through the 4 possible actions each robot can take and then appending them to state
for k in range(0, 4):
# Adding an action completes the input state for the input neural network
input_network_state_tensor = Tensor(np_append(input_network_state, k))
# Adding an action completes the input state for the output neural network
output_network_state_tensor = Tensor(np_append(output_network_state, k))
# Getting the Q value predicted by the input neural network for the given state
input_network_predicted_value = self.input_neural_net.forward(input_network_state_tensor)
# Now we know the value the output neural network is to be trained towards for its given
# input. Add both of them to the training list so that batch training can occur later
training_list.append([output_network_state_tensor, input_network_predicted_value])
# Restoring the input state from memory
input_network_state = input_network_state_memory
# Shuffling the training data before feeding it in for training
shuffle(training_list)
# Initializing the current MSE loss
sum_square_error = 0.0
# Using the batch of state and target data for training the output neural network
for batch in range(0, batch_size):
# Obtaining the completed input states for the output neural network
output_network_state_tensor = training_list[batch][0]
# Obtaining the target predictions that the output neural network should be trained towards
predicted_target_value = training_list[batch][1]
# Getting the Q value predicted by the output neural network for the given input state
output_network_predicted_value = self.output_neural_net.forward(output_network_state_tensor)
# Adding the current square error to the sum of square errors
sum_square_error += pow((output_network_predicted_value - predicted_target_value), 2)
# Represents the function that can calculate training error
training_error_function = MSELoss()
# Our goal is to reduce the mean square error loss between the target prediction and that of network
training_error = training_error_function(output_network_predicted_value, predicted_target_value)
# Clears the gradients of all optimized torch tensors
output_network_optimizer.zero_grad()
# During the backwards pass, gradients from each replica are summed into the original module
training_error.backward()
# Training actually happens here. Performs a single optimization step of weights and biases
output_network_optimizer.step()
# Dividing the sum of square errors by the batch size to get the mean square error
current_mean_square_error = sum_square_error/batch_size
print(current_mean_square_error)
# Checks if the MSE for the entire batch is within acceptable levels and then returns the output neural net
if current_mean_square_error <= accepted_mean_square_error:
# we return a list where true indicates that we achieved the accepted mean square error criteria
return [self.output_neural_net, True]
# Failed to completely train the output neural network. Return a list with second element false to indicate this
return [self.output_neural_net, False]
def _hyperrectangle_integration(lower,
upper,
correlation,
maxpts,
abseps,
releps,
info=False):
# main differences with _parallel_CDF is that it handles complex lower/upper bounds
# with infinite components and it returns information on the completion.
d = correlation.size(-1)
trind = tril_indices(d, -1)
### Infere batch_shape
lnone = lower is None
unone = upper is None
bothNone = lnone and unone
if bothNone:
pre_batch_shape = []
# broadcast lower and upper to get pre_batch_shape
elif not lnone and not unone:
pre_batch_shape = broadcast(lower, upper).shape[:-1]
cdf = False
else: # case were we compute P(Y<x): lower is [-inf, ..., -inf] and upper = x.
# Invert lower and upper if it is upper=None
cdf = True
if unone:
upper = -lower
pre_batch_shape = upper.shape[:-1]
cor = ascontiguousarray(correlation.numpy()[..., trind[0], trind[1]],
dtype=float64)
# broadcast all lower, upper, correlation
batch_shape = broadcast_shape(pre_batch_shape, cor.shape[:-1])
dtype = correlation.dtype
device = correlation.device
if bothNone: # trivial case
if info:
return (torch_ones(*batch_shape, dtype=dtype, device=device),
torch_zeros(*batch_shape, dtype=dtype, device=device),
torch_zeros(*batch_shape, dtype=torch_int32,
device=device))
else:
return torch_ones(*batch_shape, dtype=dtype, device=device)
else:
if d == 1:
val = Phi(upper.squeeze(-1)) if cdf else Phi(
upper.squeeze(-1)) - Phi(lower.squeeze(-1))
if info:
return (val,
torch_zeros(*batch_shape, dtype=dtype, device=device),
torch_zeros(*batch_shape,
dtype=torch_int32,
device=device))
else:
return val
dd = d * (d - 1) // 2 # size of flatten correlation matrix
# Broadcast:
c = broadcast_to(cor, batch_shape + [dd]).reshape(-1, dd)
N = c.shape[0] # batch number
upp = upper.numpy().astype(float64)
shape1 = batch_shape + [d]
u = broadcast_to(upp, shape1).reshape(N, d)
infu = u == Inf
if cdf:
l = np_empty(
(N, d),
dtype=float64) # never used but required by Fortran code
i = np_zeros((N, d), dtype=int32)
i.setflags(write=1)
i[infu] = -1 # basically ignores these componenents
else:
low = lower.numpy().astype(float64)
l = broadcast_to(low, shape1).reshape(N, d)
i = full((N, d), 2, dtype=int32)
infl = l == -Inf
i.setflags(write=1)
i[infl] = 0
i[infu] = 1
i[infl * infu] = -1 # basically ignores these componenents
# infin is a int vector to pass to the fortran code controlling the integral limits
# if INFIN(I) < 0, Ith limits are (-infinity, infinity);
# if INFIN(I) = 0, Ith limits are (-infinity, UPPER(I)];
# if INFIN(I) = 1, Ith limits are [LOWER(I), infinity);
# if INFIN(I) = 2, Ith limits are [LOWER(I), UPPER(I)].
# TODO better to build res and assign or build-reshap?
res = _parallel_genz_bretz(l, u, i, c, maxpts, abseps, releps, info)
if info:
values, errors, infos = res
return (tensor(values, dtype=dtype, device=device).view(batch_shape),
tensor(errors, dtype=dtype, device=device).view(batch_shape),
tensor(infos, dtype=torch_int32,
device=device).view(batch_shape))
else:
return tensor(res, dtype=dtype, device=device).view(batch_shape)
""" l.setflags(write=1)
