def area_and_parity_check_single_batch(data_mb, num_constraints=None, area=None, polygons_number=None, img_width=None, img_height=None, **kwargs): # preallocate an output array that will contain the computed # constraints values for the batch mb_constraints_values = np_empty(shape=(data_mb.shape[0], num_constraints), dtype=np_float32) for i in prange(data_mb.shape[0]): sample = data_mb[i] # preallocate an output array that will contain the computed # constraints value for the i-th element constraints = np_empty(shape=(num_constraints, ), dtype=np_float32) target_area = area * polygons_number nonzero = np_sum(sample[1:-1, 1:-1]) norm = img_width * img_height - target_area greater_area_inner = min(1, max(0, nonzero - target_area) / norm) smaller_area_inner = min(1, max(0, target_area - nonzero) / norm) constraints[0] = greater_area_inner constraints[1] = smaller_area_inner for x in prange(1, 10): # _parity_check_rl_X sum_left = np_sum(sample[x, 1:int(img_width / 2)]) constraints[x + 1] = int(sample[x][0][0] != sum_left % 2) for x in prange(1, 10): # _parity_check_ct_X sum_top = np_sum(sample[1:int(img_height / 2), x]) constraints[x + 10] = int(sample[0][x][0] != sum_top % 2) mb_constraints_values[i] = constraints return mb_constraints_values
def area_and_convexity_single_batch(data_mb, num_constraints=None, area=None, polygons_number=None, img_width=None, img_height=None, **kwargs): # preallocate an output array that will contain the computed # constraints values for the batch mb_constraints_values = np_empty(shape=(data_mb.shape[0], num_constraints), dtype=np_float32) for i in prange(data_mb.shape[0]): sample = data_mb[i] # preallocate an output array that will contain the computed # constraints value for the i-th element constraints = np_empty(shape=(num_constraints, ), dtype=np_float32) target_area = area * polygons_number nonzero = np_sum(sample) norm = img_width * img_height - target_area greater_area_inner = min(1, max(0, nonzero - target_area) / norm) smaller_area_inner = min(1, max(0, target_area - nonzero) / norm) constraints[0] = greater_area_inner constraints[1] = smaller_area_inner # convexity constraints[2] = _convex(sample, img_width, img_height) mb_constraints_values[i] = constraints return mb_constraints_values
def _do_bkpt(self): """ The _do_bkpt function calculates the min and max terminals of the node on each dimension of the node. :returns: an array containing the min terminals and one containing the max terminals :rtype: numpy.array, numpy.array """ bkpt_list_min = np_empty(self.tree.size_word) bkpt_list_max = np_empty(self.tree.size_word) for i, iSAX_letter in enumerate(self.iSAX_word): bkpt_tmp = self.tree.isax._card_to_bkpt(iSAX_letter[1]) # The case where there is no BKPT (root node) if iSAX_letter[1] < 2: bkpt_list_min[i] = self.tree.min_max[i][0] bkpt_list_max[i] = self.tree.min_max[i][1] # the case where there is no BKPT inf elif iSAX_letter[0] == 0: bkpt_list_min[i] = self.tree.min_max[i][0] bkpt_list_max[i] = bkpt_tmp[iSAX_letter[0]] # the case where there is no BKPT sup elif iSAX_letter[0] == iSAX_letter[1]-1: bkpt_list_min[i] = bkpt_tmp[iSAX_letter[0]-1] bkpt_list_max[i] = self.tree.min_max[i][1] # The general case else: bkpt_list_min[i] = bkpt_tmp[iSAX_letter[0]-1] bkpt_list_max[i] = bkpt_tmp[iSAX_letter[0]] return bkpt_list_min, bkpt_list_max
def area_and_all_parity_check_multi_batch(R_mb, n_samples=None, bs=None, num_constraints=None, area=None, polygons_number=None, img_width=None, img_height=None, **kwargs): # we have (n_samples x bs) generated items; for each of them we have to # compute the values of each constraint. # R_mb has shape [n_samples, bs] + image_shape # e.g. [n_samples, bs, 20, 20, 1] # so we need to iterate over the first two dimensions # preallocate an output array that will contain the computed # constraints values for the group of batches R_mb_constraints_values = np_empty(shape=(n_samples, bs, num_constraints), dtype=np_float32) for i in prange(R_mb.shape[0]): for j in prange(R_mb.shape[1]): sample = R_mb[i][j] # preallocate an output array that will contain the computed # constraints value for the i-th element constraints = np_empty(shape=(num_constraints, ), dtype=np_float32) target_area = area * polygons_number nonzero = np_sum(sample[1:-1, 1:-1]) norm = img_width * img_height - target_area greater_area_inner = min(1, max(0, nonzero - target_area) / norm) smaller_area_inner = min(1, max(0, target_area - nonzero) / norm) constraints[0] = greater_area_inner constraints[1] = smaller_area_inner curr_index = 1 for x in prange(1, img_width - 1): # _parity_check_rl_X sum_left = np_sum(sample[x, 1:int(img_width / 2)]) constraints[x + curr_index] = int(sample[x][0][0] != sum_left % 2) curr_index += img_width - 2 for x in prange(1, img_height - 1): # _parity_check_ct_X sum_top = np_sum(sample[1:int(img_height / 2), x]) constraints[x + curr_index] = int(sample[0][x][0] != sum_top % 2) curr_index += img_height - 2 for x in prange(1, img_width - 1): # _parity_check_rr_X sum_right = np_sum(sample[x, int(img_width / 2):img_width - 1]) constraints[x + curr_index] = int( sample[x][img_width - 1][0] != sum_right % 2) curr_index += img_width - 2 for x in prange(1, img_height - 1): # _parity_check_cb_X sum_bottom = np_sum(sample[int(img_height / 2):img_height - 1, x]) constraints[x + curr_index] = int( sample[img_height - 1][x][0] != sum_bottom % 2) R_mb_constraints_values[i][j] = constraints return R_mb_constraints_values
def calcSigma(self,func,*args,deconv=False): if deconv: self.sigmaDeconv=np_empty(self.transDeconv.shape,dtype=complex) for i in range(self.tPumpDeconv.size): self.sigmaDeconv[i,:]=func(self.transDeconv[i,:],*args) else: self.sigma=np_empty(self.trans.shape,dtype=complex) for i in range(self.tPump.size): self.sigma[i,:]=func(self.trans[i,:],*args) return
def set_data(self): data_len = self.col_mat.shape[0]*self.col_mat.shape[1] self.pos = np_empty((data_len, 3)) self.size = np_ones((data_len))*.1 self.color = np_empty((data_len, 4)) jj = 0 for ii in range(len(self.y)): for zz in range(len(self.z)): self.pos[jj,:] = np_array((self.x[ii]/self.kx,self.y[ii]/self.ky,zz/self.kz)) self.color[jj,:] = self.col_mat[ii,zz]/255. jj = jj + 1
def set_data(self): data_len = self.col_mat.shape[0] * self.col_mat.shape[1] self.pos = np_empty((data_len, 3)) self.size = np_ones((data_len)) * .1 self.color = np_empty((data_len, 4)) jj = 0 for ii in range(len(self.y)): for zz in range(len(self.z)): self.pos[jj, :] = np_array( (self.x[ii] / self.kx, self.y[ii] / self.ky, zz / self.kz)) self.color[jj, :] = self.col_mat[ii, zz] / 255. jj = jj + 1
def write_spectra_params(self): """ Writes spectra_params file (input for latm executable) """ from numpy import empty as np_empty from numpy import int as np_int # Get variables from input variables: n_component = len(self.components) resp_name = response_dict[self.response] # spectra.params file: filename = self.dirname + "/" + self.spectra_params_fname f = open(filename, "w") f.write("{0} {1}\n".format(n_component, self.static)) for ii in range(n_component): # Map component 'xyz' to digits '123' cc = list(self.components[ii]) cci = np_empty(shape=(len(cc)), dtype=np_int) for jj in range(len(cc)): cci[jj] = component_dict[cc[jj]] # resp_filename = resp_name + "." + self.components[ ii] + ".dat_" + self.case file_num = 501 + ii f.write("%i %s %i T\n" % (self.response, resp_filename, file_num)) for jj in range(len(cc)): f.write("%i " % (cci[jj])) f.write("\n") f.close()
def update_color_peak_image(self, color, index): img = np_empty((COLOR_PEAK_COLOR_PREVIEW_IMAGE_HEIGHT, COLOR_PEAK_COLOR_PREVIEW_IMAGE_WIDTH, 3), dtype=np_uint8) for i in range(3): img[:,:,i] = color.channels[i] tkimg = get_tkimg_from_nparr(img) if index == 1: self.color_peak_left_img = tkimg self.color_peak_left_lbl.configure(image=self.color_peak_left_img) elif index == 2: self.color_peak_right_img = tkimg self.color_peak_right_lbl.configure(image=self.color_peak_right_img)
def np_empty_mock(shape, dtype): """mock numpy empty to return an initialised array with all hours in 2020 if shape is 365 and dtype.kind is M. The objective is to simulate a numpy.empty that returns an uninitialized array with random content that can cause issues when tz_localize is applied with a timezone with DST""" import numpy dtype = numpy.dtype(dtype) if shape == 8784 and dtype.kind == "M": a = numpy.arange(start="2020-01-01", stop="2021-01-01", dtype="M8[h]").astype(dtype) else: a = np_empty(shape, dtype) return a
def __init__(self, tree, parent, sax, cardinality): """ Initialization function of the rootnode class :returns: a root node :rtype: RootNode """ self.iSAX_word = np_array([sax, cardinality]).T Node.__init__(self, parent=parent, name=str(self.iSAX_word)) self.tree = tree self.sax = sax self.cardinality = cardinality self.cardinality_next = np_copy(self.cardinality) self.cardinality_next = np_array([x*2 for x in self.cardinality_next]) # Number of sequences contained in the node (or by its sons) self.nb_sequences = 0 """ The incremental computing part for CFOF """ self.mean = np_empty(shape=self.tree.size_word) # Allows the incremental calculation of self.mean self.sum = np_empty(shape=self.tree.size_word) self.std = np_empty(shape=self.tree.size_word) # Allows the incremental calculation of self.std self.sn = np_empty(shape=self.tree.size_word) # Specific to internal nodes self.nodes = [] self.key_nodes = {} self.terminal = False self.level = 0 self.id = RootNode.id_global RootNode.id_global += 1
def area_and_convexity_multi_batch(R_mb, n_samples=None, bs=None, num_constraints=None, area=None, polygons_number=None, img_width=None, img_height=None, **kwargs): # we have (n_samples x bs) generated items; for each of them we have to # compute the values of each constraint. # R_mb has shape [n_samples, bs] + image_shape # e.g. [n_samples, bs, 20, 20, 1] # so we need to iterate over the first two dimensions # preallocate an output array that will contain the computed # constraints values for the group of batches R_mb_constraints_values = np_empty(shape=(n_samples, bs, num_constraints), dtype=np_float32) for i in prange(R_mb.shape[0]): for j in prange(R_mb.shape[1]): sample = R_mb[i][j] # preallocate an output array that will contain the computed # constraints value for the i-th element constraints = np_empty(shape=(num_constraints, ), dtype=np_float32) target_area = area * polygons_number nonzero = np_sum(sample) norm = img_width * img_height - target_area greater_area_inner = min(1, max(0, nonzero - target_area) / norm) smaller_area_inner = min(1, max(0, target_area - nonzero) / norm) constraints[0] = greater_area_inner constraints[1] = smaller_area_inner # convexity constraints[2] = _convex(sample, img_width, img_height) R_mb_constraints_values[i][j] = constraints return R_mb_constraints_values
def Skew(x,y,dat,noise=3): interp=sp_interp2d(x,y,dat) dx=x[1]-x[0] ySkew=np_arange(np_amin(y)-dx*x.size,np_amax(y),dx) DAT=np_empty((ySkew.size,x.size)) yMax=np_amax(y); yMin=np_amin(y) for i in range(ySkew.size): for j in range(x.size): if ySkew[i]+j*dx > yMax or ySkew[i]+j*dx < yMin: DAT[i,j]=(np_rand(1)-0.5)*noise else: DAT[i,j]=interp(x[j],ySkew[i]+j*dx) return ySkew,DAT
def shrink(img, xTimes, yTimes): ''' shrink gray scale :param img: :param xTimes: :param yTimes: :return: ''' (h, w) = img.shape tW = math.ceil(w / xTimes) tH = math.ceil(h / yTimes) # 1维寻址快 tImg = np_empty([tH, tW], dtype='uint8') sBuf = img.reshape((-1)) tBuf = tImg.reshape((-1)) tCur = tW * tH sCur = w * h while (tCur > 0): tX = tW lineCur = sCur if (xTimes > 1): # undebug while (tX > 0): tCur -= 1 sCur -= xTimes tBuf[tCur] = sBuf[sCur] tX -= 1 # tCur = lineCur - tW * yTimes sCur = lineCur - w * yTimes else: tBuf[tCur - tW:tCur] = sBuf[sCur - tW:sCur] sCur -= tW * yTimes tCur -= tW return tImg
def load(satscene, calibrate=True, area_extent=None, read_basic_or_detailed='both', **kwargs): """Load MSG SEVIRI High Resolution Wind (HRW) data from hdf5 format. """ # Read config file content conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) values = { "orbit": satscene.orbit, "satname": satscene.satname, "number": satscene.number, "instrument": satscene.instrument_name, "satellite": satscene.fullname } LOG.info("assume seviri-level5") print "... assume seviri-level5" satscene.add_to_history("hdf5 data read by mpop/nwcsaf_hrw_hdf.py") # end of scan time 4 min after start end_time = satscene.time_slot + datetime.timedelta(minutes=4) # area !!! satscene.area filename = os.path.join( satscene.time_slot.strftime(conf.get("seviri-level5", "dir", raw=True)), satscene.time_slot.strftime( conf.get("seviri-level5", "filename", raw=True)) % values) # define classes before we search for files (in order to return empty class if no file is found) HRW_basic = HRW_class() HRW_basic.detailed = False HRW_basic.date = satscene.time_slot HRW_detailed = HRW_class() HRW_detailed.detailed = True HRW_detailed.date = satscene.time_slot print "... search for file: ", filename filenames = glob(str(filename)) if len(filenames) != 0: if len(filenames) > 1: print "*** Warning, more than 1 datafile found: ", filenames filename = filenames[0] print("... read data from %s" % str(filename)) # create an instant of the HRW_class m_per_s_to_knots = 1.944 ## limit channels to read #hrw_channels=['HRV'] # limit basic or detailed or both #read_basic_or_detailed='detailed' #read_basic_or_detailed='basic' with h5py.File(filename, 'r') as hf: #print hf.attrs.keys() #print hf.attrs.values() region_name = hf.attrs['REGION_NAME'].replace("_", "") print "... read HRW data for region ", region_name LOG.info("... read HRW data for region " + region_name) sat_ID = GP_IDs[int(hf.attrs["GP_SC_ID"])] print "... derived from Meteosat ", sat_ID LOG.info("... derived from Meteosat " + sat_ID) # print('List of arrays in this file: \n', hf.keys()), len(hf.keys()) if len(hf.keys()) == 0: print "*** Warning, empty file ", filename print "" else: for key in hf.keys(): if key[4:9] == "BASIC": if 'read_basic_or_detailed' in locals(): if read_basic_or_detailed.lower() == "detailed": continue HRW_data = HRW_basic # shallow copy elif key[4:12] == "DETAILED": if 'read_basic_or_detailed' in locals(): if read_basic_or_detailed.lower() == "basic": continue HRW_data = HRW_detailed # shallow copy hrw_chn = dict_channel[key[len(key) - 9:]] if 'hrw_channels' in locals(): if hrw_channels != None: if hrw_chn not in hrw_channels: print "... " + hrw_chn + " is not in hrw_channels", hrw_channels print " skip reading this channel" continue # read all data channel = hf.get(key) # print '... read wind vectors of channel ', channel.name, hrw_chn # print " i lon lat speed[kn] dir pressure" #for i in range(channel.len()): # print '%3d %10.7f %10.7f %7.2f %7.1f %8.1f' % (channel[i]['wind_id'], channel[i]['lon'], channel[i]['lat'], \ # channel[i]['wind_speed']*m_per_s_to_knots, \ # channel[i]['wind_direction'], channel[i]['pressure']) # create string array with channel names channel_chararray = np_empty(channel.len(), dtype='|S6') channel_chararray[:] = hrw_chn HRW_data.channel = np_append(HRW_data.channel, channel_chararray) HRW_data.wind_id = np_append(HRW_data.wind_id, channel[:]['wind_id']) HRW_data.prev_wind_id = np_append( HRW_data.prev_wind_id, channel[:]['prev_wind_id']) HRW_data.segment_X = np_append(HRW_data.segment_X, channel[:]['segment_X']) HRW_data.segment_Y = np_append(HRW_data.segment_Y, channel[:]['segment_Y']) HRW_data.t_corr_method = np_append( HRW_data.t_corr_method, channel[:]['t_corr_method']) HRW_data.lon = np_append(HRW_data.lon, channel[:]['lon']) HRW_data.lat = np_append(HRW_data.lat, channel[:]['lat']) HRW_data.dlon = np_append(HRW_data.dlon, channel[:]['dlon']) HRW_data.dlat = np_append(HRW_data.dlat, channel[:]['dlat']) HRW_data.pressure = np_append(HRW_data.pressure, channel[:]['pressure']) HRW_data.wind_speed = np_append(HRW_data.wind_speed, channel[:]['wind_speed']) HRW_data.wind_direction = np_append( HRW_data.wind_direction, channel[:]['wind_direction']) HRW_data.temperature = np_append(HRW_data.temperature, channel[:]['temperature']) HRW_data.conf_nwp = np_append(HRW_data.conf_nwp, channel[:]['conf_nwp']) HRW_data.conf_no_nwp = np_append(HRW_data.conf_no_nwp, channel[:]['conf_no_nwp']) HRW_data.t_type = np_append(HRW_data.t_type, channel[:]['t_type']) HRW_data.t_level_method = np_append( HRW_data.t_level_method, channel[:]['t_level_method']) HRW_data.t_winds = np_append(HRW_data.t_winds, channel[:]['t_winds']) HRW_data.t_corr_test = np_append(HRW_data.t_corr_test, channel[:]['t_corr_test']) HRW_data.applied_QI = np_append(HRW_data.applied_QI, channel[:]['applied_QI']) HRW_data.NWP_wind_levels = np_append( HRW_data.NWP_wind_levels, channel[:]['NWP_wind_levels']) HRW_data.num_prev_winds = np_append( HRW_data.num_prev_winds, channel[:]['num_prev_winds']) HRW_data.orographic_index = np_append( HRW_data.orographic_index, channel[:]['orographic_index']) HRW_data.cloud_type = np_append(HRW_data.cloud_type, channel[:]['cloud_type']) HRW_data.wind_channel = np_append( HRW_data.wind_channel, channel[:]['wind_channel']) HRW_data.correlation = np_append(HRW_data.correlation, channel[:]['correlation']) HRW_data.pressure_error = np_append( HRW_data.pressure_error, channel[:]['pressure_error']) # sort according to wind_id inds = HRW_data.wind_id.argsort() HRW_data.subset(inds) # changes HRW_data itself # sorting without conversion to numpy arrays #[e for (wid,pwid) in sorted(zip(HRW_data.wind_id,HRW_data.prev_wind_id))] else: print "*** Error, no file found" print "" sat_ID = "no file" # but we continue the program in order to add an empty channel below ## filter data according to the given optional arguments #n1 = str(HRW_data.channel.size) #HRW_data = HRW_data.filter(**kwargs) #print " apply filters "+' ('+n1+'->'+str(HRW_data.channel.size)+')' chn_name = "HRW" satscene[chn_name].HRW_basic = HRW_basic.filter( **kwargs) # returns new object (deepcopied and filtered) satscene[chn_name].HRW_detailed = HRW_detailed.filter( **kwargs) # returns new object (deepcopied and filtered) satscene[chn_name].info['units'] = 'm/s' satscene[chn_name].info['satname'] = 'meteosat' satscene[chn_name].info['satnumber'] = sat_ID satscene[chn_name].info['instrument_name'] = 'seviri' satscene[chn_name].info['time'] = satscene.time_slot satscene[chn_name].info['is_calibrated'] = True
def load(satscene, calibrate=True, area_extent=None, read_basic_or_detailed='both', **kwargs): """Load MSG SEVIRI High Resolution Wind (HRW) data from hdf5 format. """ # Read config file content conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) values = {"orbit": satscene.orbit, "satname": satscene.satname, "number": satscene.number, "instrument": satscene.instrument_name, "satellite": satscene.fullname } LOG.info("assume seviri-level5") print "... assume seviri-level5" satscene.add_to_history("hdf5 data read by mpop/nwcsaf_hrw_hdf.py") # end of scan time 4 min after start end_time = satscene.time_slot + datetime.timedelta(minutes=4) # area !!! satscene.area filename = os.path.join( satscene.time_slot.strftime(conf.get("seviri-level5", "dir", raw=True)), satscene.time_slot.strftime(conf.get("seviri-level5", "filename", raw=True)) % values ) # define classes before we search for files (in order to return empty class if no file is found) HRW_basic = HRW_class() HRW_basic.detailed = False HRW_basic.date = satscene.time_slot HRW_detailed = HRW_class() HRW_detailed.detailed = True HRW_detailed.date = satscene.time_slot print "... search for file: ", filename filenames=glob(str(filename)) if len(filenames) != 0: if len(filenames) > 1: print "*** Warning, more than 1 datafile found: ", filenames filename = filenames[0] print("... read data from %s" % str(filename)) # create an instant of the HRW_class m_per_s_to_knots = 1.944 ## limit channels to read #hrw_channels=['HRV'] # limit basic or detailed or both #read_basic_or_detailed='detailed' #read_basic_or_detailed='basic' with h5py.File(filename,'r') as hf: #print hf.attrs.keys() #print hf.attrs.values() region_name = hf.attrs['REGION_NAME'].replace("_", "") print "... read HRW data for region ", region_name LOG.info("... read HRW data for region "+region_name) sat_ID = GP_IDs[int(hf.attrs["GP_SC_ID"])] print "... derived from Meteosat ", sat_ID LOG.info("... derived from Meteosat "+sat_ID) # print('List of arrays in this file: \n', hf.keys()), len(hf.keys()) if len(hf.keys()) == 0: print "*** Warning, empty file ", filename print "" else: for key in hf.keys(): if key[4:9] == "BASIC": if 'read_basic_or_detailed' in locals(): if read_basic_or_detailed.lower() == "detailed": continue HRW_data = HRW_basic # shallow copy elif key[4:12] == "DETAILED": if 'read_basic_or_detailed' in locals(): if read_basic_or_detailed.lower() == "basic": continue HRW_data = HRW_detailed # shallow copy hrw_chn = dict_channel[key[len(key)-9:]] if 'hrw_channels' in locals(): if hrw_channels != None: if hrw_chn not in hrw_channels: print "... "+hrw_chn+" is not in hrw_channels", hrw_channels print " skip reading this channel" continue # read all data channel = hf.get(key) # print '... read wind vectors of channel ', channel.name, hrw_chn # print " i lon lat speed[kn] dir pressure" #for i in range(channel.len()): # print '%3d %10.7f %10.7f %7.2f %7.1f %8.1f' % (channel[i]['wind_id'], channel[i]['lon'], channel[i]['lat'], \ # channel[i]['wind_speed']*m_per_s_to_knots, \ # channel[i]['wind_direction'], channel[i]['pressure']) # create string array with channel names channel_chararray = np_empty(channel.len(), dtype='|S6') channel_chararray[:] = hrw_chn HRW_data.channel = np_append(HRW_data.channel , channel_chararray ) HRW_data.wind_id = np_append(HRW_data.wind_id , channel[:]['wind_id'] ) HRW_data.prev_wind_id = np_append(HRW_data.prev_wind_id , channel[:]['prev_wind_id'] ) HRW_data.segment_X = np_append(HRW_data.segment_X , channel[:]['segment_X'] ) HRW_data.segment_Y = np_append(HRW_data.segment_Y , channel[:]['segment_Y'] ) HRW_data.t_corr_method = np_append(HRW_data.t_corr_method , channel[:]['t_corr_method'] ) HRW_data.lon = np_append(HRW_data.lon , channel[:]['lon'] ) HRW_data.lat = np_append(HRW_data.lat , channel[:]['lat'] ) HRW_data.dlon = np_append(HRW_data.dlon , channel[:]['dlon'] ) HRW_data.dlat = np_append(HRW_data.dlat , channel[:]['dlat'] ) HRW_data.pressure = np_append(HRW_data.pressure , channel[:]['pressure'] ) HRW_data.wind_speed = np_append(HRW_data.wind_speed , channel[:]['wind_speed'] ) HRW_data.wind_direction = np_append(HRW_data.wind_direction , channel[:]['wind_direction'] ) HRW_data.temperature = np_append(HRW_data.temperature , channel[:]['temperature'] ) HRW_data.conf_nwp = np_append(HRW_data.conf_nwp , channel[:]['conf_nwp'] ) HRW_data.conf_no_nwp = np_append(HRW_data.conf_no_nwp , channel[:]['conf_no_nwp'] ) HRW_data.t_type = np_append(HRW_data.t_type , channel[:]['t_type'] ) HRW_data.t_level_method = np_append(HRW_data.t_level_method , channel[:]['t_level_method'] ) HRW_data.t_winds = np_append(HRW_data.t_winds , channel[:]['t_winds'] ) HRW_data.t_corr_test = np_append(HRW_data.t_corr_test , channel[:]['t_corr_test'] ) HRW_data.applied_QI = np_append(HRW_data.applied_QI , channel[:]['applied_QI'] ) HRW_data.NWP_wind_levels = np_append(HRW_data.NWP_wind_levels , channel[:]['NWP_wind_levels'] ) HRW_data.num_prev_winds = np_append(HRW_data.num_prev_winds , channel[:]['num_prev_winds'] ) HRW_data.orographic_index = np_append(HRW_data.orographic_index, channel[:]['orographic_index'] ) HRW_data.cloud_type = np_append(HRW_data.cloud_type , channel[:]['cloud_type'] ) HRW_data.wind_channel = np_append(HRW_data.wind_channel , channel[:]['wind_channel'] ) HRW_data.correlation = np_append(HRW_data.correlation , channel[:]['correlation'] ) HRW_data.pressure_error = np_append(HRW_data.pressure_error , channel[:]['pressure_error'] ) # sort according to wind_id inds = HRW_data.wind_id.argsort() HRW_data.subset(inds) # changes HRW_data itself # sorting without conversion to numpy arrays #[e for (wid,pwid) in sorted(zip(HRW_data.wind_id,HRW_data.prev_wind_id))] else: print "*** Error, no file found" print "" sat_ID = "no file" # but we continue the program in order to add an empty channel below ## filter data according to the given optional arguments #n1 = str(HRW_data.channel.size) #HRW_data = HRW_data.filter(**kwargs) #print " apply filters "+' ('+n1+'->'+str(HRW_data.channel.size)+')' chn_name="HRW" satscene[chn_name].HRW_basic = HRW_basic.filter(**kwargs) # returns new object (deepcopied and filtered) satscene[chn_name].HRW_detailed = HRW_detailed.filter(**kwargs) # returns new object (deepcopied and filtered) satscene[chn_name].info['units'] = 'm/s' satscene[chn_name].info['satname'] = 'meteosat' satscene[chn_name].info['satnumber'] = sat_ID satscene[chn_name].info['instrument_name'] = 'seviri' satscene[chn_name].info['time'] = satscene.time_slot satscene[chn_name].info['is_calibrated'] = True
def __init__(self, size_word, threshold, data_ts, base_cardinality=2, max_card_alphabet=128, boolean_card_max=True): """ Class initialization function TreeISAX :returns: a tree iSAX :rtype: TreeISAX """ # Number of letters contained in the SAX words indexed in the tree self.size_word = size_word # Threshold before the separation of a sheet into two leaf nodes self.threshold = threshold # Cardinality of each letter at level 1 of the tree # is not very useful ... self._base_cardinality = base_cardinality # Current Max Cardinality self.bigger_current_cardinality = base_cardinality # If true, defined maximum cardinality for the alphabet of the tree self.boolean_card_max = boolean_card_max self.max_card_alphabet = max_card_alphabet # mean, standard deviation of data_ts sequences self.mu, self.sig = norm.fit(data_ts) self.min_max = np_empty(shape=(self.size_word, 2)) for i, dim in enumerate(np_array(data_ts).T.tolist()): self.min_max[i][0] = min(dim) - self.mu self.min_max[i][1] = max(dim) + self.mu self.isax = IndexableSymbolicAggregateApproximation(self.size_word, mean=self.mu, std=self.sig) # verif if all values are properly indexable tmp_max = np_max(abs(np_array(data_ts) - self.mu)) bkpt_max = abs( self.isax._card_to_bkpt(self.max_card_alphabet)[-1] - self.mu) if tmp_max > bkpt_max: ratio = tmp_max / bkpt_max self.isax = IndexableSymbolicAggregateApproximation(self.size_word, mean=self.mu, std=self.sig * ratio) # and we transmit all this at the root node self.root = RootNode( tree=self, parent=None, sax=[0] * self.size_word, cardinality=np_array([int(self._base_cardinality / 2)] * self.size_word)) self.num_nodes = 1 # Attributes for preprocessing self._minmax_nodes_computed = False self.node_list = None self.node_list_leaf = None self.node_leaf_ndarray_mean = None # Attributes for *i*\ CFOF computation self._preprocessing_computed = False self.min_array = None self.max_array = None self.min_array_leaf = None self.max_array_leaf = None self.cdf_mean = None self.cdf_std = None # Boolean value passing True after an update of the tree self._new_insertion_after_preproc = False self._new_insertion_after_minmax_nodes = False
def kappa(y_true, y_pred, weights=None, allow_off_by_one=False, min_rating=None, max_rating=None): """ Calculates the kappa inter-rater agreement between two the gold standard and the predicted ratings. Potential values range from -1 (representing complete disagreement) to 1 (representing complete agreement). A kappa value of 0 is expected if all agreement is due to chance. In the course of calculating kappa, all items in `y_true` and `y_pred` will first be converted to floats and then rounded to integers. It is assumed that y_true and y_pred contain the complete range of possible ratings. This function contains a combination of code from yorchopolis's kappa-stats and Ben Hamner's Metrics projects on Github. :param y_true: The true/actual/gold labels for the data. :type y_true: array-like of float :param y_pred: The predicted/observed labels for the data. :type y_pred: array-like of float :param weights: Specifies the weight matrix for the calculation. Options are: - None = unweighted-kappa - 'quadratic' = quadratic-weighted kappa - 'linear' = linear-weighted kappa - two-dimensional numpy array = a custom matrix of weights. Each weight corresponds to the :math:`w_{ij}` values in the wikipedia description of how to calculate weighted Cohen's kappa. :type weights: str or numpy array :param allow_off_by_one: If true, ratings that are off by one are counted as equal, and all other differences are reduced by one. For example, 1 and 2 will be considered to be equal, whereas 1 and 3 will have a difference of 1 for when building the weights matrix. :type allow_off_by_one: bool """ from numpy import round as np_round from numpy import empty as np_empty from numpy import bincount,outer,count_nonzero # Ensure that the lists are both the same length assert(len(y_true) == len(y_pred)) y_true = np_round(y_true).astype(int) y_pred = np_round(y_pred).astype(int) # Figure out normalized expected values if min_rating is None: min_rating = min(min(y_true), min(y_pred)) if max_rating is None: max_rating = max(max(y_true), max(y_pred)) # shift the values so that the lowest value is 0 # (to support scales that include negative values) y_true = y_true - min_rating y_pred = y_pred - min_rating # Build the observed/confusion matrix num_ratings = max_rating - min_rating + 1 #from sklearn.metrics import confusion_matrix observed = confusion_matrix(y_true, y_pred, labels=list(range(num_ratings))) num_scored_items = float(len(y_true)) # Build weight array if weren't passed one from six import string_types if isinstance(weights, string_types): wt_scheme = weights weights = None else: wt_scheme = '' if weights is None: weights = np_empty((num_ratings, num_ratings)) for i in range(num_ratings): for j in range(num_ratings): diff = abs(i - j) if allow_off_by_one and diff: diff -= 1 if wt_scheme == 'linear': weights[i, j] = diff elif wt_scheme == 'quadratic': weights[i, j] = diff ** 2 elif not wt_scheme: # unweighted weights[i, j] = bool(diff) else: raise ValueError('Invalid weight scheme specified for ' 'kappa: {}'.format(wt_scheme)) hist_true = bincount(y_true, minlength=num_ratings) hist_true = hist_true[: num_ratings] / num_scored_items hist_pred = bincount(y_pred, minlength=num_ratings) hist_pred = hist_pred[: num_ratings] / num_scored_items expected = outer(hist_true, hist_pred) # Normalize observed array observed = observed / num_scored_items # If all weights are zero, that means no disagreements matter. k = 1.0 if count_nonzero(weights): k -= (sum(sum(weights * observed)) / sum(sum(weights * expected))) return k
def transfer_learning(self, max_iterations, accepted_mean_square_error=0.1, batch_size=5000, learning_rate=1e-4): # Adam optimizer optimizes the parameters (weights and biases) at the learning rate specified output_network_optimizer = Adam(self.output_neural_net.parameters(), lr=learning_rate) # This is the row count difference between that of input and output grids row_difference = self.output_grid_dimension[0] - self.input_grid_dimension[0] # This is the column count difference between that of input and output grids column_difference = self.output_grid_dimension[1] - self.input_grid_dimension[1] # We cycle through iterations of each batch of training the output neural network until the max iteration for _ in range(0, max_iterations): # Training list - each element in the list contains [input state for output neural net, target value] training_list = [] # Shuffling through batches and then calculating the Mean square error for the entire batch for batch in range(0, batch_size): # Creating a matrix to hold the observation state of the input neural network map (0 or 1) output_network_known_state = np_array([[randint(0, 1) for _ in range(0, self.output_grid_dimension[1])] for _ in range(0, self.output_grid_dimension[0])]) # This is used to store the robot and target state of the input neural network input_network_state = np_empty((self.network_robot_count+1)*2, dtype=np_uint8) # Creating positions for all of the robots and target of the input neural network randomly for i in range(0, (self.network_robot_count+1)*2, 2): input_network_state[i] = randint(0, self.input_grid_dimension[0]-1) input_network_state[i+1] = randint(0, self.input_grid_dimension[1]-1) # Creates a backup copy of the input state input_network_state_memory = input_network_state # Sliding the input network state window over different sections of the output network state for i in range(0, row_difference): for j in range(0, column_difference): # Creating a matrix to hold the observation state of the input neural network map (0 or 1) input_network_known_state = output_network_known_state[i:(i+self.input_grid_dimension[0]), j:(j+self.input_grid_dimension[1])] # This is used to store the robot and target state of the output neural network output_network_state = np_empty((self.network_robot_count+1)*2, dtype=np_uint8) # Extending the input position states across the moving window within the output grid dimensions for k in range(0, (self.network_robot_count+1)*2, 2): output_network_state[k] = input_network_state[k]+i output_network_state[k+1] = input_network_state[k+1]+j # Now we flatten data in the input network state, a 2-D matrix to 1-D and append input_network_state = np_append(input_network_state, np_ndarray.flatten(input_network_known_state)) # Now we flatten data in the output network state, a 2-D matrix to 1-D and append output_network_state = np_append(output_network_state, np_ndarray.flatten(output_network_known_state)) # Looping through the 4 possible actions each robot can take and then appending them to state for k in range(0, 4): # Adding an action completes the input state for the input neural network input_network_state_tensor = Tensor(np_append(input_network_state, k)) # Adding an action completes the input state for the output neural network output_network_state_tensor = Tensor(np_append(output_network_state, k)) # Getting the Q value predicted by the input neural network for the given state input_network_predicted_value = self.input_neural_net.forward(input_network_state_tensor) # Now we know the value the output neural network is to be trained towards for its given # input. Add both of them to the training list so that batch training can occur later training_list.append([output_network_state_tensor, input_network_predicted_value]) # Restoring the input state from memory input_network_state = input_network_state_memory # Shuffling the training data before feeding it in for training shuffle(training_list) # Initializing the current MSE loss sum_square_error = 0.0 # Using the batch of state and target data for training the output neural network for batch in range(0, batch_size): # Obtaining the completed input states for the output neural network output_network_state_tensor = training_list[batch][0] # Obtaining the target predictions that the output neural network should be trained towards predicted_target_value = training_list[batch][1] # Getting the Q value predicted by the output neural network for the given input state output_network_predicted_value = self.output_neural_net.forward(output_network_state_tensor) # Adding the current square error to the sum of square errors sum_square_error += pow((output_network_predicted_value - predicted_target_value), 2) # Represents the function that can calculate training error training_error_function = MSELoss() # Our goal is to reduce the mean square error loss between the target prediction and that of network training_error = training_error_function(output_network_predicted_value, predicted_target_value) # Clears the gradients of all optimized torch tensors output_network_optimizer.zero_grad() # During the backwards pass, gradients from each replica are summed into the original module training_error.backward() # Training actually happens here. Performs a single optimization step of weights and biases output_network_optimizer.step() # Dividing the sum of square errors by the batch size to get the mean square error current_mean_square_error = sum_square_error/batch_size print(current_mean_square_error) # Checks if the MSE for the entire batch is within acceptable levels and then returns the output neural net if current_mean_square_error <= accepted_mean_square_error: # we return a list where true indicates that we achieved the accepted mean square error criteria return [self.output_neural_net, True] # Failed to completely train the output neural network. Return a list with second element false to indicate this return [self.output_neural_net, False]
def _hyperrectangle_integration(lower, upper, correlation, maxpts, abseps, releps, info=False): # main differences with _parallel_CDF is that it handles complex lower/upper bounds # with infinite components and it returns information on the completion. d = correlation.size(-1) trind = tril_indices(d, -1) ### Infere batch_shape lnone = lower is None unone = upper is None bothNone = lnone and unone if bothNone: pre_batch_shape = [] # broadcast lower and upper to get pre_batch_shape elif not lnone and not unone: pre_batch_shape = broadcast(lower, upper).shape[:-1] cdf = False else: # case were we compute P(Y<x): lower is [-inf, ..., -inf] and upper = x. # Invert lower and upper if it is upper=None cdf = True if unone: upper = -lower pre_batch_shape = upper.shape[:-1] cor = ascontiguousarray(correlation.numpy()[..., trind[0], trind[1]], dtype=float64) # broadcast all lower, upper, correlation batch_shape = broadcast_shape(pre_batch_shape, cor.shape[:-1]) dtype = correlation.dtype device = correlation.device if bothNone: # trivial case if info: return (torch_ones(*batch_shape, dtype=dtype, device=device), torch_zeros(*batch_shape, dtype=dtype, device=device), torch_zeros(*batch_shape, dtype=torch_int32, device=device)) else: return torch_ones(*batch_shape, dtype=dtype, device=device) else: if d == 1: val = Phi(upper.squeeze(-1)) if cdf else Phi( upper.squeeze(-1)) - Phi(lower.squeeze(-1)) if info: return (val, torch_zeros(*batch_shape, dtype=dtype, device=device), torch_zeros(*batch_shape, dtype=torch_int32, device=device)) else: return val dd = d * (d - 1) // 2 # size of flatten correlation matrix # Broadcast: c = broadcast_to(cor, batch_shape + [dd]).reshape(-1, dd) N = c.shape[0] # batch number upp = upper.numpy().astype(float64) shape1 = batch_shape + [d] u = broadcast_to(upp, shape1).reshape(N, d) infu = u == Inf if cdf: l = np_empty( (N, d), dtype=float64) # never used but required by Fortran code i = np_zeros((N, d), dtype=int32) i.setflags(write=1) i[infu] = -1 # basically ignores these componenents else: low = lower.numpy().astype(float64) l = broadcast_to(low, shape1).reshape(N, d) i = full((N, d), 2, dtype=int32) infl = l == -Inf i.setflags(write=1) i[infl] = 0 i[infu] = 1 i[infl * infu] = -1 # basically ignores these componenents # infin is a int vector to pass to the fortran code controlling the integral limits # if INFIN(I) < 0, Ith limits are (-infinity, infinity); # if INFIN(I) = 0, Ith limits are (-infinity, UPPER(I)]; # if INFIN(I) = 1, Ith limits are [LOWER(I), infinity); # if INFIN(I) = 2, Ith limits are [LOWER(I), UPPER(I)]. # TODO better to build res and assign or build-reshap? res = _parallel_genz_bretz(l, u, i, c, maxpts, abseps, releps, info) if info: values, errors, infos = res return (tensor(values, dtype=dtype, device=device).view(batch_shape), tensor(errors, dtype=dtype, device=device).view(batch_shape), tensor(infos, dtype=torch_int32, device=device).view(batch_shape)) else: return tensor(res, dtype=dtype, device=device).view(batch_shape) """ l.setflags(write=1)