def calcTemporalCorrelation(evaluationData, referenceData): ''' Purpose :: Calculate the temporal correlation. Assumption(s) :: The first dimension of two datasets is the time axis. Input :: evaluationData - model data array of any shape referenceData- observation data array of any shape Output:: temporalCorelation - A 2-D array of temporal correlation coefficients at each subregion sigLev - A 2-D array of confidence levels related to temporalCorelation REF: 277-281 in Stat methods in atmos sci by Wilks, 1995, Academic Press, 467pp. sigLev: the correlation between model and observation is significant at sigLev * 100 % ''' evaluationDataMask = process.create_mask_using_threshold(evaluationData, threshold = 0.75) referenceDataMask = process.create_mask_using_threshold(referenceData, threshold = 0.75) nregion = evaluationData.shape[0] temporalCorrelation = ma.zeros([nregion])-100. sigLev = ma.zeros([nregion])-100. for iregion in np.arange(nregion): temporalCorrelation[iregion], sigLev[iregion] = stats.pearsonr(evaluationData[iregion,:], referenceData[iregion,:]) sigLev[iregion] = 1 - sigLev[iregion] temporalCorrelation=ma.masked_equal(temporalCorrelation.data, -100.) sigLev=ma.masked_equal(sigLev.data, -100.) return temporalCorrelation, sigLev
def maskedEqual(array, missingValue): """ Mask an array where equal to a given (missing)value. Unfortunately ma.masked_equal does not work with structured arrays. See: https://mail.scipy.org/pipermail/numpy-discussion/2011-July/057669.html If the data is a structured array the mask is applied for every field (i.e. forming a logical-and). Otherwise ma.masked_equal is called. """ if array_is_structured(array): # Enforce the array to be masked if not isinstance(array, ma.MaskedArray): array = ma.MaskedArray(array) # Set the mask separately per field for nr, field in enumerate(array.dtype.names): if hasattr(missingValue, '__len__'): fieldMissingValue = missingValue[nr] else: fieldMissingValue = missingValue array[field] = ma.masked_equal(array[field], fieldMissingValue) check_class(array, ma.MaskedArray) # post-condition check return array else: # masked_equal works with missing is None result = ma.masked_equal(array, missingValue, copy=False) check_class(result, ma.MaskedArray) # post-condition check return result
def calculate(self, tracks): dataMean = ma.zeros((len(self.lon_range) - 1, len(self.lat_range) - 1)) dataMin = ma.zeros((len(self.lon_range) - 1, len(self.lat_range) - 1)) dataMax = ma.zeros((len(self.lon_range) - 1, len(self.lat_range) - 1)) dataMed = ma.zeros((len(self.lon_range) - 1, len(self.lat_range) - 1)) log.debug("Processing %d tracks" % (len(tracks))) for cell in self.gridCells: vcell = np.array([]) for t in tracks: ii = np.where(((t.Latitude >= cell.ymin) & (t.Latitude < cell.ymax)) & ((t.Longitude >= cell.xmin) & (t.Longitude < cell.xmax)))[0] if len(ii) > 0: vv = t.CentralPressure[ii].compress(t.CentralPressure[ii] < sys.maxint) vcell = np.append(vcell, vv.compress(vv > 0.0)) if len(vcell > 0): dataMean[cell.index[0], cell.index[1]] = np.mean(vcell) dataMin[cell.index[0], cell.index[1]] = np.min(vcell) dataMax[cell.index[0], cell.index[1]] = np.max(vcell) dataMed[cell.index[0], cell.index[1]] = np.median(vcell) dataMean = ma.masked_equal(dataMean, 0) dataMin = ma.masked_equal(dataMin, 0) dataMax = ma.masked_equal(dataMax, 0) dataMed = ma.masked_equal(dataMed, 0) return dataMean, dataMin, dataMax, dataMed
def topex_track_table(ndata,tracks,cycles): """ """ track_list=[] cycle_list=[] for track, n, cycle in itertools.izip(tracks.compressed(),ndata.compressed(),cycles.compressed()): for i in range(n): track_list.append(track) cycle_list.append(cycle) track_list=ma.masked_equal(track_list,-1) cycle_list=ma.masked_equal(cycle_list,-1) return cycle_list,track_list
def load_6hr(self, var, DTime, maskflag=True, verbose=False): if (DTime < datetime(DTime.year, 1, 1, 6)): dirYear = DTime.year -1 dY = +1 else: dirYear = DTime.year dY = 0 srcDir = os.path.join(self.baseDir, self.model, self.runName ,"y%04d"%(dirYear), "6hr") srcPath = os.path.join(srcDir ,"%s.sa.1460x%dx%d"%(var, self.ny, self.nx)) # timeslice for no leap year (e.g.:2001) timeslice = (datetime(2001+dY, DTime.month, DTime.day, DTime.hour) - datetime(2001, 1, 1, 6)).total_seconds() / (6*3600) if verbose ==True: print srcPath out = self.readslice_float32(srcPath, timeslice, self.ny, self.nx) if maskflag==True: return ma.masked_equal(out, self.miss) else: return out
def createResultArray(self, result, fillValue=0): t = time.time() # Create list of records data = [i[:] for i in result] print '\tLooping through results took - %.4f' %(time.time()-t), len(data) # Converting the none values returned into a zero value # using the ma library in numpy # - retrieve mask for None # - then assign the fillValue to those columns data = array(data) mask = ma.masked_equal(data, None).mask if mask.any(): data[mask] = fillValue #Sorting the array by primary cols identifying the agent as # postgres seems to return queries without any order # Convert it back to a regular array to enable all the other processing print '\tSize of the data set that was retrieved - ', data.shape print '\tRecords were processed after query in %.4f' %(time.time()-t) return data
def _int_connection_matrix(sources, targets, values): ''' Return a 2D connection matrix filled with integer values (typically the number of synapses) in the form of a masked matrix (values equal to 0 are masked) Parameters ---------- sources : ndarray of int The indices of the source neurons for each value. targets : ndarray of int The indices of the target neurons for each value. values : ndarray of int or int The value for each (source, target) pair. Returns ------- matrix : ma.MaskedArray The connection matrix, masked for 0 values ''' assert np.min(values) > 0 and np.max(values) < 256 full_matrix = np.zeros((np.max(targets) - np.min(targets) + 1, np.max(sources) - np.min(sources) + 1), dtype=np.uint8) full_matrix[targets - np.min(targets), sources - np.min(sources)] = values return ma.masked_equal(full_matrix, 0, copy=False)
def setUp(self): lazy_data = as_lazy_data(ma.masked_equal([1, 2, 3, 4, 5], 3)) self.lazy_cube = Cube(lazy_data) self.lazy_cube.add_dim_coord(DimCoord([6, 7, 8, 9, 10], long_name='foo'), 0) self.func = lambda x: x >= 3
def copyVariable(self, varname): source = self._openExemplar() dims = list(source.variables[varname].dimensions[:]) # substitute lat for y and lon for x for i in range(len(dims)): if dims[i] == "x": dims[i] = "lon" elif dims[i] == "y": dims[i] = "lat" # Ensure that all the required dimensions are present for dim in dims: self.copyDimension(dim) # get the missing value missing = round(source.variables[varname].missing_value, -13) # Create the variable in the fixed file the_copy = self._ncfile.createVariable(varname, source.variables[varname].dtype, dims, fill_value=missing) # copy the data to the fixed variable the_copy[:] = ma.masked_equal(source.variables[varname][:], missing) source.close() return the_copy
def getData(self, variable=None): ''' Aggreate all data from the file list in the order given. ''' filename = self.flist[0].strip() f = cdms2.open( filename, 'r' ) if( variable != None ): self.vartoread = variable else: self.vartoread = self.variable.id data = f(self.vartoread)[:] # --------------------------- # Concatenate following files # --------------------------- for filename in self.flist[ 1: ]: print "reading %s" % filename.strip() f = cdms2.open( filename.strip(), 'r' ) data2 = f(self.vartoread)[:] data = numpy.concatenate((data,data2), axis=0) f.close() data=ma.array(data=data, \ fill_value=self.missing_value, \ copy=0, \ dtype='float32' ) data=ma.masked_equal(data, self.missing_value, copy=0) return data
def transform(self, X, details=False): """ Label hidden factors for (possibly previously unseen) samples of data. Parameters: samples of data, X, shape = [n_samples, n_visible] Returns: , shape = [n_samples, n_hidden] """ Xm = ma.masked_equal(X, self.missing_values) log_marg_x = self.calculate_marginals_on_samples(self.theta, Xm) p_y_given_x, log_z = self.calculate_latent(log_marg_x) labels = self.label(p_y_given_x) if details == 'surprise': # Totally experimental log_marg_x = self.calculate_marginals_on_samples(self.theta, Xm, return_ratio=False) n_samples = Xm.shape[0] surprise = [] for l in range(n_samples): q = - sum([max([log_marg_x[j,l,i,labels[l, j]] for j in range(self.n_hidden)]) for i in range(self.n_visible)]) surprise.append(q) return p_y_given_x, log_z, np.array(surprise) elif details: return p_y_given_x, log_z else: return labels
def _check_sst_quality(self, dataset, product_type): mask_specs = product_type.get_mask_consistency_check_specs() if len(mask_specs) == 0: return sst_variable_names = product_type.get_sst_variable_names() if len(sst_variable_names) == 0: return quality_variable_name = mask_specs[0][2] quality_data = dataset.variables[quality_variable_name][:] valid_retrieval_quality = ma.masked_less(quality_data, 2) self.report["sst_valid_retrieval"] = float(valid_retrieval_quality.count()) failed_retrieval_quality = ma.masked_not_equal(quality_data, 1) sst_variable = dataset.variables[sst_variable_names[0]] fill_value = sst_variable.getncattr('_FillValue') sst_quality_one_data = ma.array(sst_variable[:], mask=failed_retrieval_quality.mask) invalid_retrieval = ma.masked_equal(sst_quality_one_data, fill_value) self.report["sst_invalid_retrieval"] = float(invalid_retrieval.count()) failed_retrieval = ma.masked_not_equal(sst_quality_one_data, fill_value) self.report["sst_failed_retrieval"] = float(failed_retrieval.count()) not_ocean = ma.masked_not_equal(quality_data, 0) self.report["not_ocean"] = float(not_ocean.count())
def make_contour_bounds_shapefile(self): import databundles.geo as dg from numpy import ma import yaml shape_file_dir = self.filesystem.path('extracts','contours') shape_file = os.path.join(shape_file_dir, 'contours.shp') # One of two. if os.path.exists(shape_file): return shape_file_dir partition = self.partitions.all[0]# There is only one hdf = partition.hdf5file hdf.open() a1,_ = hdf.get_geo('property') a2,aa = hdf.get_geo('violent') a = dg.std_norm(ma.masked_equal(a1[...] + a2[...],0)) # ... Converts to a Numpy array. # Creates the shapefile in the extracts/contour directory envelopes = dg.bound_clusters_in_raster( a, aa, shape_file_dir, 0.1,0.7, use_bb=True, use_distance=50) # Cache the envelopes for later. env_file = self.filesystem.path('build','envelopes.yaml') with open(env_file,'w') as f: f.write(yaml.dump(envelopes, indent=4, default_flow_style=False)) return shape_file_dir
def main(): #Get the raster from the disk rast_data, x_cellsize, y_cellsize = get_array("./data/elevation.tif") slope = generic_filter(rast_data, calc_slope, size=3, extra_arguments=(x_cellsize, y_cellsize)) plt.imshow(ma.masked_equal(slope, -9999), cmap=plt.winter(), origin="lower") plt.show()
def __call__(self,x,clip=False): x = array(x) ret = zeros(x.shape,'int') for i, b in enumerate(self.boundaries): ret[greater_equal(x,b)] = i+1 ret[less(x,self.vmin)] = 0 return ma.masked_equal(ret,-1)/float(self.N)
def toNumpyArray(self): """ Returns a narray """ bands = map(lambda b : b.data(),self.rasterdata.bands) nodataval = self.rasterdata.allBandStatistics()['nodata'] bands = map(lambda b : masked_equal(b,nodataval),bands) return bands
def __check_false_positives(self, reference_mask, objective_mask, check_name): # noinspection PyNoneFunctionAssignment,PyUnresolvedReferences false_positives = ma.masked_equal(np.logical_or(np.logical_not(objective_mask), reference_mask), True) false_positives_count = false_positives.count() self.report[check_name] = false_positives_count if false_positives_count > 0: filename = os.path.basename(self.source_pathname) self.report[check_name + '_failed_for'] = filename
def write_colormap(file_name, a, map, break_scheme='even', min_val=None, max_val=None, ave_val=None): """Write a QGIS colormap file""" import numpy as np import numpy.ma as ma import math header = "# QGIS Generated Color Map Export File\nINTERPOLATION:DISCRETE\n" masked = ma.masked_equal(a, 0) min_ = np.min(masked) if not min_val else min_val max_ = np.max(a) if not max_val else max_val ave_ = masked.mean() if not ave_val else ave_val if break_scheme == 'even': max_ = max_ * 1.001 # Be sure to get all values range = min_ - max_ delta = range * .001 r = np.linspace(min_ - delta, max_ + delta, num=map['n_colors'] + 1) elif break_scheme == 'jenks': from ambry.geo import jenks_breaks r = jenks_breaks(a, map['n_colors']) elif break_scheme == 'geometric': r = geometric_breaks(map['n_colors'], min_, max_) elif break_scheme == 'logistic': r = logistic_breaks(map['n_colors'], min_, max_) elif break_scheme == 'exponential': r = exponential_breaks(map['n_colors'], ave_) elif break_scheme == 'stddev': sd = np.std(a) else: raise Exception("Unknown break scheme: {}".format(break_scheme)) colors = map['map'] colors.append(None) # Causes the last item to be skipped alphas, alpha_step = np.linspace(64, 255, len(colors), retstep=True) alpha = alpha_step + 64 with open(file_name, 'w') as f: f.write(header) last_me = None for v, me in zip(r, colors): if me: f.write(','.join([str(v), str(me['R']), str(me['G']), str(me['B']), str(int(alpha)), me['letter']])) alpha += alpha_step alpha = min(alpha, 255) f.write('\n') last_me = me # Prevents 'holes' where the value is higher than the max_val if max_val: v = np.max(a) f.write(','.join( [str(v), str(last_me['R']), str(last_me['G']), str(last_me['B']), str(int(alpha)), last_me['letter']])) f.write('\n')
def applyOperation( self, input_variable, operation ): result = None try: self.setTimeBounds( input_variable ) operator = None # pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True) wpsLog.debug( " $$$ ApplyOperation: %s " % str( operation ) ) if operation is not None: type = operation.get('type','').lower() bounds = operation.get('bounds','').lower() op_start_time = time.clock() # time.time() if not bounds: if type == 'departures': ave = cdutil.averager( input_variable, axis='t', weights='equal' ) result = input_variable - ave elif type == 'climatology': result = cdutil.averager( input_variable, axis='t', weights='equal' ) else: result = input_variable time_axis = input_variable.getTime() elif bounds == 'np': if type == 'departures': result = ma.anomalies( input_variable ).squeeze() elif type == 'climatology': result = ma.average( input_variable ).squeeze() else: result = input_variable time_axis = input_variable.getTime() else: if bounds == 'djf': operator = cdutil.DJF elif bounds == 'mam': operator = cdutil.MAM elif bounds == 'jja': operator = cdutil.JJA elif bounds == 'son': operator = cdutil.SON elif bounds == 'year': operator = cdutil.YEAR elif bounds == 'annualcycle': operator = cdutil.ANNUALCYCLE elif bounds == 'seasonalcycle': operator = cdutil.SEASONALCYCLE if operator <> None: if type == 'departures': result = operator.departures( input_variable ).squeeze() elif type == 'climatology': result = operator.climatology( input_variable ).squeeze() else: result = operator( input_variable ).squeeze() time_axis = result.getTime() op_end_time = time.clock() # time.time() wpsLog.debug( " ---> Base Operation Time: %.5f" % (op_end_time-op_start_time) ) else: result = input_variable time_axis = input_variable.getTime() if isinstance( result, float ): result_data = [ result ] elif result is not None: if result.__class__.__name__ == 'TransientVariable': result = ma.masked_equal( result.squeeze().getValue(), input_variable.getMissing() ) result_data = result.tolist( numpy.nan ) else: result_data = None except Exception, err: wpsLog.debug( "Exception applying Operation '%s':\n %s" % ( str(operation), traceback.format_exc() ) ) return ( None, None )
def _prep_data(self): self.rain_rate = np.array(self.rain_rate) self.Z = ma.masked_equal(self.Z, -9.999) self.nd = np.array(self.nd) self.nd[self.nd == -9.999] = 0 self.Nd = np.array(self.nd) self.num_particles = np.array(self.num_particles) self.time = np.array(self.time) self.velocity = self.vd # np.ndarray(self.vd)
def hist(self, dataarr, path): g = self.graph ''' input array g1iage,g1ierr,g2iage,g2ierr... . . . g1nage,... ''' # dataarr = loadtxt(data, delimiter=',', skiprows=1) g.new_plot(padding=[30, 10, 30, 30], show_legend='ur') r, c = dataarr.shape for ci in range(0, c, 2): ages = dataarr[:, ci] errs = dataarr[:, ci + 1] ages = ma.masked_equal(ages, 0) errs = ma.masked_equal(errs, 0) relative_errs = errs / ages * 100 y, x = histogram(relative_errs) # convert edges to mids mids = zeros(len(x) - 1) for i in range(len(x) - 1): mids[i] = x[i] + (x[i + 1] - x[i]) / 2.0 width = x[1] - x[0] g.new_series(mids, y, type='bar', bar_width=width) # add the labels with open(path, 'r') as f: header = f.readline() for i, l in enumerate(header.split(',')): g.set_series_label(l, series=i) #=============================================================================== # hardcoded additions #=============================================================================== g.set_x_title('Relative Error (%)') g.set_y_title('Frequency')
def get_data_range(self): """ Gets the datarange of the variable in this dataset NOTE: This method might be time-consuming if called on a large dataset as it may have to iterate over every single data point. :return: min and max tuple for the range """ try: actual_range = self._variable.attributes['actual_range'] return [float(actual_range[0]), float(actual_range[1])] except (KeyError, ValueError): pass try: return [float(self._variable.attributes['valid_min']), float(self._variable.attributes['valid_max'])] except (KeyError, ValueError): pass try: fill_value = self._variable.attributes.get('_FillValue', None) missing_value = self._variable.attributes.get('missing_value', None) if fill_value is not None and missing_value is not None: valid_array = masked_equal(masked_equal(self._variable.array, missing_value), fill_value) elif fill_value is None and missing_value is None: valid_array = self._variable.array elif fill_value is not None: valid_array = masked_equal(self._variable.array, fill_value) else: valid_array = masked_equal(self._variable.array, missing_value) min = float(amin(valid_array)) max = float(amax(valid_array)) if math.isnan(min): min = 0 if math.isnan(max): max = 100 except: # Use the default result if something goes wrong log.exception("Can not calculate data range using default") min = 0 max = 100 return [min, max]
def setMask(self, maskVal=None, band="All"): if band != "All": self.unsetMask(band) self.setNoData(maskVal, band) if self.nodatad[band] is not None: self.datad[band] = ma.masked_equal(self.datad[band], self.nodatad[band]) else: self.unsetMask() for band in range(1, self.bands + 1): self.setMask(maskVal, band)
def safeDataStats(data, nodata=None): if nodata is not None: data = ma.masked_equal(data, nodata) stats = (data.min(), data.max(), data.mean(), data.std()) stddev = stats[-1] if ma.isMaskedArray(stddev) or stddev == 0: stats = (None, None, None, None) return stats
def _proportion(array, function, axis, **kwargs): # if the incoming array is masked use that to count the total number of values if isinstance(array, ma.MaskedArray): # calculate the total number of non-masked values across the given axis total_non_masked = _count(array.mask, lambda v: v == False, axis=axis, **kwargs) total_non_masked = ma.masked_equal(total_non_masked, 0) else: total_non_masked = array.shape[axis] return _count(array, function, axis=axis, **kwargs) / total_non_masked
def _grid_array_to_gdal_files(dap_grid_array, srs, geo_transform, filename_fmt='{i}.asc', missval=None): '''Generator which creates an Arc/Info ASCII Grid file for each "layer" (i.e. one step of X by Y) in a given grid :param dap_grid_array: Multidimensional arrary of rank 2 or 3 :type dap_grid_array: numpy.ndarray :param srs: Spatial reference system :type srs: osr.SpatialReference :param geo_transform: GDAL affine transform which applies to this grid :type geo_transform: list :param filename_fmt: Proposed filename template for output files. "{i}" can be included and will be filled in with the layer number. :type filename_fmt: str :param missval: Value for which data should be identified as missing :type missval: numpy.array :returns: A generator which yields pairs of (filename, file_content_generator) of the created files. Note that there will likely be more than one file for layer (e.g. an .asc file and a .prj file). ''' logger.debug("_grid_array_to_gdal_files: translating this grid {} of this srs {} transform {} to this file {}".format(dap_grid_array, srs, geo_transform, filename_fmt)) logger.debug("Investigating the shape of this grid: {}".format(dap_grid_array)) shp = dap_grid_array.shape if len(shp) == 2: ylen, xlen = shp data = [ dap_grid_array ] elif len(shp) == 3: _, ylen, xlen = shp data = iter(dap_grid_array.data) else: raise ValueError("_grid_array_to_gdal_files received a grid of rank {} rather than the required 2 or 3".format(len(shp))) target_type = numpy_to_gdal[dap_grid_array.dtype.name] meta_ds = create_gdal_mem_dataset(xlen, ylen, geo_transform, srs, target_type, missval) for i, layer in enumerate(data): if missval: layer = ma.masked_equal(layer, missval) logger.debug("Data: {}".format(layer)) meta_ds.GetRasterBand(1).WriteArray( numpy.flipud(layer) ) driver = gdal.GetDriverByName('AAIGrid') outfile = gettempdir() + sep + filename_fmt.format(i=i) dst_ds = driver.CreateCopy(outfile, meta_ds, 0) file_list = dst_ds.GetFileList() # Once we're done, close properly the dataset dst_ds = None for filename in file_list: yield named_file_generator(filename) meta_ds = None
def scatter(self, dataarr, datapath): g = self.graph g.new_plot(padding=[30, 10, 30, 30], ) _r, c = dataarr.shape for ci in range(0, c, 3): ages = ma.masked_equal(dataarr[:, ci], 0) errors = ma.masked_equal(dataarr[:, ci + 1], 0) k2o = ma.masked_equal(dataarr[:, ci + 2], 0) relative_errs = errors / ages * 100 g.new_series(relative_errs, k2o, type='scatter', marker='circle') g.set_x_limits(0, max(relative_errs) * 1.1) g.set_y_limits(0, max(k2o) * 1.1) g.set_x_title('Relative Error (%)') g.set_y_title('K2O (%)')
def topex_time_table(dt_days,dt_seconds,dt_microseconds,base_date=None): """ """ if base_date is None: base_date=datetime(year=1950,month=01,day=01,hour=0,minute=0,second=0) t=[] for d, s, ms in itertools.izip(dt_days.compressed(),dt_seconds.compressed(),dt_microseconds.compressed()): dt=timedelta(days=int(d),seconds=int(s),microseconds=int(ms)) t.append(base_date+dt) t=ma.masked_equal(t,-1) return t
def FIS_Test2(instances = 1000): test = getTestRun("1kTestRun3.db3") Weights = map(RecommenderItemCF.eval, zip(test[:instances,5],test[:instances,8],test[:instances,14])) Recs = map(Weighted_avg,Weights,test[:instances,3],test[:instances,12],test[:instances,10]) k = test[:instances,:] k = numpy.column_stack([k, Recs]) import numpy.ma as ma recoms = ma.masked_equal(k[:instances,20],None) rats = ma.array(k[:,3], mask = recoms.mask) r = sum( numpy.abs( recoms.compressed()-rats.compressed() ) ) / len(rats.compressed()) return r ,k
def __get_masked_data(variable): """ :type variable: Variable :rtype : ma.MaskedArray """ data = variable[:] try: fill_value = variable.getncattr('_FillValue') return ma.masked_equal(data, fill_value) except AttributeError: return ma.array(data)
# open the shapefile vDriver = ogr.GetDriverByName("ESRI Shapefile") vSrc = vDriver.Open(shpF, 0) vLayer = vSrc.GetLayer() nFeat = vLayer.GetFeatureCount() for f, feature in enumerate(vLayer): f=0 feature = vLayer[f] name = feature.GetField(trgField) print(name) descr = feature.GetField(descrField) predP = get_zone_pixels(feature, shpF, predF, 1)#.compressed() trainP = get_zone_pixels(feature, shpF, trainF, 1)#.compressed() predP = ma.masked_equal(predP, predND) trainP = ma.masked_equal(trainP, trainND) trainP = ma.masked_equal(trainP, 0) """ print(np.unique(predP.compressed())) print(np.unique(trainP.compressed())) hist, bin_edges = np.histogram(predP.compressed(), density=True) plt.hist(predP.compressed(), bins=bin_edges) plt.show() hist, bin_edges = np.histogram(trainP.compressed(), density=True) plt.hist(trainP.compressed(), bins=bin_edges) plt.show() print(predP.shape)
def MapThatShit(NDAI_BASE,date,spl_arr,drought_avg_tci_cmap): # get array informaiton extent, img_extent = setMap(NDAI_BASE) ds = gdal.Open(NDAI_BASE) array = ds.ReadAsArray() #array[ma.getmaskarray(NDAI)] = 0 #nan = ds.GetRasterBand(1).GetNoDataValue() array = ma.masked_equal(array, 0) array = np.flipud(array) logging.info(extent) # set shape of figure width = array.shape[0] height = array.shape[1] base_width = 10. base_height = (base_width * height) / width print base_width, base_height logging.info(base_width, base_height) # figure fig = plt.figure(figsize=(base_height,base_width)) ax = plt.subplot(1,1,1, projection=ccrs.PlateCarree()) ax.set_extent(extent, ccrs.Geodetic()) gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True, linewidth=1, color='gray', alpha=0.5, linestyle='--') gl.ylabels_right = False gl.xlabels_top = False # prulletaria on the map ogr2ogr = r'C:\Program Files\GDAL//ogr2ogr.exe' base_geom = r'D:\Data\ChinaShapefile\natural_earth' # ocean in_file_ocean = base_geom + '\physical//10m_ocean.shp' outfile_ocean = 'ocean.shp' # country boudaries in_file_bound = base_geom + '\cultural//10m_admin_0_boundary_lines_land.shp' outfile_bound = 'boundaries.shp' # costlie in_file_coast = base_geom + '\physical//10m_coastline.shp' outfile_coast = 'coastline.shp' #lakes in_file_lakes = base_geom + '\physical//10m_lakes.shp' outfile_lakes = 'lakes.shp' # run the clip functions command = [ogr2ogr, '-f', "ESRI Shapefile", outfile_ocean, in_file_ocean,'-clipsrc', str(spl_arr[0]),str(spl_arr[1]),str(spl_arr[2]),str(spl_arr[3]),'-overwrite'] print (sp.list2cmdline(command)) norm = sp.Popen(sp.list2cmdline(command),stdout=sp.PIPE, shell=True) norm.communicate() command = [ogr2ogr, '-f', "ESRI Shapefile", outfile_bound, in_file_bound,'-clipsrc', str(spl_arr[0]),str(spl_arr[1]),str(spl_arr[2]),str(spl_arr[3]),'-overwrite'] print (sp.list2cmdline(command)) norm = sp.Popen(sp.list2cmdline(command),stdout=sp.PIPE, shell=True) norm.communicate() command = [ogr2ogr, '-f', "ESRI Shapefile", outfile_coast, in_file_coast,'-clipsrc', str(spl_arr[0]),str(spl_arr[1]),str(spl_arr[2]),str(spl_arr[3]),'-overwrite'] print (sp.list2cmdline(command)) norm = sp.Popen(sp.list2cmdline(command),stdout=sp.PIPE, shell=True) norm.communicate() command = [ogr2ogr, '-f', "ESRI Shapefile", outfile_lakes, in_file_lakes,'-clipsrc', str(spl_arr[0]),str(spl_arr[1]),str(spl_arr[2]),str(spl_arr[3]),'-overwrite'] print (sp.list2cmdline(command)) norm = sp.Popen(sp.list2cmdline(command),stdout=sp.PIPE, shell=True) norm.communicate() ax.add_geometries(Reader(outfile_ocean).geometries(), ccrs.PlateCarree(), facecolor='lightblue') ax.add_geometries(Reader(outfile_bound).geometries(), ccrs.PlateCarree(), facecolor='',linestyle=':', linewidth=2) ax.add_geometries(Reader(outfile_coast).geometries(), ccrs.PlateCarree(), facecolor='') ax.add_geometries(Reader(outfile_lakes).geometries(), ccrs.PlateCarree(), facecolor='lightskyblue') # ticks of classes bounds = [ 0. , 82.875, 95.625, 108.375, 127.5 , 146.625, 159.375, 172.125, 255. ] # ticklabels plus colorbar ticks = ['-1','-0.35','-0.25','-0.15','+0','+.15','+.25','+.35','+1'] cmap = cmap_discretize(drought_avg_tci_cmap,8) norm = matplotlib.colors.BoundaryNorm(bounds, cmap.N) im = ax.imshow(array, origin='upper', extent=img_extent,norm=norm, cmap=cmap, vmin=0, vmax=255, interpolation='nearest')#, transform=ccrs.Mercator()) title = 'NDAI '+date plt.title(title, fontsize=22) cb = plt.colorbar(im, fraction=0.0476, pad=0.04, ticks=bounds,norm=norm, orientation='horizontal') cb.set_label('Normalized Drought Anomaly Index') cb.set_ticklabels(ticks) spl_arr_str = str(spl_arr) spl_arr_str = spl_arr_str.replace('[','') spl_arr_str = spl_arr_str.replace(']','') spl_arr_str = spl_arr_str.replace(', ','#') spl_arr_str = spl_arr_str.replace('.','.') # and save the shit outpath = 'NDAI_'+date+'_'+spl_arr_str+'.png' plt.savefig(outpath, dpi=200, bbox_inches='tight') print outpath logging.info(outpath) #plt.tight_layout() plt.show() plt.close(fig) fig.clf() return outpath
pnum = len(pname) #print np.shape(sfcelv_hydroweb) colors = ['xkcd:pastel blue', 'xkcd:aqua green', 'xkcd:soft pink'] labels = ["$<1m$", "$<5m$", "$>5m$"] #========== dataxx0 = [] datayy0 = [] dataxx1 = [] datayy1 = [] dataxx2 = [] datayy2 = [] dataxx3 = [] datayy3 = [] for point in np.arange(pnum): cmf_mean = np.mean(sfcelv_cmf[:, point]) obs_mean = np.mean(ma.masked_equal(sfcelv_hydroweb[:, point], -9999.0)) BIAS = abs(cmf_mean - obs_mean) # print (cmf_mean,obs_mean,BIAS) datayy0.append(BIAS) dataxx0.append(abs(elediff[point])) if disttomouth[point] <= 1.0: datayy1.append(BIAS) dataxx1.append(abs(elediff[point])) print("<=1m", pname[point], BIAS, disttomouth[point]) elif disttomouth[point] <= 5.0: datayy2.append(BIAS) dataxx2.append(abs(elediff[point])) print("<=5m", pname[point], BIAS, disttomouth[point]) else: datayy3.append(BIAS) dataxx3.append(abs(elediff[point]))
def getTimeseries(productcode, subproductcode, version, mapsetcode, geom, start_date, end_date, aggregate): # Extract timeseries from a list of files and return as JSON object # It applies to a single dataset (prod/sprod/version/mapset) and between 2 dates # Several types of aggregation foreseen: # # mean : Sum(Xi)/N(Xi) -> min/max not considered e.g. Rain # cumulate: Sum(Xi) -> min/max not considered e.g. Fire # # count: N(Xi where min < Xi < max) e.g. Vegetation anomalies # surface: count * PixelArea e.g. Water Bodies # percent: count/Ntot e.g. Vegetation anomalies # precip: compute the precipitation volume in m3*1E6 Rain (only) # # History: 1.0 : Initial release - since 2.0.1 -> now renamed '_green' from greenwich package # 1.1 : Since Feb. 2017, it is based on a different approach (gdal.RasterizeLayer instead of greenwich) # in order to solve the issue with MULTIPOLYGON # ogr.UseExceptions() # Get Mapset Info mapset_info = querydb.get_mapset(mapsetcode=mapsetcode) # Prepare for computing conversion to area: the pixel size at Lat=0 is computed # The correction to the actual latitude (on AVERAGE value - will be computed below) const_d2km = 12364.35 area_km_equator = abs(float(mapset_info.pixel_shift_lat)) * abs( float(mapset_info.pixel_shift_long)) * const_d2km # Get Product Info product_info = querydb.get_product_out_info(productcode=productcode, subproductcode=subproductcode, version=version) if product_info.__len__() > 0: # Get info from product_info scale_factor = 0 scale_offset = 0 nodata = 0 date_format = '' for row in product_info: scale_factor = row.scale_factor scale_offset = row.scale_offset nodata = row.nodata date_format = row.date_format date_type = row.data_type_id # Create an output/temp shapefile, for managing the output layer (really mandatory ?? Can be simplified ???) try: tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='_getTimeseries', dir=es_constants.base_tmp_dir) except: logger.error('Cannot create temporary dir ' + es_constants.base_tmp_dir + '. Exit') raise NameError('Error in creating tmpdir') out_shape = tmpdir + os.path.sep + "output_shape.shp" outDriver = ogr.GetDriverByName('ESRI Shapefile') # Create the output shapefile outDataSource = outDriver.CreateDataSource(out_shape) dest_srs = ogr.osr.SpatialReference() dest_srs.ImportFromEPSG(4326) outLayer = outDataSource.CreateLayer("Layer", dest_srs) # outLayer = outDataSource.CreateLayer("Layer") idField = ogr.FieldDefn("id", ogr.OFTInteger) outLayer.CreateField(idField) featureDefn = outLayer.GetLayerDefn() feature = ogr.Feature(featureDefn) feature.SetGeometry(geom) # area = geom.GetArea() feature.SetField("id", 1) outLayer.CreateFeature(feature) feature = None [list_files, dates_list] = getFilesList(productcode, subproductcode, version, mapsetcode, date_format, start_date, end_date) # Built a dictionary with filenames/dates dates_to_files_dict = dict(list(zip(dates_list, list_files))) # Generate unique list of files unique_list = set(list_files) uniqueFilesValues = [] geo_mask_created = False for infile in unique_list: single_result = { 'filename': '', 'meanvalue_noscaling': nodata, 'meanvalue': None } if infile.strip() != '' and os.path.isfile(infile): # try: # Open input file orig_ds = gdal.Open(infile, gdal.GA_ReadOnly) orig_cs = osr.SpatialReference() orig_cs.ImportFromWkt(orig_ds.GetProjectionRef()) orig_geoT = orig_ds.GetGeoTransform() x_origin = orig_geoT[0] y_origin = orig_geoT[3] pixel_size_x = orig_geoT[1] pixel_size_y = -orig_geoT[5] in_data_type_gdal = conv_data_type_to_gdal(date_type) # Create a mask from the geometry, with the same georef as the input file[s] if not geo_mask_created: # Read polygon extent and round to raster resolution x_min, x_max, y_min, y_max = outLayer.GetExtent() x_min_round = int(old_div( (x_min - x_origin), pixel_size_x)) * pixel_size_x + x_origin x_max_round = ( int(old_div( (x_max - x_origin), (pixel_size_x))) + 1) * pixel_size_x + x_origin y_min_round = ( int(old_div( (y_min - y_origin), (pixel_size_y))) - 1) * pixel_size_y + y_origin y_max_round = int( old_div((y_max - y_origin), (pixel_size_y))) * pixel_size_y + y_origin # # # Create the destination data source x_res = int( round( old_div((x_max_round - x_min_round), pixel_size_x))) y_res = int( round( old_div((y_max_round - y_min_round), pixel_size_y))) # # # Create mask in memory mem_driver = gdal.GetDriverByName('MEM') mem_ds = mem_driver.Create('', x_res, y_res, 1, in_data_type_gdal) mask_geoT = [ x_min_round, pixel_size_x, 0, y_max_round, 0, -pixel_size_y ] mem_ds.SetGeoTransform(mask_geoT) mem_ds.SetProjection(orig_cs.ExportToWkt()) # # # Create a Layer with '1' for the pixels to be selected gdal.RasterizeLayer(mem_ds, [1], outLayer, burn_values=[1]) # gdal.RasterizeLayer(mem_ds, [1], outLayer, None, None, [1]) # Read the polygon-mask band = mem_ds.GetRasterBand(1) geo_values = mem_ds.ReadAsArray() # Create a mask from geo_values (mask-out the '0's) geo_mask = ma.make_mask(geo_values == 0) geo_mask_created = True # # # Clean/Close objects mem_ds = None mem_driver = None outDriver = None outLayer = None # Read data from input file x_offset = int(old_div((x_min - x_origin), pixel_size_x)) y_offset = int(old_div((y_origin - y_max), pixel_size_y)) band_in = orig_ds.GetRasterBand(1) data = band_in.ReadAsArray(x_offset, y_offset, x_res, y_res) # Catch the Error ES2-105 (polygon not included in Mapset) if data is None: logger.error( 'ERROR: polygon extends out of file mapset for file: %s' % infile) return [] # Create a masked array from the data (considering Nodata) masked_data = ma.masked_equal(data, nodata) # Apply on top of it the geo mask mxnodata = ma.masked_where(geo_mask, masked_data) # Test ONLY # write_ds_to_geotiff(mem_ds, '/data/processing/exchange/Tests/mem_ds.tif') if aggregate['aggregation_type'] == 'count' or aggregate[ 'aggregation_type'] == 'percent' or aggregate[ 'aggregation_type'] == 'surface' or aggregate[ 'aggregation_type'] == 'precip': if mxnodata.count() == 0: meanResult = None else: mxrange = mxnodata min_val = aggregate['aggregation_min'] max_val = aggregate['aggregation_max'] if min_val is not None: min_val_scaled = old_div((min_val - scale_offset), scale_factor) mxrange = ma.masked_less(mxnodata, min_val_scaled) # See ES2-271 if max_val is not None: # Scale threshold from physical to digital value max_val_scaled = old_div( (max_val - scale_offset), scale_factor) mxrange = ma.masked_greater( mxrange, max_val_scaled) elif max_val is not None: # Scale threshold from physical to digital value max_val_scaled = old_div((max_val - scale_offset), scale_factor) mxrange = ma.masked_greater( mxnodata, max_val_scaled) if aggregate['aggregation_type'] == 'percent': # 'percent' meanResult = float(mxrange.count()) / float( mxnodata.count()) * 100 elif aggregate['aggregation_type'] == 'surface': # 'surface' # Estimate 'average' Latitude y_avg = (y_min + y_max) / 2.0 pixelAvgArea = area_km_equator * math.cos( old_div(y_avg, 180) * math.pi) meanResult = float(mxrange.count()) * pixelAvgArea elif aggregate['aggregation_type'] == 'precip': # 'surface' # Estimate 'average' Latitude y_avg = (y_min + y_max) / 2.0 pixelAvgArea = area_km_equator * math.cos( old_div(y_avg, 180) * math.pi) n_pixels = mxnodata.count() avg_precip = mxnodata.mean() # Result is in km * km * mmm i.e. 1E3 m*m*m -> we divide by 1E3 to get 1E6 m*m*m meanResult = float( n_pixels) * pixelAvgArea * avg_precip * 0.001 else: # 'count' meanResult = float(mxrange.count()) # Both results are equal finalvalue = meanResult else: # if aggregate['type'] == 'mean' or if aggregate['type'] == 'cumulate': if mxnodata.count() == 0: finalvalue = None meanResult = None else: if aggregate['aggregation_type'] == 'mean': # 'mean' meanResult = mxnodata.mean() else: # 'cumulate' meanResult = mxnodata.sum() finalvalue = (meanResult * scale_factor + scale_offset) # Assign results single_result['filename'] = infile single_result['meanvalue_noscaling'] = meanResult single_result['meanvalue'] = finalvalue else: logger.debug('ERROR: raster file does not exist - %s' % infile) uniqueFilesValues.append(single_result) # Define a dictionary to associate filenames/values files_to_values_dict = dict( (x['filename'], x['meanvalue']) for x in uniqueFilesValues) # Prepare array for result resultDatesValues = [] # Returns a list of 'filenames', 'dates', 'values' for mydate in dates_list: my_result = {'date': datetime.date.today(), 'meanvalue': nodata} # Assign the date my_result['date'] = mydate # Assign the filename my_filename = dates_to_files_dict[mydate] # Map from array of Values my_result['meanvalue'] = files_to_values_dict[my_filename] # Map from array of dates resultDatesValues.append(my_result) try: shutil.rmtree(tmpdir) except: logger.debug('ERROR: Error in deleting tmpdir. Exit') # Return result return resultDatesValues else: logger.debug( 'ERROR: product not registered in the products table! - %s %s %s' % (productcode, subproductcode, version)) return []
def DenseFusion(self, img, depth, posecnn_res): my_result_wo_refine = [] itemid = 1 # this is simplified for single label decttion, if multi-label used, check DFYW3.py for more depth = np.array(depth) # img = img seg_res = posecnn_res x1, y1, x2, y2 = seg_res["box"] banana_bbox_draw = self.posecnn.get_box_rcwh(seg_res["box"]) rmin, rmax, cmin, cmax = int(x1), int(x2), int(y1), int(y2) depth = depth[:, :, 1] # because depth has 3 dimensions RGB but they are the all the same with each other mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) # ok label_banana = np.squeeze(seg_res["mask"]) label_banana = ma.getmaskarray(ma.masked_greater(label_banana, 0.5)) label_banana_nonzeros = label_banana.flatten().nonzero() mask_label = ma.getmaskarray(ma.masked_equal( label_banana, itemid)) # label from banana label mask = mask_label * mask_depth mask_nonzeros = mask[:].flatten().nonzero() choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) > self.num_points: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num_points] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: print("len of choose is 0, check error") choose = np.pad(choose, (0, self.num_points - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) xmap_masked = self.xmap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = self.ymap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) pt2 = depth_masked / self.cam_scale pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) img_np = np.array(img) img_masked = np.array(img)[:, :, :3] img_masked = np.transpose(img_masked, (2, 0, 1)) img_masked = img_masked[:, rmin:rmax, cmin:cmax] cloud = torch.from_numpy(cloud.astype(np.float32)) choose = torch.LongTensor(choose.astype(np.int32)) img_masked = self.norm(torch.from_numpy(img_masked.astype(np.float32))) index = torch.LongTensor([itemid - 1]) cloud = Variable(cloud).cuda() choose = Variable(choose).cuda() img_masked = Variable(img_masked).cuda() index = Variable(index).cuda() cloud = cloud.view(1, self.num_points, 3) img_masked = img_masked.view(1, 3, img_masked.size()[1], img_masked.size()[2]) pred_r, pred_t, pred_c, emb = self.estimator(img_masked, cloud, choose, index) pred_r = pred_r / torch.norm(pred_r, dim=2).view(1, self.num_points, 1) pred_c = pred_c.view(self.bs, self.num_points) how_max, which_max = torch.max(pred_c, 1) pred_t = pred_t.view(self.bs * self.num_points, 1, 3) points = cloud.view(self.bs * self.num_points, 1, 3) my_r = pred_r[0][which_max[0]].view(-1).cpu().data.numpy() my_t = (points + pred_t)[which_max[0]].view(-1).cpu().data.numpy() my_pred = np.append(my_r, my_t) my_result_wo_refine.append(my_pred.tolist()) my_result = [] for ite in range(0, self.iteration): T = Variable(torch.from_numpy( my_t.astype(np.float32))).cuda().view(1, 3).repeat( self.num_points, 1).contiguous().view(1, self.num_points, 3) my_mat = quaternion_matrix(my_r) R = Variable(torch.from_numpy(my_mat[:3, :3].astype( np.float32))).cuda().view(1, 3, 3) my_mat[0:3, 3] = my_t new_cloud = torch.bmm((cloud - T), R).contiguous() pred_r, pred_t = self.refiner(new_cloud, emb, index) pred_r = pred_r.view(1, 1, -1) pred_r = pred_r / (torch.norm(pred_r, dim=2).view(1, 1, 1)) my_r_2 = pred_r.view(-1).cpu().data.numpy() my_t_2 = pred_t.view(-1).cpu().data.numpy() my_mat_2 = quaternion_matrix(my_r_2) my_mat_2[0:3, 3] = my_t_2 my_mat_final = np.dot(my_mat, my_mat_2) my_r_final = copy.deepcopy(my_mat_final) my_r_final[0:3, 3] = 0 my_r_final = quaternion_from_matrix(my_r_final, True) my_t_final = np.array( [my_mat_final[0][3], my_mat_final[1][3], my_mat_final[2][3]]) my_pred = np.append(my_r_final, my_t_final) my_result.append(my_pred.tolist()) my_result_np = np.array(my_result) my_result_mean = np.mean(my_result, axis=0) my_r = my_result_mean[:4] my_t = my_result_mean[4:] my_r_quaternion = my_r return my_r_quaternion, my_t
def __getitem__(self, index): img = Image.open('{0}/{1}-color.png'.format(self.root, self.list[index])) depth = np.array( Image.open('{0}/{1}-depth.png'.format(self.root, self.list[index]))) label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, self.list[index]))) meta = scio.loadmat('{0}/{1}-meta.mat'.format(self.root, self.list[index])) symmetries = np.loadtxt(self.symdir + 'symmetries.txt') symmetries = symmetries.reshape(21, 5, 3) if self.list[index][:8] != 'data_syn' and int( self.list[index][5:9]) >= 60: cam_cx = self.cam_cx_2 cam_cy = self.cam_cy_2 cam_fx = self.cam_fx_2 cam_fy = self.cam_fy_2 else: cam_cx = self.cam_cx_1 cam_cy = self.cam_cy_1 cam_fx = self.cam_fx_1 cam_fy = self.cam_fy_1 cam_intri = [cam_cx, cam_cy, cam_fx, cam_fy] cam_intri = np.array(cam_intri) mask_back = ma.getmaskarray(ma.masked_equal(label, 0)) add_front = False if self.add_noise: for k in range(5): seed = random.choice(self.list) front = np.array( self.trancolor( Image.open('{0}/{1}-color.png'.format( self.root, seed)).convert("RGB"))) front = np.transpose(front, (2, 0, 1)) f_label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, seed))) front_label = np.unique(f_label).tolist()[1:] if len(front_label) < self.front_num: continue front_label = random.sample(front_label, self.front_num) for f_i in front_label: mk = ma.getmaskarray(ma.masked_not_equal(f_label, f_i)) if f_i == front_label[0]: mask_front = mk else: mask_front = mask_front * mk t_label = label * mask_front if len(t_label.nonzero()[0]) > 1000: label = t_label add_front = True break obj = meta['cls_indexes'].flatten().astype(np.int32) # get class index while 1: idx = np.random.randint(0, len(obj)) mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label, obj[idx])) mask = mask_label * mask_depth mask_real = len(mask.nonzero()[0]) if mask_real > self.minimum_num_pt: break if self.add_noise: img = self.trancolor(img) rmin, rmax, cmin, cmax = get_bbox(mask_label) img = np.transpose(np.array(img)[:, :, :3], (2, 0, 1))[:, rmin:rmax, cmin:cmax] img_masked = img if self.add_noise and add_front: img_masked = img_masked * mask_front[ rmin:rmax, cmin:cmax] + front[:, rmin:rmax, cmin:cmax] * ~( mask_front[rmin:rmax, cmin:cmax]) if self.list[index][:8] == 'data_syn': img_masked = img_masked + np.random.normal( loc=0.0, scale=7.0, size=img_masked.shape) order = idx target_r = meta['poses'][:, :, idx][:, 0:3] target_t = np.array([meta['poses'][:, :, idx][:, 3:4].flatten()]) add_t = np.array([ random.uniform(-self.noise_trans, self.noise_trans) for i in range(3) ]) # transform sym vectors into points cls_idx = int(obj[idx]) - 1 model_s = symmetries[cls_idx, :, :] target_mode = 0 if cls_idx in self.one_sym_list: multi_s = np.zeros((2, 3)) multi_s[0, :] = model_s[0, :] multi_s[1, :] = model_s[1, :] + model_s[0, :] if cls_idx in self.axis_and_ref_list: target_mode = 2 else: target_mode = 0 elif cls_idx in self.only_axis_list: multi_s = np.zeros((2, 3)) multi_s[0, :] = model_s[0, :] multi_s[1, :] = model_s[4, :] + model_s[0, :] target_mode = 1 elif cls_idx in self.two_sym_list: multi_s = np.zeros((3, 3)) multi_s[0, :] = model_s[0, :] multi_s[1, :] = model_s[1, :] + model_s[0, :] multi_s[2, :] = model_s[2, :] + model_s[0, :] target_mode = 0 elif cls_idx in self.three_sym_list: multi_s = np.zeros((4, 3)) multi_s[0, :] = model_s[0, :] multi_s[1, :] = model_s[1, :] + model_s[0, :] multi_s[2, :] = model_s[2, :] + model_s[0, :] multi_s[3, :] = model_s[3, :] + model_s[0, :] target_mode = 0 else: multi_s = np.zeros((5, 3)) # print("not in symmetry list") choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) > self.num_pt: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num_pt] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num_pt - len(choose)), 'wrap') depth_masked = depth[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) #(1000,1)get masked depth xmap_masked = self.xmap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) # (1000,1) ymap_masked = self.ymap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) # (1000,1) choose = np.array([choose]) # (1,1000) cam_scale = meta['factor_depth'][0][0] # cam_scale = 10000 pt2 = depth_masked / cam_scale pt0 = (ymap_masked - cam_cx) * pt2 / cam_fx pt1 = (xmap_masked - cam_cy) * pt2 / cam_fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) # (1000,3) # print('cloud_shape = ', cloud.shape) if self.add_noise: cloud = np.add(cloud, add_t) # (1000,3) dellist = [j for j in range(0, len(self.cld[obj[idx]]))] if self.refine: dellist = random.sample( dellist, len(self.cld[obj[idx]]) - self.num_pt_mesh_large) else: dellist = random.sample( dellist, len(self.cld[obj[idx]]) - self.num_pt_mesh_small) model_points = np.delete(self.cld[obj[idx]], dellist, axis=0) target = np.dot(model_points, target_r.T) target_s = np.add(np.dot(multi_s, target_r.T), target_t) target_num = target_s.shape[0] - 1 if self.add_noise: target = np.add(target, target_t + add_t) else: target = np.add(target, target_t) return torch.from_numpy(cloud.astype(np.float32)), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy(img_masked.astype(np.float32))), \ torch.LongTensor([int(obj[idx]) - 1]), \ torch.from_numpy(target_s.astype(np.float32)),\ torch.LongTensor([target_num]),\ torch.LongTensor([target_mode]), \ mask_real / (640 * 480)
def test_testOddFeatures(self): # Test of other odd features x = arange(20) x = x.reshape(4, 5) x.flat[5] = 12 assert_(x[1, 0] == 12) z = x + 10j * x assert_(eq(z.real, x)) assert_(eq(z.imag, 10 * x)) assert_(eq((z * conjugate(z)).real, 101 * x * x)) z.imag[...] = 0.0 x = arange(10) x[3] = masked assert_(str(x[3]) == str(masked)) c = x >= 8 assert_(count(where(c, masked, masked)) == 0) assert_(shape(where(c, masked, masked)) == c.shape) z = where(c, x, masked) assert_(z.dtype is x.dtype) assert_(z[3] is masked) assert_(z[4] is masked) assert_(z[7] is masked) assert_(z[8] is not masked) assert_(z[9] is not masked) assert_(eq(x, z)) z = where(c, masked, x) assert_(z.dtype is x.dtype) assert_(z[3] is masked) assert_(z[4] is not masked) assert_(z[7] is not masked) assert_(z[8] is masked) assert_(z[9] is masked) z = masked_where(c, x) assert_(z.dtype is x.dtype) assert_(z[3] is masked) assert_(z[4] is not masked) assert_(z[7] is not masked) assert_(z[8] is masked) assert_(z[9] is masked) assert_(eq(x, z)) x = array([1., 2., 3., 4., 5.]) c = array([1, 1, 1, 0, 0]) x[2] = masked z = where(c, x, -x) assert_(eq(z, [1., 2., 0., -4., -5])) c[0] = masked z = where(c, x, -x) assert_(eq(z, [1., 2., 0., -4., -5])) assert_(z[0] is masked) assert_(z[1] is not masked) assert_(z[2] is masked) assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2))) assert_(eq(masked_where(greater_equal(x, 2), x), masked_greater_equal(x, 2))) assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2))) assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2))) assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2))) assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2))) assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2))) assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4])) assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199])) assert_(eq(masked_inside(array(list(range(5)), mask=[1, 0, 0, 0, 0]), 1, 3).mask, [1, 1, 1, 1, 0])) assert_(eq(masked_outside(array(list(range(5)), mask=[0, 1, 0, 0, 0]), 1, 3).mask, [1, 1, 0, 0, 1])) assert_(eq(masked_equal(array(list(range(5)), mask=[1, 0, 0, 0, 0]), 2).mask, [1, 0, 1, 0, 0])) assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1], mask=[1, 0, 0, 0, 0]), 2).mask, [1, 0, 1, 0, 1])) assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]), [99, 99, 3, 4, 5])) atest = ones((10, 10, 10), dtype=np.float32) btest = zeros(atest.shape, MaskType) ctest = masked_where(btest, atest) assert_(eq(atest, ctest)) z = choose(c, (-x, x)) assert_(eq(z, [1., 2., 0., -4., -5])) assert_(z[0] is masked) assert_(z[1] is not masked) assert_(z[2] is masked) x = arange(6) x[5] = masked y = arange(6) * 10 y[2] = masked c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0]) cm = c.filled(1) z = where(c, x, y) zm = where(cm, x, y) assert_(eq(z, zm)) assert_(getmask(zm) is nomask) assert_(eq(zm, [0, 1, 2, 30, 40, 50])) z = where(c, masked, 1) assert_(eq(z, [99, 99, 99, 1, 1, 1])) z = where(c, 1, masked) assert_(eq(z, [99, 1, 1, 99, 99, 99]))
def extract_veldisp(gal_ID, z, ba, phi, r50, maps_filename, plot_diagnostics=True, IMAGE_DIR=None, IMAGE_FORMAT='eps'): ''' Extract the velocity dispersions as a function of distance from the center of the galaxy. PARAMETERS ========== gal_ID : string <plate> - <IFU> of galaxy z : float galaxy redshift ba : float galaxy semimajor to semiminor axis ratio phi : float Galaxy rotation angle E of N [degrees] r50 : float 50% light radius of galaxy (from elliptical petrosian fit) [arcsec] maps_filename : string Location of data map fits file plot_diagnostics : boolean Flag to determine whether or not to plot the various plot diagnostics. Default is True (plot all figures). IMAGE_DIR : string File path to which diagnostic images are saved. Default value is None (do not save images). IMAGE_FORMAT : string Saved image file format. Default format is eps. ''' ############################################################################ # Read in stellar velocity dispersion map #--------------------------------------------------------------------------- maps = fits.open(maps_filename) # Extract average r-band image r_band = maps['SPX_MFLUX'].data # Extract (and correct for instrumental resolution effects) stellar velocity # dispersion map star_sigma = np.sqrt(maps['STELLAR_SIGMA'].data**2 - maps['STELLAR_SIGMACORR'].data**2) # See https://www.sdss.org/dr16/manga/manga-data/working-with-manga-data/ for # correction details. # Stellar velocity dispersion mask extension star_sigma_mask_extension = maps['STELLAR_SIGMA'].header['QUALDATA'] #--------------------------------------------------------------------------- # Mask arrays #--------------------------------------------------------------------------- mStar_sigma = ma.array(star_sigma, mask=maps[star_sigma_mask_extension].data > 0) mr_band = ma.masked_equal(r_band, 0) ############################################################################ ############################################################################ # Check if all of the array is masked. #--------------------------------------------------------------------------- num_masked_spaxels = np.sum(mStar_sigma.mask) - np.sum(mr_band.mask) frac_masked_spaxels = num_masked_spaxels/np.sum(~mr_band.mask) unmasked_data = False if frac_masked_spaxels < 1: unmasked_data = True ############################################################################ ############################################################################ # Find coordinates of center spaxel # # The center of the galaxy is defined to be the same as the brightest spaxel # in the galaxy. #--------------------------------------------------------------------------- if unmasked_data: optical_center = np.asarray( mr_band.max() == mr_band).nonzero() x_center = optical_center[1][0] y_center = optical_center[0][0] else: rows, cols = mr_band.shape x_center = int(0.5*rows) y_center = int(0.5*cols) optical_center = ([y_center], [x_center]) ############################################################################ ############################################################################ # Normalize velocity dispersion values by central value #--------------------------------------------------------------------------- mStar_sigma_norm = mStar_sigma/mStar_sigma[optical_center] ############################################################################ ############################################################################ # Calculate distance from each spaxel to center spaxel, in spaxel units #--------------------------------------------------------------------------- y, x = np.indices(mStar_sigma.shape) distance_spaxels = np.hypot(x - x_center, y - y_center) deproj_distance_spaxels = deproject_r(x - x_center, y - y_center, distance_spaxels, ba, phi) ############################################################################ ############################################################################ # Convert distances to arcseconds, kpc #--------------------------------------------------------------------------- distance_arcsec = distance_spaxels*MANGA_SPAXEL_SIZE deproj_distance_arcsec = deproj_distance_spaxels*MANGA_SPAXEL_SIZE deproj_distance_arcsec_norm = deproj_distance_arcsec/r50 dist_to_galaxy_kpc = ( z * c / (H0/1000)) spaxel_scale_factor = dist_to_galaxy_kpc * np.tan( MANGA_SPAXEL_SIZE*(1/60)*(1/60)*(np.pi/180)) distance_kpc = spaxel_scale_factor*distance_spaxels deproj_distance_kpc = spaxel_scale_factor*deproj_distance_spaxels ############################################################################ ############################################################################ # Create table of distances (spaxels, arcseconds, and kpc) and stellar # velocity dispersion value (raw, normalized to center spaxel value) #--------------------------------------------------------------------------- veldisp_table = Table() veldisp_table['vel_disp'] = mStar_sigma.flatten() veldisp_table['vel_disp_norm'] = mStar_sigma_norm.flatten() veldisp_table['r_spaxels'] = distance_spaxels.flatten() veldisp_table['r_deproj_spaxels'] = deproj_distance_spaxels.flatten() veldisp_table['r_arcsec'] = distance_arcsec.flatten() veldisp_table['r_deproj_arcsec'] = deproj_distance_arcsec.flatten() veldisp_table['r_deproj_arcsec_norm'] = deproj_distance_arcsec_norm.flatten() veldisp_table['r_kpc'] = distance_kpc.flatten() veldisp_table['r_deproj_kpc'] = deproj_distance_kpc.flatten() ############################################################################ if plot_diagnostics: plot_diagnostic_panel(gal_ID, mr_band, mStar_sigma, mStar_sigma_norm, veldisp_table, deproj_distance_arcsec_norm, IMAGE_DIR=IMAGE_DIR) plot_veldisp_r( veldisp_table['r_deproj_arcsec_norm'], veldisp_table['vel_disp_norm'], gal_ID, IMAGE_DIR=IMAGE_DIR) if IMAGE_DIR is None: plt.show() return veldisp_table
def setUp(self): masked_data = ma.masked_equal([1, 2, 3, 4, 5], 3) self.cube = Cube(as_lazy_data(masked_data)) self.cube.add_dim_coord(DimCoord([6, 7, 8, 9, 10], long_name="foo"), 0)
def msd_calc(track, length=10): """Calculates mean squared displacement of input track. Returns numpy array containing MSD data calculated from an individual track. Parameters ---------- track : pandas.core.frame.DataFrame Contains, at a minimum a 'Frame', 'X', and 'Y' column Returns ------- new_track : pandas.core.frame.DataFrame Similar to input track. All missing frames of individual trajectories are filled in with NaNs, and two new columns, MSDs and Gauss are added: MSDs, calculated mean squared displacements using the formula MSD = <(xpos-x0)**2> Gauss, calculated Gaussianity Examples -------- >>> data1 = {'Frame': [1, 2, 3, 4, 5], ... 'X': [5, 6, 7, 8, 9], ... 'Y': [6, 7, 8, 9, 10]} >>> df = pd.DataFrame(data=data1) >>> new_track = msd.msd_calc(df, 5) >>> data1 = {'Frame': [1, 2, 3, 4, 5], ... 'X': [5, 6, 7, 8, 9], ... 'Y': [6, 7, 8, 9, 10]} >>> df = pd.DataFrame(data=data1) >>> new_track = msd.msd_calc(df) """ meansd = np.zeros(length) gauss = np.zeros(length) new_frame = np.linspace(1, length, length) old_frame = track['Frame'] oldxy = [track['X'], track['Y']] fxy = [interpolate.interp1d(old_frame, oldxy[0], bounds_error=False, fill_value=np.nan), interpolate.interp1d(old_frame, oldxy[1], bounds_error=False, fill_value=np.nan)] intxy = [ma.masked_equal(fxy[0](new_frame), np.nan), ma.masked_equal(fxy[1](new_frame), np.nan)] data1 = {'Frame': new_frame, 'X': intxy[0], 'Y': intxy[1] } new_track = pd.DataFrame(data=data1) for frame in range(0, length-1): xy = [np.square(nth_diff(new_track['X'], n=frame+1)), np.square(nth_diff(new_track['Y'], n=frame+1))] with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) meansd[frame+1] = np.nanmean(xy[0] + xy[1]) gauss[frame+1] = np.nanmean(xy[0]**2 + xy[1]**2 )/(2*(meansd[frame+1]**2)) new_track['MSDs'] = pd.Series(meansd, index=new_track.index) new_track['Gauss'] = pd.Series(gauss, index=new_track.index) return new_track
def predict(self, rgb_img, depth, label_img, label, bboxes, intrinsic_matrix): num_points = self.num_points iteration = self.iteration batch_size = 1 lst = np.array(label.flatten(), dtype=np.int32) img_height, img_width, _ = rgb_img.shape cameramodel = cameramodels.PinholeCameraModel.from_intrinsic_matrix( intrinsic_matrix, img_height, img_width) translations = [] rotations = [] for idx in range(len(lst)): itemid = lst[idx] rmin, cmin, rmax, cmax = bboxes[idx] mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label_img, itemid)) mask = mask_label * mask_depth choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: translations.append([]) rotations.append([]) continue if len(choose) > num_points: c_mask = np.zeros(len(choose), dtype=int) c_mask[:num_points] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, num_points - len(choose)), 'wrap') xmap = np.array([[j for i in range(img_width)] for j in range(img_height)]) ymap = np.array([[i for i in range(img_width)] for j in range(img_height)]) depth_masked = depth[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) xmap_masked = xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) ymap_masked = ymap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) choose = np.array([choose]) cloud = cameramodel.batch_project_pixel_to_3d_ray( np.concatenate([ymap_masked, xmap_masked], axis=1), depth_masked) img_masked = np.array(rgb_img)[:, :, :3] img_masked = np.transpose(img_masked, (2, 0, 1)) img_masked = img_masked[:, rmin:rmax, cmin:cmax] with torch.no_grad(): cloud = torch.from_numpy(cloud.astype(np.float32)) choose = torch.LongTensor(choose.astype(np.int32)) img_masked = normalize( torch.from_numpy(img_masked.astype(np.float32))) index = torch.LongTensor([itemid - 1]) cloud = cloud.cuda() choose = choose.cuda() img_masked = img_masked.cuda() index = index.cuda() cloud = cloud.view(1, num_points, 3) img_masked = img_masked.view(1, 3, img_masked.size()[1], img_masked.size()[2]) pred_rot, pred_trans, pred_score, emb = self.estimator( img_masked, cloud, choose, index) pred_rot = pred_rot / torch.norm(pred_rot, dim=2).view( 1, num_points, 1) pred_score = pred_score.view(batch_size, num_points) _, which_max = torch.max(pred_score, 1) pred_trans = pred_trans.view(batch_size * num_points, 1, 3) points = cloud.view(batch_size * num_points, 1, 3) rotation = pred_rot[0][which_max[0]].view(-1).cpu().\ data.numpy() translation = (points + pred_trans)[which_max[0]].view(-1).\ cpu().data.numpy() for _ in range(iteration): T = torch.from_numpy(translation.astype(np.float32)).\ cuda().view(1, 3).\ repeat(num_points, 1).contiguous().\ view(1, num_points, 3) trans_matrix = np.eye(4) trans_matrix[:3, :3] = quaternion2matrix( quaternion_normalize(rotation)) R = torch.from_numpy(trans_matrix[:3, :3].astype( np.float32)).cuda().view(1, 3, 3) trans_matrix[0:3, 3] = translation new_cloud = torch.bmm((cloud - T), R).contiguous() refined_rot, refined_trans = self.refiner( new_cloud, emb, index) refined_rot = refined_rot.view(1, 1, -1) refined_rot = refined_rot / (torch.norm( refined_rot, dim=2).view(1, 1, 1)) rotation_2 = refined_rot.view(-1).cpu().data.numpy() translation_2 = refined_trans.view(-1).cpu().data.numpy() trans_matrix_2 = np.eye(4) trans_matrix_2[:3, :3] = quaternion2matrix( quaternion_normalize(rotation_2)) trans_matrix_2[0:3, 3] = translation_2 trans_matrix_final = np.dot(trans_matrix, trans_matrix_2) rotation_final = matrix2quaternion( trans_matrix_final[:3, :3]) translation_final = np.array([ trans_matrix_final[0][3], trans_matrix_final[1][3], trans_matrix_final[2][3] ]) rotation = rotation_final translation = translation_final translations.append(translation) rotations.append(quaternion_normalize(rotation)) return rotations, translations
def plot_all_experiments(experiments, bucket='ccurtis.data', folder='test', yrange=(10**-1, 10**1), fps=100.02, xrange=(10**-2, 10**0), upload=True, outfile='test.png', exponential=True): """Plots precision-weighted averages of MSD datasets. Plots pre-calculated precision-weighted averages of MSD datasets calculated from precision_averaging and stored in an AWS S3 bucket. Parameters ---------- group : list of str List of experiment names to plot. Each experiment must have an MSD and SEM file associated with it in s3. bucket : str S3 bucket from which to download data. folder : str Folder in s3 bucket from which to download data. yrange : list of float Y range of plot xrange: list of float X range of plot upload : bool True to upload to S3 outfile : str Filename of output image """ n = len(experiments) color = iter(cm.viridis(np.linspace(0, 0.9, n))) fig = plt.figure(figsize=(8.5, 8.5)) plt.xlim(xrange[0], xrange[1]) plt.ylim(yrange[0], yrange[1]) plt.xlabel('Tau (s)', fontsize=25) plt.ylabel(r'Mean Squared Displacement ($\mu$m$^2$)', fontsize=25) geo = {} gstder = {} counter = 0 for experiment in experiments: aws.download_s3('{}/geomean_{}.csv'.format(folder, experiment), 'geomean_{}.csv'.format(experiment), bucket_name=bucket) aws.download_s3('{}/geoSEM_{}.csv'.format(folder, experiment), 'geoSEM_{}.csv'.format(experiment), bucket_name=bucket) geo[counter] = np.genfromtxt('geomean_{}.csv'.format(experiment)) gstder[counter] = np.genfromtxt('geoSEM_{}.csv'.format(experiment)) geo[counter] = ma.masked_equal(geo[counter], 0.0) gstder[counter] = ma.masked_equal(gstder[counter], 0.0) frames = np.shape(gstder[counter])[0] xpos = np.linspace(0, frames-1, frames)/fps c = next(color) if exponential: plt.loglog(xpos, np.exp(geo[counter]), c=c, linewidth=6, label=experiment) plt.loglog(xpos, np.exp(geo[counter] - 1.96*gstder[counter]), c=c, dashes=[6, 2], linewidth=4) plt.loglog(xpos, np.exp(geo[counter] + 1.96*gstder[counter]), c=c, dashes=[6, 2], linewidth=4) else: plt.loglog(xpos, geo[counter], c=c, linewidth=6, label=experiment) plt.loglog(xpos, geo[counter] - 1.96*gstder[counter], c=c, dashes=[6, 2], linewidth=4) plt.loglog(xpos, geo[counter] + 1.96*gstder[counter], c=c, dashes=[6, 2], linewidth=4) counter = counter + 1 plt.legend(frameon=False, prop={'size': 16}) if upload: fig.savefig(outfile, bbox_inches='tight') aws.upload_s3(outfile, folder+'/'+outfile, bucket_name=bucket)
def precision_averaging(group, geomean, geo_stder, weights, save=True, bucket='ccurtis.data', folder='test', experiment='test'): """Calculates precision-weighted averages of MSD datasets. Parameters ---------- group : list of str List of experiment names to average. Each element corresponds to a key in geo_stder and geomean. geomean : dict of numpy.ndarray Each entry in dictionary corresponds to an MSD profiles, they key corresponding to an experiment name. geo_stder : dict of numpy.ndarray Each entry in dictionary corresponds to the standard errors of an MSD profile, the key corresponding to an experiment name. weights : numpy.ndarray Precision weights to be used in precision averaging. Returns ------- geo : numpy.ndarray Precision-weighted averaged MSDs from experiments specified in group geo_stder : numpy.ndarray Precision-weighted averaged SEMs from experiments specified in group """ frames = np.shape(geo_stder[group[0]])[0] slices = len(group) video_counter = 0 geo_holder = np.zeros((slices, frames)) gstder_holder = np.zeros((slices, frames)) w_holder = np.zeros((slices, frames)) for sample in group: w_holder[video_counter, :] = (1/(geo_stder[sample]*geo_stder[sample]) )/weights geo_holder[video_counter, :] = w_holder[video_counter, : ] * geomean[sample] gstder_holder[video_counter, :] = 1/(geo_stder[sample]*geo_stder[sample] ) video_counter = video_counter + 1 w_holder = ma.masked_equal(w_holder, 0.0) w_holder = ma.masked_equal(w_holder, 1.0) geo_holder = ma.masked_equal(geo_holder, 0.0) geo_holder = ma.masked_equal(geo_holder, 1.0) gstder_holder = ma.masked_equal(gstder_holder, 0.0) gstder_holder = ma.masked_equal(gstder_holder, 1.0) geo = ma.sum(geo_holder, axis=0) geo_stder = ma.sqrt((1/ma.sum(gstder_holder, axis=0))) if save: geo_f = 'geomean_{}.csv'.format(experiment) gstder_f = 'geoSEM_{}.csv'.format(experiment) np.savetxt(geo_f, geo, delimiter=',') np.savetxt(gstder_f, geo_stder, delimiter=',') aws.upload_s3(geo_f, '{}/{}'.format(folder, geo_f), bucket_name=bucket) aws.upload_s3(gstder_f, '{}/{}'.format(folder, gstder_f), bucket_name=bucket) geodata = Bunch(geomean=geo, geostd=geo_stder, weighthold=w_holder, geostdhold=gstder_holder) return geodata
def geomean_msdisp(prefix, umppx=0.16, fps=100.02, upload=True, remote_folder="01_18_Experiment", bucket='ccurtis.data', backup_frames=651): """Comptes geometric averages of mean squared displacement datasets Calculates geometric averages and stadard errors for MSD datasets. Might error out if not formatted as output from all_msds2. Parameters ---------- prefix : string Prefix of file name to be plotted e.g. features_P1.csv prefix is P1. umppx : float Microns per pixel of original images. fps : float Frames per second of video. upload : bool True if you want to upload to s3. remote_folder : string Folder in S3 bucket to upload to. bucket : string Name of S3 bucket to upload to. Returns ------- geo_mean : numpy.ndarray Geometric mean of trajectory MSDs at all time points. geo_stder : numpy.ndarray Geometric standard errot of trajectory MSDs at all time points. """ merged = pd.read_csv('msd_{}.csv'.format(prefix)) try: particles = int(max(merged['Track_ID'])) frames = int(max(merged['Frame'])) ypos = np.zeros((particles+1, frames+1)) for i in range(0, particles+1): ypos[i, :] = merged.loc[merged.Track_ID == i, 'MSDs']*umppx*umppx xpos = merged.loc[merged.Track_ID == i, 'Frame']/fps geo_mean = np.nanmean(ma.log(ypos), axis=0) geo_stder = ma.masked_equal(stats.sem(ma.log(ypos), axis=0, nan_policy='omit'), 0.0) except ValueError: geo_mean = np.nan*np.ones(backup_frames) geo_stder = np.nan*np.ones(backup_frames) np.savetxt('geomean_{}.csv'.format(prefix), geo_mean, delimiter=",") np.savetxt('geoSEM_{}.csv'.format(prefix), geo_stder, delimiter=",") if upload: aws.upload_s3('geomean_{}.csv'.format(prefix), remote_folder+'/'+'geomean_{}.csv'.format(prefix), bucket_name=bucket) aws.upload_s3('geoSEM_{}.csv'.format(prefix), remote_folder+'/'+'geoSEM_{}.csv'.format(prefix), bucket_name=bucket) return geo_mean, geo_stder
depth = np.array(Image.open('{0}/{1}-depth.png'.format(opt.dataset_root, testlist[now]))) posecnn_meta = scio.loadmat('{0}/results_PoseCNN_RSS2018/{1}.mat'.format(ycb_toolbox_dir, '%06d' % now)) label = np.array(posecnn_meta['labels']) posecnn_rois = np.array(posecnn_meta['rois']) lst = posecnn_rois[:, 1:2].flatten() my_result_wo_refine = [] my_result = [] for idx in range(len(lst)): itemid = lst[idx] try: rmin, rmax, cmin, cmax = get_bbox(posecnn_rois) mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label, itemid)) mask = mask_label * mask_depth choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] # generate randomly num_points points from the posecnn segmentation # yes but why? --> for pointnet if len(choose) > num_points: c_mask = np.zeros(len(choose), dtype=int) c_mask[:num_points] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, num_points - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) xmap_masked = xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
def full_prediction(image, depth, meta, segmentor, estimator, refiner, to_tensor, normalize, device, cuda, color_dict, class_names=None, point_clouds=None, plot=False, color_prediction=False, bbox=False, put_text=False): start_time = time.time() output_dict = {'predictions': {}, 'elapsed_times': {}} if color_prediction: output_dict['segmented_prediction'] = np.array(copy.deepcopy(image), dtype=np.float) output_dict['pose_prediction'] = np.array(copy.deepcopy(image), dtype=np.float) # preprocesses input x = copy.deepcopy(image) x = to_tensor(x) x = normalize(x) x = x.to(device) x = x.unsqueeze(0) # get segmetation label pred = segmentor.predict(x) pred = F.softmax(pred, dim=1) if cuda: pred = pred.cpu() pred = pred[0] # crop and preproceses label to pass into the pose estimation pred_arg = torch.argmax(pred, dim=0).numpy() found_cls, counts = np.unique(pred_arg, return_counts=True) if 0 in found_cls: start = 1 else: start = 0 for i, cls in enumerate(found_cls[start:]): if counts[i+start] > 100: cls_pred_arg = copy.deepcopy(pred_arg) cls_pred_arg[cls_pred_arg != cls] = 0 cls_pred = cls_pred_arg * pred[cls].numpy() ret, labels = cv2.connectedComponents(np.array(cls_pred_arg, dtype=np.uint8), connectivity=8) biggest = 1 biggest_score = 0 unique = np.unique(labels) if 0 in unique: start2 = 1 else: start2 = 0 for u in unique[start2:]: score = np.mean(cls_pred[labels == u]) if score > biggest_score: biggest_score = score biggest = u cls_pred[labels != biggest] = 0 cls_pred[cls_pred != 0] = 255 cls_pred = np.array(cls_pred, dtype=np.uint8) output_dict['predictions'][class_names[cls-1]] = {'mask': cls_pred} if color_prediction: for c, color_value in enumerate(color_dict[class_names[cls-1]]['value']): c_mask = np.zeros(cls_pred.shape) c_mask[cls_pred != 0] = color_value output_dict['segmented_prediction'][:, :, c][cls_pred != 0] = \ output_dict['segmented_prediction'][:, :, c][cls_pred != 0] * 0.7 + c_mask[cls_pred != 0] * 0.3 if bbox: bbox = get_bbox(cls_pred) cv2.rectangle(output_dict['segmented_prediction'], (bbox[2], bbox[0]), (bbox[3], bbox[1]), color_dict[class_names[cls-1]]['value'], 2) if put_text: bbox = get_bbox(cls_pred) try: cv2.putText(output_dict['segmented_prediction'], 'Segmentation', (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2, cv2.LINE_AA) cv2.putText(output_dict['segmented_prediction'], class_names[cls - 1], (bbox[2] + 10, bbox[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, color_dict[class_names[cls-1]]['value'], 2, cv2.LINE_AA) except: pass if color_prediction: output_dict['segmented_prediction'][output_dict['segmented_prediction'] < 0] = 0 output_dict['segmented_prediction'][output_dict['segmented_prediction'] > 255] = 255 output_dict['segmented_prediction'] = np.array(output_dict['segmented_prediction'], dtype=np.uint8) output_dict['elapsed_times']['segmentation'] = time.time()-start_time start_time_pose = time.time() xmap = np.array([[j for i in range(640)] for j in range(480)]) ymap = np.array([[i for i in range(640)] for j in range(480)]) num_points = 1000 # estimate the pose for cls in output_dict['predictions']: #print('cls name', cls, class_names.index(cls)) mask_label = ma.getmaskarray(ma.masked_equal(output_dict['predictions'][cls]['mask'], 255)) rmin, rmax, cmin, cmax = get_bbox(mask_label) mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask = mask_label * mask_depth # select some points on the object choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: continue if len(choose) > num_points: c_mask = np.zeros(len(choose), dtype=int) c_mask[:num_points] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, num_points - len(choose)), 'wrap') # select the choosen points depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) xmap_masked = xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = ymap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) # compute the cartesian space position of each pixel and sample to point cloud pt2 = depth_masked * meta['depth_scale'] pt0 = (ymap_masked - meta['intr']['ppx']) * pt2 / meta['intr']['fx'] pt1 = (xmap_masked - meta['intr']['ppy']) * pt2 / meta['intr']['fy'] points = np.concatenate((pt0, pt1, pt2), axis=1) #np_points = copy.deepcopy(points) points = torch.from_numpy(points.astype(np.float32)).unsqueeze(0).to(device) choose = torch.LongTensor(choose.astype(np.int32)).unsqueeze(0).to(device) img = np.transpose(np.array(image)[:, :, :3], (2, 0, 1))[:, rmin:rmax, cmin:cmax] img = normalize(torch.from_numpy(img.astype(np.float32))).unsqueeze(0).to(device) idx = torch.LongTensor([int(class_names.index(cls))]).unsqueeze(0).to(device) pred_r, pred_t, pred_c, emb = estimator(img, points, choose, idx) new_points = get_new_points(pred_r, pred_t, pred_c, points) _, my_r, my_t = my_estimator_prediction(pred_r, pred_t, pred_c, num_points, 1, points) # refine the pose for ite in range(0, 2): pred_r, pred_t = refiner(new_points, emb, idx) _, my_r, my_t = my_refined_prediction(pred_r, pred_t, my_r, my_t) output_dict['predictions'][cls]['position'] = my_t output_dict['predictions'][cls]['rotation'] = my_r if color_prediction: my_r = quaternion_matrix(my_r)[:3, :3] np_pred = np.dot(point_clouds[class_names.index(cls)], my_r.T) np_pred = np.add(np_pred, my_t) output_dict['pose_prediction'] = pc_utils.pointcloud2image(output_dict['pose_prediction'], np_pred, 3, meta['intr'], color=color_dict[cls]['value']) if put_text: try: cv2.putText(output_dict['pose_prediction'], 'Pose Estimation', (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2, cv2.LINE_AA) except: pass if color_prediction: output_dict['pose_prediction'][output_dict['pose_prediction'] < 0] = 0 output_dict['pose_prediction'][output_dict['pose_prediction'] > 255] = 255 output_dict['pose_prediction'] = np.array(output_dict['pose_prediction'], dtype=np.uint8) output_dict['elapsed_times']['pose_estimation'] = time.time() - start_time_pose if plot and color_prediction: title = '' for cls in output_dict['predictions']: title += '{}: {}, '.format(color_dict[cls]['tag'], cls) fig, axs = plt.subplots(1, 2, constrained_layout=True) fig.suptitle(title) plt.subplot(1, 2, 1) plt.imshow(output_dict['segmented_prediction']) plt.axis('off') plt.title('segmented_prediction') plt.subplot(1, 2, 2) plt.imshow(output_dict['pose_prediction']) plt.axis('off') plt.title('pose_prediction') plt.show() del_keys = [] for cls in output_dict['predictions'].keys(): #print(output_dict['predictions'][cls].keys()) if 'position' not in output_dict['predictions'][cls]: del_keys.append(cls) elif 'rotation' not in output_dict['predictions'][cls]: del_keys.append(cls) elif 'mask' not in output_dict['predictions'][cls]: del_keys.append(cls) for cls in del_keys: print('Deleting cls "{}"'.format(cls)) del output_dict['predictions'][cls] output_dict['elapsed_times']['total'] = time.time() - start_time # return dict with found objects and their pose. also return the painted image return output_dict
def __getitem__(self, idx): index = random.randint(0, self.data_len - 10) label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, self.path[index]))) meta = scio.loadmat('{0}/{1}-meta.mat'.format(self.root, self.path[index])) if not self.use_noise: rgb = np.array( Image.open('{0}/{1}-color.png'.format( self.root, self.path[index])).convert("RGB")) else: rgb = np.array( self.trancolor( Image.open('{0}/{1}-color.png'.format( self.root, self.path[index])).convert("RGB"))) if self.path[index][:8] == 'data_syn': rgb = Image.open('{0}/{1}-color.png'.format( self.root, self.path[index])).convert("RGB") rgb = ImageEnhance.Brightness(rgb).enhance(1.5).filter( ImageFilter.GaussianBlur(radius=0.8)) rgb = np.array(self.trancolor(rgb)) seed = random.randint(0, self.back_len - 10) back = np.array( self.trancolor( Image.open('{0}/{1}-color.png'.format( self.root, self.path[seed])).convert("RGB"))) back_label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, self.path[seed]))) mask = ma.getmaskarray(ma.masked_equal(label, 0)) back = np.transpose(back, (2, 0, 1)) rgb = np.transpose(rgb, (2, 0, 1)) rgb = rgb + np.random.normal(loc=0.0, scale=5.0, size=rgb.shape) rgb = back * mask + rgb label = back_label * mask + label rgb = np.transpose(rgb, (1, 2, 0)) # scipy.misc.imsave('embedding_final/rgb_{0}.png'.format(index), rgb) # scipy.misc.imsave('embedding_final/label_{0}.png'.format(index), label) if self.use_noise: choice = random.randint(0, 3) if choice == 0: rgb = np.fliplr(rgb) label = np.fliplr(label) elif choice == 1: rgb = np.flipud(rgb) label = np.flipud(label) elif choice == 2: rgb = np.fliplr(rgb) rgb = np.flipud(rgb) label = np.fliplr(label) label = np.flipud(label) obj = meta['cls_indexes'].flatten().astype(np.int32) obj = np.append(obj, [0], axis=0) target = copy.deepcopy(label) rgb = np.transpose(rgb, (2, 0, 1)) orig = copy.deepcopy(rgb) rgb = self.norm(torch.from_numpy(rgb.astype(np.float32))) #rgb = torch.from_numpy(rgb.astype(np.float32)) target = torch.from_numpy(target.astype(np.int64)) #return rgb, target # Normal return '{0}{1}'.format( self.root, self.path[index]), orig, rgb, target # To test and see orig image
def setUp(self): self.cube = iris.cube.Cube(ma.masked_equal([1, 2, 3, 4, 5], 3)) self.cube.add_dim_coord(DimCoord([6, 7, 8, 9, 10], long_name='foo'), 0) self.func = lambda x: x >= 3
def __getitem__(self, index): if self.list_rgb[index].find('renders') != -1: img = Image.open(self.list_rgb[index]) ori_img = np.array(img) depth = np.array(Image.open(self.list_depth[index])) label = np.array(Image.open(self.list_label[index])) pose = pickle.load(open(self.list_pose[index], 'rb'), encoding='utf-8')['RT'] cx = self.render_cx cy = self.render_cy fx = self.render_fx fy = self.render_fy mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(1))) mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask = mask_label * mask_depth elif self.list_rgb[index].find('fuse') != -1: img = Image.open(self.list_rgb[index]) ori_img = np.array(img) depth = np.array(Image.open(self.list_depth[index])) label = np.array(Image.open(self.list_label[index])) cx = self.cam_cx cy = self.cam_cy fx = self.cam_fx fy = self.cam_fy idx = self.objlist.index(self.obj_id) pose = pickle.load(open(self.list_pose[index], 'rb'), encoding='utf-8')[1][idx] begins = pickle.load(open(self.list_pose[index], 'rb'), encoding='utf-8')[0][idx] mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(self.obj_id))) mask = mask_label * mask_depth cx += begins[1] cy += begins[0] else: img = Image.open(self.list_rgb[index]) ori_img = np.array(img) depth = np.array(Image.open(self.list_depth[index])) label = np.array(Image.open(self.list_label[index])) pose = np.load(self.list_pose[index]) cx = self.cam_cx cy = self.cam_cy fx = self.cam_fx fy = self.cam_fy if len(label.shape) == 2: mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255))) else: mask_label = ma.getmaskarray(ma.masked_equal(label, np.array([255, 255, 255])))[:, :, 0] mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask = mask_label * mask_depth if self.mode == 'train': img = self.trancolor(img) img = np.array(img)[:, :, :3] img = np.transpose(img, (2, 0, 1)) img_masked = img # rmin, rmax, cmin, cmax = get_bbox_mask(mask) rmin, rmax, cmin, cmax = get_bbox(mask_to_bbox(mask_label)) img_masked = img_masked[:, rmin:rmax, cmin:cmax] choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: cc = torch.LongTensor([0]) return(cc, cc, cc, cc, cc, cc, cc, cc, cc, cc) if len(choose) > self.num: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) xmap_masked = self.xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = self.ymap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) cam_scale = 1.0 pt2 = depth_masked / cam_scale pt0 = (ymap_masked - cx) * pt2 / fx pt1 = (xmap_masked - cy) * pt2 / fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) / 1000 model_points = self.pt / 1000 dellist = [j for j in range(0, len(model_points))] dellist = random.sample(dellist, len(model_points) - self.num) model_points = np.delete(model_points, dellist, axis=0) target_r = pose[:3, :3] target_t = pose[:3, 3] target = np.dot(model_points, target_r.T) target = np.add(target, target_t) model_kp = self.kp scene_kp = np.add(np.dot(model_kp, target_r.T), target_t) vertex_gt = compute_vertex_hcoords(cloud, scene_kp) return torch.from_numpy(cloud.astype(np.float32)), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy((img_masked/255).astype(np.float32))), \ torch.from_numpy(target.astype(np.float32)), \ torch.from_numpy(model_points.astype(np.float32)), \ torch.from_numpy(model_kp.astype(np.float32)), \ torch.from_numpy(vertex_gt.astype(np.float32)), \ torch.LongTensor([0]), \ torch.from_numpy(target_r.astype(np.float32)), \ torch.from_numpy(target_t.astype(np.float32))
nc2 = xr.open_dataset(fname2) sfcelv_hydroweb2 = nc2.sfcelv_hydroweb.values sfcelv_cmf2 = nc2.sfcelv_cmf.values lons2 = nc2.lon.values lats2 = nc2.lat.values pnames2 = nc2.name.values pnum2 = len(pnames2) print np.shape(sfcelv_cmf2), pnum2 #- masked out -9999 values sfcelv_cmf2 = ma.masked_where(sfcelv_hydroweb2 == -9999.0, sfcelv_cmf2).filled(-9999.0) #sfcelv_hydroweb=ma.masked_where(sfcelv_hydroweb==-9999.0,sfcelv_hydroweb) sfcelv_diff2 = ma.masked_where(sfcelv_hydroweb2 == -9999.0, (sfcelv_cmf2 - sfcelv_hydroweb2)**2).filled(-9999.0) sfcelv_rmse2 = np.mean(ma.masked_equal(sfcelv_diff2, -9999.0), axis=0) #.compressed()# sfcelv_rmse2 = sfcelv_rmse2.filled() print np.shape(sfcelv_rmse2) print sfcelv_rmse2[0:10] # sfcelv_bias_com=ma.masked_equal(sfcelv_bias1,-9999.0).compressed() os.system("mkdir -p ./fig") # river width sup = 2 w = 0.02 alpha = 1 width = 0.5 land = "#C0C0C0"
def draw_subregions(subregions, lats, lons, fname, fmt='png', ptitle='', parallels=None, meridians=None, subregion_masks=None): ''' Draw subregion domain(s) on a map. :param subregions: The subregion objects to plot on the map. :type subregions: :class:`list` of subregion objects (Bounds objects) :param lats: Array of latitudes values. :type lats: :class:`numpy.ndarray` :param lons: Array of longitudes values. :type lons: :class:`numpy.ndarray` :param fname: The filename of the plot. :type fname: :mod:`string` :param fmt: (Optional) filetype for the output. :type fmt: :mod:`string` :param ptitle: (Optional) plot title. :type ptitle: :mod:`string` :param parallels: (Optional) :class:`list` of :class:`int` or :class:`float` for the parallels to be drawn. See the `Basemap documentation <http://matplotlib.org/basemap/users/graticule.html>`_ for additional information. :type parallels: :class:`list` of :class:`int` or :class:`float` :param meridians: (Optional) :class:`list` of :class:`int` or :class:`float` for the meridians to be drawn. See the `Basemap documentation <http://matplotlib.org/basemap/users/graticule.html>`_ for additional information. :type meridians: :class:`list` of :class:`int` or :class:`float` :param subregion_masks: (Optional) :class:`dict` of :class:`bool` arrays for each subregion for giving finer control of the domain to be drawn, by default the entire domain is drawn. :type subregion_masks: :class:`dict` of :class:`bool` arrays ''' # Set up the figure fig = plt.figure() fig.set_size_inches((8.5, 11.)) fig.dpi = 300 ax = fig.add_subplot(111) # Determine the map boundaries and construct a Basemap object lonmin = lons.min() lonmax = lons.max() latmin = lats.min() latmax = lats.max() m = Basemap(projection='cyl', llcrnrlat=latmin, urcrnrlat=latmax, llcrnrlon=lonmin, urcrnrlon=lonmax, resolution='l', ax=ax) # Draw the borders for coastlines and countries m.drawcoastlines(linewidth=1) m.drawcountries(linewidth=.75) m.drawstates() # Create default meridians and parallels. The interval between # them should be 1, 5, 10, 20, 30, or 40 depending on the size # of the domain length = max((latmax - latmin), (lonmax - lonmin)) / 5 if length <= 1: dlatlon = 1 elif length <= 5: dlatlon = 5 else: dlatlon = np.round(length, decimals=-1) if meridians is None: meridians = np.r_[np.arange(0, -180, -dlatlon)[::-1], np.arange(0, 180, dlatlon)] if parallels is None: parallels = np.r_[np.arange(0, -90, -dlatlon)[::-1], np.arange(0, 90, dlatlon)] # Draw parallels / meridians m.drawmeridians(meridians, labels=[0, 0, 0, 1], linewidth=.75, fontsize=10) m.drawparallels(parallels, labels=[1, 0, 0, 1], linewidth=.75, fontsize=10) # Set up the color scaling cmap = plt.cm.rainbow norm = mpl.colors.BoundaryNorm(np.arange(1, len(subregions) + 3), cmap.N) # Process the subregions for i, reg in enumerate(subregions): if subregion_masks is not None and reg.name in subregion_masks.keys(): domain = (i + 1) * subregion_masks[reg.name] else: domain = (i + 1) * np.ones((2, 2)) nlats, nlons = domain.shape domain = ma.masked_equal(domain, 0) reglats = np.linspace(reg.lat_min, reg.lat_max, nlats) reglons = np.linspace(reg.lon_min, reg.lon_max, nlons) reglons, reglats = np.meshgrid(reglons, reglats) # Convert to to projection coordinates. Not really necessary # for cylindrical projections but keeping it here in case we need # support for other projections. x, y = m(reglons, reglats) # Draw the subregion domain m.pcolormesh(x, y, domain, cmap=cmap, norm=norm, alpha=.5) # Label the subregion xm, ym = x.mean(), y.mean() m.plot(xm, ym, marker='$%s$' % ("R" + str(i + 1)), markersize=12, color='k') # Add the title ax.set_title(ptitle) # Save the figure fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight', dpi=fig.dpi) fig.clf()
def draw_subregions(subregions, lats, lons, fname, fmt='png', ptitle='', parallels=None, meridians=None, subregion_masks=None): ''' Purpose:: Function to draw subregion domain(s) on a map Input:: subregions - a list of subRegion objects lats - array of latitudes lons - array of longitudes fname - a string specifying the filename of the plot fmt - an optional string specifying the filetype, default is .png ptitle - an optional string specifying plot title parallels - an optional list of ints or floats for the parallels to be drawn meridians - an optional list of ints or floats for the meridians to be drawn subregion_masks - optional dictionary of boolean arrays for each subRegion for giving finer control of the domain to be drawn, by default the entire domain is drawn. ''' # Set up the figure fig = plt.figure() fig.set_size_inches((8.5, 11.)) fig.dpi = 300 ax = fig.add_subplot(111) # Determine the map boundaries and construct a Basemap object lonmin = lons.min() lonmax = lons.max() latmin = lats.min() latmax = lats.max() m = Basemap(projection='cyl', llcrnrlat=latmin, urcrnrlat=latmax, llcrnrlon=lonmin, urcrnrlon=lonmax, resolution='l', ax=ax) # Draw the borders for coastlines and countries m.drawcoastlines(linewidth=1) m.drawcountries(linewidth=.75) m.drawstates() # Create default meridians and parallels. The interval between # them should be 1, 5, 10, 20, 30, or 40 depending on the size # of the domain length = max((latmax - latmin), (lonmax - lonmin)) / 5 if length <= 1: dlatlon = 1 elif length <= 5: dlatlon = 5 else: dlatlon = np.round(length, decimals=-1) if meridians is None: meridians = np.r_[np.arange(0, -180, -dlatlon)[::-1], np.arange(0, 180, dlatlon)] if parallels is None: parallels = np.r_[np.arange(0, -90, -dlatlon)[::-1], np.arange(0, 90, dlatlon)] # Draw parallels / meridians m.drawmeridians(meridians, labels=[0, 0, 0, 1], linewidth=.75, fontsize=10) m.drawparallels(parallels, labels=[1, 0, 0, 1], linewidth=.75, fontsize=10) # Set up the color scaling cmap = plt.cm.rainbow norm = mpl.colors.BoundaryNorm(np.arange(1, len(subregions) + 3), cmap.N) # Process the subregions for i, reg in enumerate(subregions): if subregion_masks is not None and reg.name in subregion_masks.keys(): domain = (i + 1) * subregion_masks[reg.name] else: domain = (i + 1) * np.ones((2, 2)) nlats, nlons = domain.shape domain = ma.masked_equal(domain, 0) reglats = np.linspace(reg.latmin, reg.latmax, nlats) reglons = np.linspace(reg.lonmin, reg.lonmax, nlons) reglons, reglats = np.meshgrid(reglons, reglats) # Convert to to projection coordinates. Not really necessary # for cylindrical projections but keeping it here in case we need # support for other projections. x, y = m(reglons, reglats) # Draw the subregion domain m.pcolormesh(x, y, domain, cmap=cmap, norm=norm, alpha=.5) # Label the subregion xm, ym = x.mean(), y.mean() m.plot(xm, ym, marker='$%s$' % (reg.name), markersize=12, color='k') # Add the title ax.set_title(ptitle) # Save the figure fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight', dpi=fig.dpi) fig.clf()
def __getitem__(self, index): img = Image.open(self.list_rgb[index]) ori_img = np.array(img) depth = np.array(Image.open(self.list_depth[index])) label = np.array(Image.open(self.list_label[index])) obj = self.list_obj[index] rank = self.list_rank[index] if obj == 2: for i in range(0, len(self.meta[obj][rank])): if self.meta[obj][rank][i]['obj_id'] == 2: meta = self.meta[obj][rank][i] break else: meta = self.meta[obj][rank][0] mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) if self.mode == 'eval': mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255))) else: mask_label = ma.getmaskarray( ma.masked_equal(label, np.array([255, 255, 255])))[:, :, 0] mask = mask_label * mask_depth if self.add_noise: img = self.trancolor(img) img = np.array(img)[:, :, :3] img = np.transpose(img, (2, 0, 1)) img_masked = img if self.mode == 'eval': rmin, rmax, cmin, cmax = get_bbox(mask_to_bbox(mask_label)) else: rmin, rmax, cmin, cmax = get_bbox(meta['obj_bb']) img_masked = img_masked[:, rmin:rmax, cmin:cmax] #p_img = np.transpose(img_masked, (1, 2, 0)) #scipy.misc.imsave('evaluation_result/{0}_input.png'.format(index), p_img) target_r = np.resize(np.array(meta['cam_R_m2c']), (3, 3)) target_t = np.array(meta['cam_t_m2c']) add_t = np.array([ random.uniform(-self.noise_trans, self.noise_trans) for i in range(3) ]) choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: cc = torch.LongTensor([0]) return (cc, cc, cc, cc, cc, cc) if len(choose) > self.num: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) xmap_masked = self.xmap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = self.ymap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) cam_scale = 1.0 pt2 = depth_masked / cam_scale pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) cloud = cloud / 1000.0 if self.add_noise: cloud = np.add(cloud, add_t) #fw = open('evaluation_result/{0}_cld.xyz'.format(index), 'w') #for it in cloud: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) #fw.close() model_points = self.pt[obj] / 1000.0 dellist = [j for j in range(0, len(model_points))] dellist = random.sample(dellist, len(model_points) - self.num_pt_mesh_small) model_points = np.delete(model_points, dellist, axis=0) #fw = open('evaluation_result/{0}_model_points.xyz'.format(index), 'w') #for it in model_points: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) #fw.close() target = np.dot(model_points, target_r.T) if self.add_noise: target = np.add(target, target_t / 1000.0 + add_t) out_t = target_t / 1000.0 + add_t else: target = np.add(target, target_t / 1000.0) out_t = target_t / 1000.0 #fw = open('evaluation_result/{0}_tar.xyz'.format(index), 'w') #for it in target: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) #fw.close() return torch.from_numpy(cloud.astype(np.float32)), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy(img_masked.astype(np.float32))), \ torch.from_numpy(target.astype(np.float32)), \ torch.from_numpy(model_points.astype(np.float32)), \ torch.LongTensor([self.objlist.index(obj)])
def mainFunction(f): ############################################################################# """ # biomass predF = '/vol/v3/lt_stem_v3.1/models/biomassfiaald_20180708_0859/2000/biomassfiaald_20180708_0859_2000_mean.tif' trainF = '/vol/v2/datasets/biomass/nbcd/fia_ald/nbcd_fia_ald_biomass_clipped_to_conus.tif' shpF = '/vol/v2/datasets/Eco_Level_III_US/us_eco_l3_no_states_multipart.shp' trainND = -32768 predND = -9999 trgField = 'US_L3CODE' descrField = 'US_L3NAME' outDir = '/vol/v3/lt_stem_v3.1/evaluation/biomassfiaald_20180708_0859/ecoregion_correlation' xyLim = (500, 500) xLab = 'Reference (tons/ha)' yLab = 'Prediction (tons/ha)' annoXY = (15,420) """ # biomass hexagon predF = '/vol/v3/lt_stem_v3.1/models/biomassfiaald_20180708_0859/2000/biomassfiaald_20180708_0859_2000_mean.tif' trainF = '/vol/v2/datasets/biomass/nbcd/fia_ald/nbcd_fia_ald_biomass_clipped_to_conus.tif' shpF = '/vol/v1/general_files/datasets/spatial_data/hexagons/hexagons_conus_albers_30km_with_id.shp' trainND = -32768 predND = -9999 trgField = 'id' descrField = 'id' outDir = '/vol/v3/lt_stem_v3.1/evaluation/biomassfiaald_20180708_0859/hexagon_correlation' xyLim = (500, 500) xLab = 'Reference (tons/ha)' yLab = 'Prediction (tons/ha)' annoXY = (15, 420) """ # cc predF = '/vol/v3/lt_stem_v3.1/models/canopy_20180915_1631/2001/canopy_20180915_1631_2001_mean.tif' trainF = '/vol/v2/stem/conus/reference_rasters/nlcd_2001_canopy_clipped_to_conus_train.tif' #shpF = '/vol/v2/datasets/Eco_Level_III_US/us_eco_l3_no_states_multipart.shp' shpF = '/vol/v1/general_files/datasets/spatial_data/hexagons/hexagons_conus_albers_30km_with_id.shp' trainND = 255 predND = 255 trgField = 'id' descrField = 'id' #trgField = 'US_L3CODE' #descrField = 'US_L3NAME' #outDir = '/vol/v3/lt_stem_v3.1/evaluation/canopy_20180915_1631/ecoregion_correlation' outDir = '/vol/v3/lt_stem_v3.1/evaluation/canopy_20180915_1631/hexagon_correlation' xyLim = (100, 100) xLab = 'Reference (%)' yLab = 'Prediction (%)' annoXY = (5,82) """ ############################################################################# # get color setup norm = colors.Normalize(vmin=0, vmax=1) f2rgb = cm.ScalarMappable(norm=norm, cmap=cm.get_cmap('YlGnBu_r')) # open the shapefile vDriver = ogr.GetDriverByName("ESRI Shapefile") vSrc = vDriver.Open(shpF, 0) vLayer = vSrc.GetLayer() commonBox = get_intersec([predF, trainF]) #for f in range(vLayer.GetFeatureCount()): feature = vLayer[f] name = feature.GetField(trgField) print('f: ' + str(f)) outFig = os.path.join( outDir, (trgField.replace(' ', '_').lower() + '_' + str(name) + '.png')) if os.path.exists(outFig): #break return descr = feature.GetField(descrField) predP, coords = get_zone_pixels(feature, shpF, predF, 1, [ commonBox[0], commonBox[2], commonBox[3], commonBox[1] ]) #.compressed() [commonBox[0], commonBox[2], commonBox[3], commonBox[1]] trainP, coords = get_zone_pixels( feature, shpF, trainF, 1, [coords[0], coords[1], coords[2], coords[3]]) #.compressed() predP = ma.masked_equal(predP, predND) trainP = ma.masked_equal(trainP, trainND) trainP = ma.masked_equal(trainP, 0) combMask = np.logical_not( np.logical_not(predP.mask) * np.logical_not(trainP.mask)) predP[combMask] = ma.masked trainP[combMask] = ma.masked predP = predP.compressed() trainP = trainP.compressed() if (predP.shape[0] == 0) | (trainP.shape[0] == 0) | (predP == 0).all() | ( trainP == 0).all(): predP = np.array([0, 0, 1, 1], dtype='float64') trainP = np.array([0, 0, 1, 1], dtype='float64') mae = round(np.mean(np.absolute(np.subtract(predP, trainP))), 1) rmse = round(np.sqrt(np.mean((predP - trainP)**2)), 1) totPixs = trainP.shape[0] sampSize = round(totPixs * 1) pickFrom = range(sampSize) #sampIndex = np.random.choice(pickFrom, size=sampSize) sampIndex = pickFrom r = round(np.corrcoef(trainP[sampIndex], predP[sampIndex])[0][1], 2) if (mae == 0) & (r == 1): r = 0.0 rColor = f2hex(f2rgb, r) p = sns.jointplot(trainP[sampIndex], predP[sampIndex], kind="hex", color='blue', xlim=(0, xyLim[0]), ylim=(0, xyLim[1]), size=5) p.ax_joint.set_xlabel(xLab) p.ax_joint.set_ylabel(yLab) p.ax_joint.annotate( 'r: ' + str(r) + '\nrmse: ' + str(rmse) + '\nmae: ' + str(mae), annoXY) plt.tight_layout() outFig = os.path.join( outDir, (trgField.replace(' ', '_').lower() + '_' + str(name) + '.png')) p.savefig(outFig) df = pd.DataFrame( { 'id': name, 'descr': descr, 'r': r, 'rmse': rmse, 'mae': mae, 'color': rColor, 'img': os.path.basename(outFig) }, index=[0]) outCSV = outFig.replace('.png', '.csv') df.to_csv(outCSV, ',', index=False)
def __getitem__(self, index): # print("------------------- Get data -------------------") img = Image.open('{0}/{1}-color.png'.format(self.root, self.list[index])) depth = np.array( Image.open('{0}/{1}-depth.png'.format(self.root, self.list[index]))) label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, self.list[index]))) meta = scio.loadmat('{0}/{1}-meta.mat'.format(self.root, self.list[index])) if self.list[index][:8] != 'data_syn' and int( self.list[index][5:9]) >= 60: cam_cx = self.cam_cx_2 cam_cy = self.cam_cy_2 cam_fx = self.cam_fx_2 cam_fy = self.cam_fy_2 else: cam_cx = self.cam_cx_1 cam_cy = self.cam_cy_1 cam_fx = self.cam_fx_1 cam_fy = self.cam_fy_1 mask_back = ma.getmaskarray(ma.masked_equal(label, 0)) add_front = False if self.add_noise: for k in range(5): seed = random.choice(self.syn) front = np.array( self.trancolor( Image.open('{0}/{1}-color.png'.format( self.root, seed)).convert("RGB"))) front = np.transpose(front, (2, 0, 1)) f_label = np.array( Image.open('{0}/{1}-label.png'.format(self.root, seed))) front_label = np.unique(f_label).tolist()[1:] if len(front_label) < self.front_num: continue front_label = random.sample(front_label, self.front_num) for f_i in front_label: mk = ma.getmaskarray(ma.masked_not_equal(f_label, f_i)) if f_i == front_label[0]: mask_front = mk else: mask_front = mask_front * mk t_label = label * mask_front if len(t_label.nonzero()[0]) > 1000: label = t_label add_front = True break obj = meta['cls_indexes'].flatten().astype(np.int32) while 1: idx = np.random.randint(0, len(obj)) mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label, obj[idx])) mask = mask_label * mask_depth # print("idx : " + str(idx)) # print("mask : " + str(mask)) if len(mask.nonzero()[0]) > self.minimum_num_pt: break if self.add_noise: img = self.trancolor(img) rmin, rmax, cmin, cmax = get_bbox(mask_label) img = np.transpose(np.array(img)[:, :, :3], (2, 0, 1))[:, rmin:rmax, cmin:cmax] if self.list[index][:8] == 'data_syn': seed = random.choice(self.real) back = np.array( self.trancolor( Image.open('{0}/{1}-color.png'.format( self.root, seed)).convert("RGB"))) back = np.transpose(back, (2, 0, 1))[:, rmin:rmax, cmin:cmax] img_masked = back * mask_back[rmin:rmax, cmin:cmax] + img else: img_masked = img if self.add_noise and add_front: img_masked = img_masked * mask_front[ rmin:rmax, cmin:cmax] + front[:, rmin:rmax, cmin:cmax] * ~( mask_front[rmin:rmax, cmin:cmax]) if self.list[index][:8] == 'data_syn': img_masked = img_masked + np.random.normal( loc=0.0, scale=7.0, size=img_masked.shape) # p_img = np.transpose(img_masked, (1, 2, 0)) # scipy.misc.imsave('temp/{0}_input.png'.format(index), p_img) # scipy.misc.imsave('temp/{0}_label.png'.format(index), mask[rmin:rmax, cmin:cmax].astype(np.int32)) target_r = meta['poses'][:, :, idx][:, 0:3] target_t = np.array([meta['poses'][:, :, idx][:, 3:4].flatten()]) add_t = np.array([ random.uniform(-self.noise_trans, self.noise_trans) for i in range(3) ]) choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) > self.num_pt: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num_pt] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num_pt - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) xmap_masked = self.xmap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = self.ymap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) cam_scale = meta['factor_depth'][0][0] pt2 = depth_masked / cam_scale pt0 = (ymap_masked - cam_cx) * pt2 / cam_fx pt1 = (xmap_masked - cam_cy) * pt2 / cam_fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) if self.add_noise: cloud = np.add(cloud, add_t) # fw = open('temp/{0}_cld.xyz'.format(index), 'w') # for it in cloud: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) # fw.close() dellist = [j for j in range(0, len(self.cld[obj[idx]]))] if self.refine: dellist = random.sample( dellist, len(self.cld[obj[idx]]) - self.num_pt_mesh_large) else: dellist = random.sample( dellist, len(self.cld[obj[idx]]) - self.num_pt_mesh_small) model_points = np.delete(self.cld[obj[idx]], dellist, axis=0) # fw = open('temp/{0}_model_points.xyz'.format(index), 'w') # for it in model_points: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) # fw.close() target = np.dot(model_points, target_r.T) if self.add_noise: target = np.add(target, target_t + add_t) else: target = np.add(target, target_t) # fw = open('temp/{0}_tar.xyz'.format(index), 'w') # for it in target: # fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2])) # fw.close() return torch.from_numpy(cloud.astype(np.float32)), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy(img_masked.astype(np.float32))), \ torch.from_numpy(target.astype(np.float32)), \ torch.from_numpy(model_points.astype(np.float32)), \ torch.LongTensor([int(obj[idx]) - 1])
def __getitem__(self, index): img = Image.open(self.imgs[index]) ori_img = np.array(img) depth = np.array(Image.open(self.depths[index])) label = np.array(Image.open(self.labels[index])) obj = self.objects[index] rank = self.ranks[index] if obj == 2: for i in range(0, len(self.meta[obj][rank])): if self.meta[obj][rank][i]['obj_id'] == 2: meta = self.meta[obj][rank][i] break else: meta = self.meta[obj][rank][0] mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255))) mask = mask_label * mask_depth img = np.array(img)[:, :, :3] # add noise to image rows, cols, chs = img.shape gauss = np.random.normal(0, self.sigma, (rows, cols, chs)) * 255.0 gauss = np.reshape(gauss.astype(img.dtype), (rows, cols, chs)) img = img + gauss # # // show noisy image # cv2.imshow(f'Noisy img std {self.sigma}', cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) # cv2.waitKey(0) # # // img = np.transpose(img, (2, 0, 1)) masked_img = img masked_bbox = mask_2_bbox(mask_label) rmin, rmax, cmin, cmax = get_bbox(masked_bbox) masked_img = masked_img[:, rmin:rmax, cmin:cmax] # # // Uncomment to show cropped image # crop_img = np.transpose(masked_img, (1, 2, 0)) # cv2.imshow('Cropped image', cv2.cvtColor(crop_img, cv2.COLOR_RGB2BGR)) # # // target_r = np.resize(np.array(meta['cam_R_m2c']), (3, 3)) target_t = np.array(meta['cam_t_m2c']) choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: cc = torch.LongTensor([0]) return (cc, cc, cc, cc, cc, cc) if len(choose) > self.num: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num - len(choose)), 'wrap') depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype( np.float32) xmap_masked = self.xmap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) ymap_masked = self.ymap[ rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32) choose = np.array([choose]) cam_scale = 1.0 pt2 = depth_masked / cam_scale pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy cloud = np.concatenate((pt0, pt1, pt2), axis=1) cloud = cloud / 1000.0 # add noise to cloud rows, cols = cloud.shape gauss = np.random.normal(0, self.sigma, (rows, cols)).astype(cloud.dtype) gauss = np.reshape(gauss, (rows, cols)) cloud = cloud + gauss # # // show input cloud # pcd_input = o3d.geometry.PointCloud() # pcd_input.points = o3d.utility.Vector3dVector(cloud) # pcd_input.paint_uniform_color([0, 0, 1]) # visualize(pcd_input, self.imgs[index], self.depths[index], 'Input point mask') # o3d.visualization.draw_geometries([pcd_input], window_name=f'Input cloud with sigma {self.sigma}') # # // model_pts = self.pt[obj] / 1000.0 dellist = [j for j in range(0, len(model_pts))] dellist = random.sample(dellist, len(model_pts) - self.num_pt_mesh_small) model_pts = np.delete(model_pts, dellist, axis=0) # # // show init pose # pcd_init = o3d.geometry.PointCloud() # pcd_init.points = o3d.utility.Vector3dVector(model_pts) # pcd_init.paint_uniform_color([1, 0, 0]) # visualize(pcd_init, self.imgs[index], self.depths[index], 'Init') # o3d.visualization.draw_geometries([pcd_init], window_name='Init pose') # # // target = np.dot(model_pts, target_r.T) target = np.add(target, target_t / 1000.0) # # // show ground truth # pcd_target = o3d.geometry.PointCloud() # pcd_target.points = o3d.utility.Vector3dVector(target) # pcd_target.paint_uniform_color([1, 0, 0]) # visualize(pcd_target, self.imgs[index], self.depths[index], 'Ground truth pose') # o3d.visualization.draw_geometries([pcd_target], window_name='Ground truth cloud') # # // out_t = target_t / 1000.0 return torch.from_numpy(cloud.astype(np.float32)), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy(masked_img.astype(np.float32))), \ torch.from_numpy(target.astype(np.float32)), \ torch.from_numpy(model_pts.astype(np.float32)), \ torch.LongTensor([self.objs.index(obj)])
nclouds = np.mean(nclouds_bin_all, axis=0) hn_normalized = np.mean(hn_normalized_all, axis=0) filledbin_all = np.argmin(hn_normalized[:]) fit = CSD_fit(hn_normalized[0:10], size[0:10]) logfit = fit[3] a = fit[0] b = fit[1] c = fit[2] print 'a, b, c:' print a, b, c print logfit sizelog = np.log10(size) hnlog = ma.filled(np.log10(ma.masked_equal(hn_normalized, 0)), np.nan) ncloudslog = ma.filled(np.log10(ma.masked_equal(nclouds, 0)), np.nan) #res = ma.filled(log2(ma.masked_equal(m, 0)), 0) mindistance_plus = mindistance_mean + mindistance_std mindistance_minus = mindistance_mean - mindistance_std filledbin = np.argmin(mindistance_mean) slope, intercept, r_value, p_value, std_err = stats.linregress( size[0:filledbin], mindistance_mean[0:filledbin]) print 'r-squared:', r_value**2 line = intercept + slope * size print 'slope:', slope
def _NTAI_CAL(date,spl_arr, CoverageID_LST): #request image cube for the specified date and area by WCS. #firstly we get the temporal length of avaliable NDVI data from the DescribeCoverage of WCS endpoint='http://192.168.1.104:8080/rasdaman/ows' field={} field['SERVICE']='WCS' field['VERSION']='2.0.1' field['REQUEST']='DescribeCoverage' field['COVERAGEID']=CoverageID_LST#'LST_MOD11C2005_uptodate'#'LST_MOD11C2005'#'trmm_3b42_coverage_1' url_values = urllib.urlencode(field,doseq=True) full_url = endpoint + '?' + url_values data = urllib.urlopen(full_url).read() root = etree.fromstring(data) lc = root.find(".//{http://www.opengis.net/gml/3.2}lowerCorner").text uc = root.find(".//{http://www.opengis.net/gml/3.2}upperCorner").text start_date=int((lc.split(' '))[2]) end_date=int((uc.split(' '))[2]) #print [start_date, end_date] #generate the dates list cur_date=datetime.strptime(date,"%Y-%m-%d") #startt=145775 start=datetime.fromtimestamp((start_date-(datetime(1970,01,01)-datetime(1601,01,01)).days)*24*60*60) #print start #tmp_date=datetime(start.year,cur_date.month,cur_date.day) #if tmp_date > start : # start=(tmp_date-datetime(1601,01,01)).days #else: start=(datetime(start.year+1,cur_date.month,cur_date.day)-datetime(1601,01,01)).days #datelist=range(start+1,end_date-1,365) #print datelist #find the position of the requested date in the datelist #cur_epoch=(cur_date-datetime(1601,01,01)).days #cur_pos=min(range(len(datelist)),key=lambda x:abs(datelist[x]-cur_epoch)) #print ('Current position:',cur_pos) try: datelist, cur_pos = datelist_irregular_coverage(root, start_date, start, cur_date) print 'irregular' logging.info('irregular') except IndexError: datelist, cur_pos = datelist_regular_coverage(root, start_date, start, cur_date) print 'regular' logging.info('regular') #retrieve the data cube cube_arr=[] for d in datelist: print 'LST', d field={} field['SERVICE']='WCS' field['VERSION']='2.0.1' field['REQUEST']='GetCoverage' field['COVERAGEID']=CoverageID_LST#'LST_MOD11C2005_uptodate'#'LST_MOD11C2005'#'trmm_3b42_coverage_1' field['SUBSET']=['ansi('+str(d)+')', 'Lat('+str(spl_arr[1])+','+str(spl_arr[3])+')', 'Long('+str(spl_arr[0])+','+str(spl_arr[2])+')'] field['FORMAT']='image/tiff' url_values = urllib.urlencode(field,doseq=True) full_url = endpoint + '?' + url_values #print full_url tmpfilename='test'+str(d)+'.tif' f,h = urllib.urlretrieve(full_url,tmpfilename) #print h ds=gdal.Open(tmpfilename) cube_arr.append(ds.ReadAsArray()) #print d #calculate the regional VCI cube_arr_ma=ma.masked_equal(numpy.asarray(cube_arr),0) #nan val is 0 for lst #VCI=(cube_arr_ma[cur_pos,:,:]-numpy.amin(cube_arr_ma,0))*1.0/(numpy.amax(cube_arr_ma,0)-numpy.amin(cube_arr_ma,0)) NTAI=(cube_arr_ma[cur_pos,:,:]-numpy.mean(cube_arr_ma,0))*1.0/(numpy.amax(cube_arr_ma,0)-numpy.amin(cube_arr_ma,0)) return NTAI, cur_date
def gran_AOT(aotList, shrink=1): ''' Returns the granulated AOT ''' try: reload(viirsAero) reload(viirs_edr_data) del (viirsAeroObj) del (latArr) del (lonArr) del (aotArr) del (retArr) del (qualArr) del (lsmArr) except: pass LOG.debug("Creating viirsAeroObj...") reload(viirsAero) viirsAeroObj = viirsAero.viirsAero() LOG.debug("done") # Determine the correct fillValue trimObj = ViirsTrimTable() eps = 1.e-6 # Build up the swath... for grans in np.arange(len(aotList)): LOG.debug("\nIngesting granule {} ...".format(grans)) retList = viirsAeroObj.ingest(aotList[grans], 'aot', shrink, 'linear') try: latArr = np.vstack((latArr, viirsAeroObj.Lat[:, :])) lonArr = np.vstack((lonArr, viirsAeroObj.Lon[:, :])) ModeGran = viirsAeroObj.ModeGran LOG.debug("subsequent geo arrays...") except NameError: latArr = viirsAeroObj.Lat[:, :] lonArr = viirsAeroObj.Lon[:, :] ModeGran = viirsAeroObj.ModeGran LOG.debug("first geo arrays...") try: aotArr = np.vstack((aotArr, viirsAeroObj.ViirsAProdSDS[:, :])) retArr = np.vstack((retArr, viirsAeroObj.ViirsAProdRet[:, :])) qualArr = np.vstack((qualArr, viirsAeroObj.ViirsCMquality[:, :])) #lsmArr = np.vstack((lsmArr ,viirsAeroObj.LandSeaMask[:,:])) LOG.debug("subsequent aot arrays...") except NameError: aotArr = viirsAeroObj.ViirsAProdSDS[:, :] retArr = viirsAeroObj.ViirsAProdRet[:, :] qualArr = viirsAeroObj.ViirsCMquality[:, :] #lsmArr = viirsAeroObj.LandSeaMask[:,:] LOG.debug("first aot arrays...") LOG.debug("Intermediate aotArr.shape = {}".format(str(aotArr.shape))) LOG.debug("Intermediate retArr.shape = {}".format(str(retArr.shape))) LOG.debug("Intermediate qualArr.shape = {}".format(str(qualArr.shape))) #LOG.debug("Intermediate lsmArr.shape = {}".format(str(lsmArr.shape))) lat_0 = latArr[np.shape(latArr)[0] / 2, np.shape(latArr)[1] / 2] lon_0 = lonArr[np.shape(lonArr)[0] / 2, np.shape(lonArr)[1] / 2] LOG.debug("lat_0,lon_0 = ", lat_0, lon_0) try: # Determine masks for each fill type, for the VCM IP aotFillMasks = {} for fillType in trimObj.sdrTypeFill.keys(): fillValue = trimObj.sdrTypeFill[fillType][aotArr.dtype.name] if 'float' in fillValue.__class__.__name__: aotFillMasks[fillType] = ma.masked_inside( aotArr, fillValue - eps, fillValue + eps).mask if (aotFillMasks[fillType].__class__.__name__ != 'ndarray'): aotFillMasks[fillType] = None elif 'int' in fillValue.__class__.__name__: aotFillMasks[fillType] = ma.masked_equal(aotArr, fillValue).mask if (aotFillMasks[fillType].__class__.__name__ != 'ndarray'): aotFillMasks[fillType] = None else: LOG.debug("Dataset was neither int not float... a worry") pass # Construct the total mask from all of the various fill values fillMask = ma.array(np.zeros(aotArr.shape, dtype=np.bool)) for fillType in trimObj.sdrTypeFill.keys(): if aotFillMasks[fillType] is not None: fillMask = fillMask * ma.array(np.zeros(aotArr.shape,dtype=np.bool),\ mask=aotFillMasks[fillType]) # Define any masks based on the quality flags... ViirsCMqualityMask = ma.masked_equal(qualArr, 0) # VCM quality == poor ViirsAProdRetMask = ma.masked_not_equal(retArr, 0) # Interp/NAAPS/Climo # Define the land and water masks #ViirsLandMask = ma.masked_greater(lsmArr,1) #ViirsWaterMask = ma.masked_less(lsmArr,2) # Define the total mask totalMask = fillMask * ViirsCMqualityMask * ViirsAProdRetMask try: data = ma.array(aotArr, mask=totalMask.mask) lats = ma.array(latArr, mask=totalMask.mask) lons = ma.array(lonArr, mask=totalMask.mask) except ma.core.MaskError: LOG.debug( ">> error: Mask Error, probably mismatched geolocation and product array sizes, aborting..." ) sys.exit(1) except Exception, err: LOG.debug(">> error: {}...".format(str(err))) sys.exit(1)
def __getitem__(self, index): #outputs: #pointcloud: xyz points of projected cropped depth image (just the object region) #choose: select num_points points from the cropped region of the image, this is a mask #img_masked: cropped region of image (RGB) with object #target: Rt matrix #model_points: just the model points, num_pt_mesh_small of them #obj: class (in our case, just a single number since we only have 1 object) img = Image.open(self.list_rgb[index]) ori_img = np.array(img) depth = np.array(Image.open(self.list_depth[index])) label = np.array(Image.open(self.list_label[index])) obj = self.list_obj[index] #transform indices are 1 off from image indices gt_trans = self.meta[obj][self.list_meta[index] + 1] #remove infinities mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, np.max(depth))) #get mask crop mask_label = ma.getmaskarray(ma.masked_equal(label, 65535)) mask = mask_label * mask_depth if self.add_noise: img = self.trancolor(img) img = np.array(img)[:, :, :3] #remove horizon from unity by setting infinite distance poitns to gray img[depth == np.max(depth)] = self.gray img = np.transpose(img, (2, 0, 1)) img_masked = img rmin, rmax, cmin, cmax = get_bbox(mask_label) img_masked = img_masked[:, rmin:rmax, cmin:cmax] target_r_quat = convert_quat(gt_trans[1]) target_r = quaternion_rotation_matrix(target_r_quat) target_t = gt_trans[0] * 1000 target_t[2] = -target_t[2] add_t = np.array([ random.uniform(-self.noise_trans, self.noise_trans) for i in range(3) ]) choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0] if len(choose) == 0: cc = torch.LongTensor([0]) return (cc, cc, cc, cc, cc, cc) if len(choose) > self.num: c_mask = np.zeros(len(choose), dtype=int) c_mask[:self.num] = 1 np.random.shuffle(c_mask) choose = choose[c_mask.nonzero()] else: choose = np.pad(choose, (0, self.num - len(choose)), 'wrap') depth_projected = self.udp.project_depth(depth)[rmin:rmax, cmin:cmax].reshape( (-1, 3)) depth_masked = depth_projected[choose].astype(np.float32) choose = np.array([choose]) cloud = depth_masked if self.add_noise: cloud = np.add(cloud, add_t) model_points = self.pt[obj] * 10 dellist = [j for j in range(0, len(model_points))] dellist = random.sample(dellist, len(model_points) - self.num_pt_mesh_small) model_points = np.delete(model_points, dellist, axis=0) #want to swap the axes of the target x_90 = np.zeros((3, 3)) x_90[0, 0] = 1 x_90[1, 2] = 1 x_90[2, 1] = 1 x_180 = np.zeros((3, 3)) x_180[0, 0] = 1 x_180[1, 1] = -1 x_180[2, 2] = -1 y_180 = np.zeros((3, 3)) y_180[0, 0] = -1 y_180[1, 1] = 1 y_180[2, 2] = -1 target = np.copy(model_points) # target_mean = np.mean(target, axis=0) # target -= target_mean # target = np.dot(target, swap_left.T) # target += target_mean target = np.dot(target, (target_r @ y_180).T) if self.add_noise: target = np.add(target, target_t + add_t * 10000) else: target = np.add(target, target_t) #AT THE VERY END, CONVERT EVERYTHING TO METERS return torch.from_numpy(cloud.astype(np.float32) / 10000.), \ torch.LongTensor(choose.astype(np.int32)), \ self.norm(torch.from_numpy(img_masked.astype(np.float32))), \ torch.from_numpy(target.astype(np.float32) / 10000.), \ torch.from_numpy(model_points.astype(np.float32) / 10000.), \ torch.LongTensor([self.objlist.index(obj)])