def coord_regression(predictions_b, predictions, train, test, method): mean_error = [] if (method == 1): machine_learn = KNeighborsRegressor(n_neighbors=5, weights='distance') elif (method == 2): #machine_learn = MLPClassifier(solver='sgd',learning_rate = 'adaptive',verbose='true',activation='tanh',alpha=1e-5) machine_learn = MLPClassifier(solver='sgd', learning_rate='adaptive', verbose='false', activation='tanh', alpha=1e-5, max_iter=400) #THE BEST #machine_learn = MLPClassifier(hidden_layer_sizes=(100,5), solver='sgd',learning_rate = 'adaptive',verbose='true',activation='tanh',alpha=1e-5,max_iter=500) #model = MLPClassifier(learning_rate = 'adaptive') #solvers = ['lbfgs', 'sgd', 'adam'] #activations = ['identity', 'logistic', 'tanh', 'relu'] #max_its = [200,400,600] #machine_learn = GridSearchCV(estimator=model, param_grid=dict(activation =activations,max_iter=max_its),n_jobs=7) #GRID #for each building for j in range(3): new_train1 = train.loc[ train['BUILDINGID'] == j] #select for training only buildings with that label (0,1, or 2) ind = [x for x in range(len(predictions_b)) if predictions_b[x] == j ] #get the position of the samples that have building == i new_test1 = test.iloc[ind, :] if (ind): #for each floor for i in range(5): new_train2 = new_train1.loc[new_train1['FLOOR'] == i] if (not new_train2.empty): indexes = [ x for x in range(len(predictions)) if (predictions[x] == i and predictions_b[x] == j) ] #get the position of the samples that have building == i else: index = [] if (indexes): #if list is not empty X_train = new_train2.ix[:, 0:519] Y_train = new_train2[['LONGITUDE', 'LATITUDE']] machine_learn.fit(X_train, Y_train) #testing samples with prediction building == i new_test2 = test.iloc[indexes, :] X_test = new_test2.ix[:, 0:519] Y_test = new_test2[['LONGITUDE', 'LATITUDE']] #Turn into list predicts_lon_lat = machine_learn.predict(X_test).tolist() Y_test = Y_test.values.tolist() distance = [] for j in range(len(predicts_lon_lat)): #change the latitude and longitude unit myProj = Proj( "+proj=utm +zone=23K, +south +ellps=WGS84 +datum=WGS84 +units=m +no_defs" ) lon_pred, lat_pred = myProj(predicts_lon_lat[j][0], predicts_lon_lat[j][1], inverse=True) lon_Y, lat_Y = myProj(Y_test[j][0], Y_test[j][1], inverse=True) #join in a unique list Y = [] Y.append(lon_Y) Y.append(lat_Y) predict = [] predict.append(lon_pred) predict.append(lat_pred) #The distance between the two latitudes is the error distance.append(vincenty(Y, predict).meters) print "distance" print distance #If you want to use haversine distance, uncomment the line below #print haversine(lon_Y, lat_Y, lon_pred, lat_pred) mean_error.append(np.mean(distance)) #print(np.mean(distance)) return np.mean(mean_error)
def utm2deg(x, y, utmzone=43): p = Proj(proj='utm', zone=43, ellps='WGS84') Lon, Lat = p(x, y, inverse=True) return Lon, Lat
else: sys.exit() # output the plots if output == cfg.PLAN_VIEW: # Set the map limit around Mossmorran extent = [-3.3733, -3.244456, 56.060534, 56.132276] fig, ax = plt.subplots() # x and y are curently in units of metres from the central point (0,0) # convert them to eastings and northings xe = cfg.UTM_easting + x yn = cfg.UTM_northing + y # now convert to lat-lon p2 = Proj( "+proj=utm +zone=30V, +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs" ) xp, yp = p2(xe, yn, inverse=True) xmin = np.max(xp) xmax = np.min(xp) ymax = np.max(yp) ymin = np.min(yp) print(xmin, xmax, ymin, ymax) data = np.mean(C1, axis=2) * 1e6 maxc = np.max(data) minc = np.min(data) print(minc, maxc) plt.contourf(xp, yp,
def wgs84_to_tm128(longitude, latitude): return transform( Proj(**WGS84), Proj(**TM128), longitude, latitude )
def wgs84_to_tm127(longitude, latitude): return map(lambda x:2.5*x, transform( Proj(**WGS84), Proj(**TM127), longitude, latitude ))
def get_lon_lat(): wgs84 = Proj(init='epsg:4326') mercator = Proj(init='epsg:3857') lon, lat = transform(mercator, wgs84, center_x, center_y) return lon, lat
Facies = Facies.fillna(0) Facies['Lava'] = Facies[0] Facies['Pflow'] = Facies[1] Facies['Pfall'] = Facies[2] # ============================================================================= # Facies['Lava'] = Facies[0]+Facies[1] # Facies['Pflow'] = Facies[3]+Facies[4]+Facies[9] # Facies['Pfall'] = Facies[7]+Facies[8] # Facies['Sed'] = Facies[2]+Facies[5]+Facies[6] # ============================================================================= #Facies.to_csv('FaciesRecap.csv', sep=',') thick = pd.read_csv("TD.csv").set_index('WELL').sort_index() xy = pd.read_csv("allcoordinate.csv").set_index('WELL').sort_index() projection = Proj("+proj=utm +zone=48S, +south +ellps=WGS84 +datum=WGS84 +units=m +no_defs") xy['X'], xy['Y'] = projection(xy['X'].values, xy['Y'].values, inverse=True) prod = pd.read_csv("Production.csv").set_index('WELL').fillna(0) dataset = Facies.iloc[:,0:3] dataset = dataset.merge(thick, left_index=True, right_index=True, how='inner') dataset = dataset.merge(xy, left_index=True, right_index=True, how='inner') dataset = dataset.merge(prod, left_index=True, right_index=True, how='inner') x = dataset.iloc[:,4] y = dataset.iloc[:,5] oil = dataset.iloc[:,7] gas = dataset.iloc[:,8] thick = dataset.iloc[:,3] wells = dataset.index #Lava
def parFunc(c): # set path dataPath = "/media/Data/Data/PIG/" templatePath = "/home/setholinger/Documents/Projects/PIG/detections/templateMatch/multiTemplate/run3/" outPath = "/home/setholinger/Documents/Projects/PIG/location/polarization/" # set data paramters nproc = 15 _3D = 1 norm_component = 0 type = "short" if _3D: type = type + "_3D" fs = 100 numCluster = 14 if _3D: outPath = outPath + "3D_clustering/" if norm_component: outPath = outPath + "normalized_components/" templatePath = templatePath + type.split( "_")[0] + "_normalized_3D_clustering/" else: templatePath = templatePath + type + "_clustering/" else: templatePath = templatePath + type + "_clustering/" # set threshold parameters # xcorr_percent_thresh = 0.1 will compute polarizations for the 10% best correlated events norm_thresh = 2.75 MAD = 0 xcorr_percent_thresh = 0.1 # set windowing parameters snipLen = 500 winLen = 10 slide = 5 numSteps = int((snipLen - winLen) / slide) # set stations and components chans = ["HHN", "HHE", "HHZ"] stations = ["PIG2", "PIG4", "PIG5"] #stat_coords = np.array([[-100.748596,-75.016701],[-100.786598,-75.010696],[-100.730904,-75.009201],[-100.723701,-75.020302],[-100.802696,-75.020103]]) stat_coords = np.array([[-100.786598, -75.010696], [-100.723701, -75.020302], [-100.802696, -75.020103]]) plotStat = 'PIG2' # read imagery data, get coordinate system, convert station coordinates to x and y, and take average station location file = "/media/Data/Data/PIG/TIF/LC08_L1GT_001113_20131012_20170429_01_T2_B4.TIF" sat_data = rasterio.open(file) p2 = Proj(sat_data.crs, preserve_units=False) p1 = Proj(proj='latlong', preserve_units=False) [stat_x, stat_y] = transform(p1, p2, stat_coords[:, 0], stat_coords[:, 1]) avg_stat_x = np.mean(stat_x) avg_stat_y = np.mean(stat_y) # set frequency prefiltFreq = [0.05, 1] freq = [0.05, 1] # load waveforms #waves = obspy.read(templatePath + type + '_waveforms_' + str(prefiltFreq[0]) + "-" + str(prefiltFreq[1]) + 'Hz.h5') # load detection times detFile = h5py.File(templatePath + "detection_times.h5", "r") detTimes = list(detFile["times"]) detFile.close() # load clustering results clustFile = h5py.File( templatePath + str(numCluster) + "/" + str(numCluster) + "_cluster_predictions_" + str(prefiltFreq[0]) + "-" + str(prefiltFreq[1]) + "Hz.h5", "r") pred = np.array(list(clustFile["cluster_index"])) centroids = list(clustFile["centroids"]) clustFile.close() # read in correlation results for the current cluster corrFile = h5py.File( templatePath + str(numCluster) + "/centroid" + str(c) + "_correlations_" + str(prefiltFreq[0]) + "-" + str(prefiltFreq[1]) + "Hz.h5", "r") cluster_xcorr_coef = np.array(list(corrFile["corrCoefs"])) corrFile.close() # get indices of waveforms in current cluster and events in cluster above xcorr threshold clusterInd = [i for i, x in enumerate(pred == c) if x] if MAD: mad = stats.median_absolute_deviation(abs(cluster_xcorr_coef)) mad = round(mad, 2) xcorr_percent_thresh = mad / max(abs(cluster_xcorr_coef)) print(mad) print(xcorr_percent_thresh) n_events = round(xcorr_percent_thresh * len(clusterInd)) else: n_events = round(xcorr_percent_thresh * len(clusterInd)) threshInd = abs(cluster_xcorr_coef).argsort()[-1 * n_events:][::-1] # make array for storing pca vector sums and storing data to plot all_first_components = np.zeros((numSteps * len(threshInd), 2), "float64") clusterEventsAligned = np.zeros((len(threshInd), snipLen * fs), 'float64') # loop through indices of events in current cluster for i in range(len(threshInd)): # make array for storage of pca components and empty obspy stream for storing one trace from each station first_component_sums = np.zeros((numSteps, 2), "float64") event_stat = obspy.read() event_stat.clear() # loop through stations to get one trace from each to find earliest arrival try: for stat in range(len(stations)): # get times bounds for current event and read event #eventLims = [waves[clusterInd[threshInd[i]]].stations.starttime,waves[clusterInd[threshInd[i]]].stations.starttime + snipLen] starttime = UTCDateTime(detTimes[clusterInd[threshInd[i]]]) endtime = starttime + snipLen eventLims = [starttime, endtime] event_stat += readEvent(dataPath + "MSEED/noIR/", stations[stat], chans[1], eventLims, freq) # find station with earliest arrival first_stat = observed_first_arrival(event_stat) # loop though stations to perform PCA on all windows in the event on each station's data for stat in range(len(stations)): # compute pca components for all windows in the event first_components = compute_pca(dataPath, stations[stat], chans, fs, winLen, slide, numSteps, freq, eventLims) # correct polarization direction based on first arrival first_components_corrected = correct_polarization( first_components, stations, first_stat, avg_stat_x, avg_stat_y, stat_x, stat_y) # sum results (this is vector sum across stations of pca first components for each window) first_component_sums = first_component_sums + first_components_corrected # give user output every % complete if round(i / len(threshInd) * 100) > round( (i - 1) / len(threshInd) * 100): print( str(round(i / len(threshInd) * 100)) + " % complete (cluster " + str(c) + ")") # fill results vector all_first_components[i * numSteps:(i + 1) * numSteps, :] = first_component_sums except: # give output if no data on current station print("Skipping cluster " + str(c) + " event " + str(i) + " (missing data on " + stations[stat] + ")") # make plots fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8), gridspec_kw={'height_ratios': [1, 0.4]}) # do some stuff corners = np.array([ [sat_data.bounds[0], sat_data.bounds[1]], # bottom left [sat_data.bounds[0], sat_data.bounds[3]], # top left [sat_data.bounds[2], sat_data.bounds[1]], # bottom right [sat_data.bounds[2], sat_data.bounds[3]] ]) # top right corners_lon, corners_lat = transform(p2, p1, corners[:, 0], corners[:, 1]) # plot imagery show(sat_data, ax=ax[0], cmap="gray") # handle axes percent = xcorr_percent_thresh * 100 plt.suptitle("Cluster " + str(c) + " Polarizations (top " + str(percent) + "% of events)") # make array for storage back_azimuths = np.empty((0, 1), "float64") # plot pca compoments that exceed norm threshold # be wary- the transformed coordinate system's x-axis is meters north and the y-axis is meters east, so the pca_first_component[~,0] (which is cartesian x) is in [L] east # and therefore along the transformed y-axis and the pca_first_component[~,1] (which is cartesian y) is in [L] north and therefore along the transformed x-axis count = 0 for s in range(len(all_first_components)): # only plot and save results if length of resultant vector has a norm exceeding the threshold if np.linalg.norm(all_first_components[s, :]) > norm_thresh: # calculate back azimuths and save in array baz = compute_baz(all_first_components[s, :]) back_azimuths = np.vstack((back_azimuths, baz)) count += 1 # compute histogram of back azimuths baz_hist, edges = np.histogram(back_azimuths, bins=np.linspace(0, 360, 37)) # set up colormap colors = [cm.plasma(x) for x in np.linspace(0, 1, max(baz_hist) + 1)] # plot all rays in 10-degree bins with length proportional to # of windows in that bin rays = np.zeros((36, 2), 'float64') scale = 40000 max_width = 2 * np.pi * scale / 36 max_width = 8 for i in range(36): angle = i * 10 rays[i, :] = compute_rays(angle) rayLength = baz_hist[i] / max(baz_hist) * scale [x, y] = [ np.linspace(avg_stat_x, avg_stat_x + rays[i, 0] * rayLength, 100), np.linspace(avg_stat_y, avg_stat_y + rays[i, 1] * rayLength, 100) ] lwidths = np.linspace(0, max_width, 100) * rayLength / scale points = np.array([x, y]).T.reshape(-1, 1, 2) segments = np.concatenate([points[:-1], points[1:]], axis=1) lc = LineCollection(segments, linewidths=lwidths, color='maroon', alpha=0.5, zorder=len(all_first_components) * 10) ax[0].add_collection(lc) # define, transform, and plot lat/lon grid lat = [-74, -75] lon = [-98, -100, -102, -104] x_lab_pos = [] y_lab_pos = [] line = np.linspace(corners_lat[0] + 1, corners_lat[2] - 1, 100) for l in lon: line_x, line_y = transform(p1, p2, np.linspace(l, l, 100), line) ax[0].plot(line_x, line_y, linestyle='--', linewidth=1, c='gray', alpha=1) y_lab_pos.append(line_y[np.argmin(np.abs(line_x - corners[0, 0]))]) line = np.linspace(corners_lon[0] - 2, corners_lon[1] + 1, 100) for l in lat: line_x, line_y = transform(p1, p2, line, np.linspace(l, l, 100)) ax[0].plot(line_x, line_y, linestyle='--', linewidth=1, c='gray', alpha=1) x_lab_pos.append(line_x[np.argmin(np.abs(line_y - corners[0, 1]))]) ax[0].set_xlim([corners[0, 0], corners[2, 0]]) ax[0].set_ylim([corners[0, 1], corners[1, 1]]) ax[0].set_xticks(x_lab_pos) ax[0].set_xticklabels( labels=[str(lat[0]) + '$^\circ$', str(lat[1]) + '$^\circ$']) ax[0].set_xlabel("Latitude") ax[0].set_yticks(y_lab_pos) ax[0].set_yticklabels(labels=[ str(lon[0]) + '$^\circ$', str(lon[1]) + '$^\circ$', str(lon[2]) + '$^\circ$', str(lon[3]) + '$^\circ$' ]) ax[0].set_ylabel("Longitude") # colors k1 = plt.rcParams['axes.prop_cycle'].by_key()['color'][0] k2 = plt.rcParams['axes.prop_cycle'].by_key()['color'][1] # plot ice front front_x = [ -1.644e6, -1.64e6, -1.638e6, -1.626e6, -1.611e6, -1.6095e6, -1.6055e6, -1.6038e6, -1.598e6, -1.6005e6, -1.6e6, -1.595e6 ] front_y = [ -3.34e5, -3.33e5, -3.44e5, -3.445e5, -3.475e5, -3.43e5, -3.4e5, -3.413e5, -3.356e5, -3.32e5, -3.289e5, -3.29e5 ] ax[0].plot(front_x, front_y, c=k1, zorder=len(all_first_components) * 10) # plot rift rift1_x = [-1.63e6, -1.6233e6, -1.6132e6, -1.6027e6] rift1_y = [-3.255e5, -3.237e5, -3.236e5, -3.281e5] ax[0].plot(rift1_x, rift1_y, c=k2, zorder=len(all_first_components) * 10) rift2_x = [-1.63e6, -1.6232e6, -1.6132e6] rift2_y = [-3.28e5, -3.2706e5, -3.236e5] ax[0].plot(rift2_x, rift2_y, c=k2, zorder=len(all_first_components) * 10) # plot station locations ax[0].scatter(stat_x, stat_y, marker="^", c='black', zorder=count * 10) # add north arrow ax[0].arrow(avg_stat_x - 65000, avg_stat_y + 70000, -10000, 0, width=500, head_width=3000, head_length=3000, fc="k", ec="k", zorder=len(all_first_components) * 10) ax[0].text(avg_stat_x - 74000, avg_stat_y + 73000, "N", size="large", zorder=len(all_first_components) * 10) # add distance scale ax[0].plot(np.linspace(avg_stat_x - 60000, avg_stat_x - 80000, 10), np.ones(10) * avg_stat_y - 30000, c="k") ax[0].text(avg_stat_x - 82000, avg_stat_y - 26000, "20 km", size="medium") # plot events and centroid waveFile = h5py.File( templatePath + str(numCluster) + "/aligned_cluster" + str(c) + "_waveform_matrix_" + str(prefiltFreq[0]) + "-" + str(prefiltFreq[1]) + "Hz.h5", "r") alignedWaves = np.array(list(waveFile["waveforms"])) waveFile.close() alignedWave_fs = 2 t = np.linspace(0, snipLen, snipLen * alignedWave_fs + 1) if _3D: t = np.linspace(0, snipLen * 3, (snipLen * alignedWave_fs + 1) * 3) maxAmps = np.zeros((len(threshInd), 1), 'float64') for w in range(len(threshInd)): ax[1].plot(t, alignedWaves[threshInd[w]], 'k', alpha=0.1) maxAmps[w] = np.max(np.abs(alignedWaves[threshInd[w]])) ax[1].plot( t, centroids[c].ravel() / np.max(abs(centroids[c].ravel())) * np.median(maxAmps)) ax[1].set_ylim([-4 * np.median(maxAmps), 4 * np.median(maxAmps)]) ax[1].title.set_text('Centroid and Events Used in Polarization Analysis') ax[1].ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) ax[1].set_xlabel("Time (seconds)") ax[1].set_ylabel("Velocity (m/s)") ax[1].set_xlim([t[0], t[-1]]) if _3D: clusterChans = ["HHZ", "HHN", "HHE"] ax[1].set_xticks([ 0, snipLen / 2, snipLen, snipLen * 3 / 2, snipLen * 2, snipLen * 5 / 2, snipLen * 3 ]) xPos = [snipLen, snipLen * 2] for xc in xPos: ax[1].axvline(x=xc, color='k', linestyle='--') ax[1].set_xticklabels([ '0', '250\n' + clusterChans[0], '500 0 ', '250\n' + clusterChans[1], '500 0 ', '250\n' + clusterChans[2], '500' ]) ax[1].grid(linestyle=":") ax[1].grid() plt.tight_layout() #plt.show() plt.savefig(outPath + "win_len_" + str(winLen) + "/norm>" + str(norm_thresh) + "/top_" + str(percent) + "%/cluster_" + str(c) + "_polarizations.png") plt.close() # save actual backazimuth data outFile = h5py.File( outPath + "win_len_" + str(winLen) + "/norm>" + str(norm_thresh) + "/top_" + str(percent) + "%/cluster_" + str(c) + "_backazimuths.h5", "w") outFile.create_dataset("backazimuths", data=back_azimuths) outFile.close()
def get_proj_from_srid(srid): return Proj(init='EPSG:{}'.format(srid))
def main(): global inputArgs, grib, dir_path #Make our global vars: grib is the object that will hold our Grib Class. dir_path = os.path.dirname(os.path.realpath(__file__)) comparison_days = [0, -7] inputArgs = handle_args( sys.argv) #All input arguments if run on the command line. for deltaDay in comparison_days: # MAKE SURE TO UNCOMMENT #inputArgs.date2 when putting back into production if deltaDay == 0: date2 = None else: date2 = ((datetime.datetime.now(pytz.timezone('US/Pacific'))) + datetime.timedelta(days=deltaDay)).strftime("%Y%m%d") ############## # Debugging inputArgs.date = (datetime.datetime.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") #inputArgs.date = time.strftime("%Y%m%d") #inputArgs.date = str(dayNum) inputArgs.date2 = date2 #Comment this out for just one date inputArgs.map = True # Make the map and save png to folder. inputArgs.plot = False findValueAtPoint = False # Find all the values at specific lat/lng points within an excel file. ################# grib = Grib() #Assign variable to the Grib Class. grib.model = inputArgs.model #Our model will always be "snodas" for this program grib.displayunits = inputArgs.displayunits grib.basin = inputArgs.basin # Basin can be "French_Meadows", "Hell_Hole", or "MFP", this gets shapefile # Bounding box will clip the raster to focus in on a region of interest (e.g. CA) This makes the raster MUCH smaller # and easier to work with. See gdal.Open -> gdal.Translate below for where this is acutally used. grib.bbox = [ -125.0, 50.0, -115.0, 30.0 ] #[upper left lon, upper left lat, lower left lon, lower left lat] grib = get_snowdas( grib, inputArgs.date ) #Get the snodas file and save data into the object variable grib #pngFile = makePNG() #Any reprojections of grib.gribAll have already been done in get_snowdas. #The original projection of snodas is EPSG:4326 (lat/lng), so it has been changed to EPSG:3875 (x/y) in get_snodas projInfo = grib.gribAll.GetProjection() geoinformation = grib.gribAll.GetGeoTransform( ) #Get the geoinformation from the grib file. xres = geoinformation[1] yres = geoinformation[5] xmin = geoinformation[0] xmax = geoinformation[0] + (xres * grib.gribAll.RasterXSize) ymin = geoinformation[3] + (yres * grib.gribAll.RasterYSize) ymax = geoinformation[3] spatialRef = osr.SpatialReference() spatialRef.ImportFromWkt(projInfo) spatialRefProj = spatialRef.ExportToProj4() # create a grid of xy (or lat/lng) coordinates in the original projection xy_source = np.mgrid[xmin:xmax:xres, ymax:ymin:yres] xx, yy = xy_source # A numpy grid of all the x/y into lat/lng # This will convert your projection to lat/lng (it's this simple). lons, lats = Proj(spatialRefProj)(xx, yy, inverse=True) # Find the center point of each grid box. # This says move over 1/2 a grid box in the x direction and move down (since yres is -) in the # y direction. Also, the +yres (remember, yres is -) is saying the starting point of this array will # trim off the y direction by one row (since it's shifted off the grid) xy_source_centerPt = np.mgrid[xmin + (xres / 2):xmax:xres, ymax + (yres / 2):ymin:yres] xxC, yyC = xy_source_centerPt lons_centerPt, lats_centerPt = Proj(spatialRefProj)(xxC, yyC, inverse=True) if grib.basin != "Hell_Hole": mask = createMask(xxC, yyC, spatialRefProj) grib.basinTotal, grib.basinSWE = calculateBasin( mask, grib, xres, yres) # The shape file for the Hell Hole basin includes the SMUD domain. Therefore, if we want to extract the SMUD # domain, then we will create another mask on top of the Hell Hole mask. This means that any grid point # outside of both domains will still = 0. The SMUD domain will contain it's mask AND the Hell Hole mask. if grib.basin == 'Hell_Hole': grib.basin = 'Hell_Hole_SMUD' #This is just to get the correct directory structure submask = createMask( xxC, yyC, spatialRefProj ) # All areas in array = to 1 will be in SMUD's basin smud_BasinArea = grib.basinArea # Used to remove SMUD's basin area (in m^2) from Hell_Hole's basin. # Get SMUD's information for the SMUD submask grib.SMUDbasinTotal, grib.SMUDbasinSWE = calculateBasin( submask, grib, xres, yres) grib.basin = 'Hell_Hole' # reset back mask = createMask( xxC, yyC, spatialRefProj ) #This will be the entire Hell Hole basin, which includes SMUD hhMask = mask + submask # Hell hole basin is now anywhere where hhMask = 1 and SMUD is anywhere it = 2 hhMask[hhMask != 1] = 0 # Set anything outside of Hell Hole's mask = 0 grib.basinArea = grib.basinArea - smud_BasinArea # grib.basinArea currently includes smuds basin, so remove it. # Get HellHoles's information from the hell hole submask grib.basinTotal, grib.basinSWE = calculateBasin( hhMask, grib, xres, yres) print("Current Basin Total: " + str(grib.basinTotal) + " SMUD Total: " + str(grib.SMUDbasinTotal)) #grib.basinTotal = grib.basinTotal - (0.92 * smudBasinTotal[0]) mask = hhMask # Need to do this so we can use the correct mask in compareDates and in calculateByElevation # Calculate the difference between two rasters if inputArgs.date2 != None: grib.basinTotal, grib.basinSWE = compareDates( mask, grib, xres, yres) #Need to do this after Hell_Hole's data has been manipulated (to account for SMUD) elevation_bins = calculateByElevation(mask, grib, xres, yres) #Send data for writing to Excel File if deltaDay == 0: excel_output(elevation_bins) if inputArgs.plot == True: makePlot(elevation_bins, deltaDay) print(elevation_bins) print(inputArgs.date, " Basin Total: ", grib.basinTotal) #findValue will return a dataframe with SWE values at various lat/lng points. df_ptVal = None if findValueAtPoint == True: df_ptVal = findPointValue(spatialRefProj, xy_source) if inputArgs.map == True: fig = plt.figure() ax = fig.add_subplot(111) m = Basemap(llcrnrlon=-122.8, llcrnrlat=37.3, urcrnrlon=-119.0, urcrnrlat=40.3, ax=ax) #m.arcgisimage(service='World_Imagery', xpixels=2000, verbose=True) im = Image.open( urlopen( "http://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/export?bbox=-122.8,37.3,-119.0,40.3&bboxSR=4326&imageSR=4326&size=2000,1578&dpi=96&format=png32&transparent=true&f=image" )) m.imshow(im, origin='upper') #For inset # loc =>'upper right': 1, # 'upper left': 2, # 'lower left': 3, # 'lower right': 4, # 'right': 5, # 'center left': 6, # 'center right': 7, # 'lower center': 8, # 'upper center': 9, # 'center': 10 axin = inset_axes(m.ax, width="40%", height="40%", loc=8) m2 = Basemap(llcrnrlon=-120.7, llcrnrlat=38.7, urcrnrlon=-120.1, urcrnrlat=39.3, ax=axin) im2 = Image.open( urlopen( "http://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/export?bbox=-120.7,38.7,-120.09999999999998,39.3&bboxSR=4326&imageSR=4326&size=2000,1999&dpi=96&format=png32&transparent=true&f=image" )) m2.imshow(im2, origin='upper') #m2.arcgisimage(service='World_Imagery', xpixels=2000, verbose=True) mark_inset(ax, axin, loc1=2, loc2=4, fc="none", ec="0.5") ###################################DEBUGGING AREA############################################################### # Debugging: Test to prove a given lat/lng pair is accessing the correct grid box: #*********TEST 1: Test for center points #grib.data[0,0] = 15 #increase the variable by some arbitrary amount so it stands out. #xpts, ypts = m(lons_centerPt[0,0],lats_centerPt[0,0]) #This should be in the dead center of grid[0,0] #m.plot(xpts,ypts, 'ro') #*********TEST 2: Test for first grid box # Test to see a if the point at [x,y] is in the upper right corner of the cell (it better be!) #xpts, ypts = m(lons[0, 0], lats[0, 0]) # should be in upper right corner of cell #m.plot(xpts, ypts, 'bo') # *********TEST 3: Test for first grid box # Test to see the location of center points of each grid in polygon # To make this work, uncomment the variables in def create_mask #debug_Xpoly_center_pts, debug_Ypoly_center_pts = m(debugCenterX, debugCenterY) #m.plot(debug_Xpoly_center_pts, debug_Ypoly_center_pts, 'bo') # *********TEST 4: Test grid box size (In lat lng coords) # This is for use in a Basemap projection with lat/lon (e.g. EPSG:4326) #testX = np.array([[-120.1, -120.1], [-120.10833, -120.10833]]) #testY = np.array([[39.0, 39.00833], [39.0, 39.00833]]) # testVal = np.array([[4,4],[4,4]]) # For use in basemap projection with x/y (e.g. espg:3857. In m=basemap just include the argument projection='merc') # testX = np.array([[500975, 500975], [(500975 + 1172), (500975 + 1172)]]) # testY = np.array([[502363, (502363 + 1172)], [502363, (502363 + 1172)]]) #testVal = np.array([[18, 18], [18, 18]]) #im1 = m.pcolormesh(testX, testY, testVal, cmap=plt.cm.jet, vmin=0.1, vmax=10, latlon=False, alpha=0.5) # Test to see all points # xtest, ytest = m(lons,lats) # m.plot(xtest,ytest, 'bo') ################################################################################################################ hr = 0 makeMap(lons, lats, hr, m, m2, df_ptVal, deltaDay) return
def test_get_timeseries(self): """ Simple regression test of WRF data repository. """ EPSG, bbox, bpoly = self.wrf_epsg_bbox # Period start n_hours = 60 t0 = api.YMDhms(1999, 10) date_str = "{}-{:02}".format(t0.year, t0.month) utc = api.Calendar() # No offset gives Utc period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours)) base_dir = path.join(shyftdata_dir, "repository", "wrf_data_repository") f1 = "wrfout_d03_{}".format(date_str) wrf1 = WRFDataRepository(EPSG, base_dir, filename=f1, allow_subset=True) wrf1_data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity", "radiation") sources = wrf1.get_timeseries(wrf1_data_names, period, geo_location_criteria=bpoly) self.assertTrue(len(sources) > 0) self.assertTrue(set(sources) == set(wrf1_data_names)) self.assertTrue(sources["temperature"][0].ts.size() == n_hours + 1) r0 = sources["radiation"][0].ts p0 = sources["precipitation"][0].ts temp0 = sources["temperature"][0].ts self.assertTrue(r0.size() == n_hours + 1) self.assertTrue(p0.size() == n_hours + 1) self.assertTrue(r0.time(0) == temp0.time(0)) self.assertTrue(p0.time(0) == temp0.time(0)) self.assertTrue(r0.time(r0.size() - 1) == temp0.time(temp0.size() - 1)) self.assertTrue(p0.time(r0.size() - 1) == temp0.time(temp0.size() - 1)) self.assertTrue(p0.time(0), period.start) # Number test: # asserting shyft-sources time series are same as time series of corresponding location in wrf dataset. dset = Dataset(path.join(base_dir, f1)) lat = dset.variables["XLAT"] lon = dset.variables["XLONG"] wrf_data = {} wrf_data["temperature"] = dset.variables["T2"][:] wrf_data["precipitation"] = dset.variables["PREC_ACC_NC"][:] wrf_data["radiation"] = dset.variables["SWDOWN"][:] pressure = dset.variables["PSFC"][:] mixing_ratio = dset.variables["Q2"][:] wrf_data["relative_humidity"] = wrf1._calculate_rel_hum( wrf_data["temperature"], pressure, mixing_ratio) wrf_data["temperature"] -= 273.16 data_cs = "latlong" target_cs = "+init=EPSG:32643" data_proj = Proj(proj=data_cs) target_proj = Proj(target_cs) x, y = transform(data_proj, target_proj, lon[0, :, :], lat[0, :, :]) for name, wrf_d in wrf_data.items(): srs = sources[name] for i, s in enumerate(srs): mp = s.mid_point() x_ts, y_ts, z_ts = mp.x, mp.y, mp.z ts = s.ts ts_values = ts.v.to_numpy() # find indixes in wrf-dataset m = (x == x_ts) & (y == y_ts) idxs = np.where(m > 0) x_idx, y_idx = idxs[0][0], idxs[1][ 0] # assumung geo-location is unique in dataset self.assertTrue( np.allclose(ts_values, wrf_d[:n_hours + 1, x_idx, y_idx], rtol=1e-4, atol=1e-4), f"wrf and shyft-TS of {name} are not the same.")
def get_region_model(self, region_id, catchments=None): """ Return a fully specified shyft api region_model for region_id, based on data found in netcdf dataset. Parameters ----------- region_id: string unique identifier of region in data catchments: list of unique integers catchment indices when extracting a region consisting of a subset of the catchments has attribs to construct params and cells etc. Returns ------- region_model: shyft.api type """ with Dataset(self._data_file) as dset: grp = dset.groups["elevation"] xcoord = grp.variables["xcoord"][:] ycoord = grp.variables["ycoord"][:] dataset_epsg = None if hasattr(grp, "epsg"): dataset_epsg = grp.epsg if hasattr(grp, "EPSG"): dataset_epsg = grp.EPSG if not dataset_epsg: raise interfaces.InterfaceError( "netcdf: epsg attr not found in group elevation") if dataset_epsg != self._epsg: source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(self._epsg) - 32600, "WGS84", "WGS84") target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(dataset_epsg) - 32600, "WGS84", "WGS84") source_proj = Proj(source_cs) target_proj = Proj(target_cs) mesh2d = np.dstack( transform(source_proj, target_proj, *np.meshgrid(xcoord, ycoord))).reshape(-1, 2) dx = xcoord[1] - xcoord[0] dy = ycoord[1] - ycoord[0] x_corners = np.empty(len(xcoord) + 1, dtype=xcoord.dtype) y_corners = np.empty(len(ycoord) + 1, dtype=ycoord.dtype) x_corners[1:] = xcoord + dx / 2.0 x_corners[0] = xcoord[0] - dx / 2.0 y_corners[1:] = ycoord + dy / 2.0 y_corners[0] = ycoord[0] - dy / 2.0 xc, yc = transform(source_proj, target_proj, *np.meshgrid(x_corners, y_corners)) areas = np.empty((len(xcoord), len(ycoord)), dtype=xcoord.dtype) for i in range(len(xcoord)): for j in range(len(ycoord)): pts = [(xc[j, i], yc[j, i]), (xc[j, i + 1], yc[j, i + 1]), (xc[j + 1, i + 1], yc[j + 1, i + 1]), (xc[j + 1, i], yc[j + 1, i])] areas[i, j] = Polygon(pts).area areas = areas.flatten()[self.mask] else: mesh2d = np.dstack(np.meshgrid(xcoord, ycoord)).reshape(-1, 2) areas = np.ones(len(xcoord) * len(ycoord), dtype=xcoord.dtype)[ self.mask] * (xcoord[1] - xcoord[0]) * (ycoord[1] - ycoord[0]) elevation = grp.variables["elevation"][:] coordinates = np.hstack((mesh2d, elevation.reshape(-1, 1)))[self.mask] catchments = dset.groups["catchments"].variables[ "catchments"][:].reshape(-1)[self.mask] c_ids = dset.groups["catchments"].variables["catchment_indices"][:] def frac_extract(name): g = dset.groups # Alias for readability return g[name].variables[name][:].reshape(-1)[self.mask] ff = frac_extract("forest-fraction") lf = frac_extract("lake-fraction") rf = frac_extract("reservoir-fraction") gf = frac_extract("glacier-fraction") # Construct bounding region box_fields = set(("upper_left_x", "upper_left_y", "step_x", "step_y", "nx", "ny", "EPSG")) if box_fields.issubset(self._rconf.domain()): tmp = self._rconf.domain() epsg = tmp["EPSG"] x_min = tmp["upper_left_x"] x_max = x_min + tmp["nx"] * tmp["step_x"] y_max = tmp["upper_left_x"] y_min = y_max - tmp["ny"] * tmp["step_y"] bounding_region = BoundingBoxRegion(np.array([x_min, x_max]), np.array([y_min, y_max]), epsg, self._epsg) else: bounding_region = BoundingBoxRegion(xcoord, ycoord, dataset_epsg, self._epsg) # Construct region parameter: name_map = { "gamma_snow": "gs", "priestley_taylor": "pt", "kirchner": "kirchner", "actual_evapotranspiration": "ae", "skaugen": "skaugen", "hbv_snow": "snow" } region_parameter = self._region_model.parameter_t() for p_type_name, value_ in iteritems(self._mconf.model_parameters()): if p_type_name in name_map: if hasattr(region_parameter, name_map[p_type_name]): sub_param = getattr(region_parameter, name_map[p_type_name]) for p, v in iteritems(value_): setattr(sub_param, p, v) elif p_type_name == "p_corr_scale_factor": region_parameter.p_corr.scale_factor = value_ # TODO: Move into yaml file similar to p_corr_scale_factor radiation_slope_factor = 0.9 # Construct cells cell_vector = self._region_model.cell_t.vector_t() for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, catchments, ff, lf, rf, gf): cell = self._region_model.cell_t() cell.geo = api.GeoCellData( api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor, api.LandTypeFractions(gf, lf, rf, ff, 1.0 - gf - lf - rf - ff)) cell_vector.append(cell) # Construct catchment overrides catchment_parameters = self._region_model.parameter_t.map_t() for k, v in iteritems(self._rconf.parameter_overrides()): if k in c_ids: param = self._region_model.parameter_t(region_parameter) for p_type_name, value_ in iteritems(v): if p_type_name in name_map: sub_param = getattr(param, name_map[p_type_name]) for p, pv in iteritems(value_): setattr(sub_param, p, pv) elif p_type_name == "p_corr_scale_factor": param.p_corr.scale_factor = value_ else: # Avoid unknown params to go unadvertised raise RegionConfigError( "parameter {} is not in the set of allowed ones". format(p_type_name)) catchment_parameters[k] = param region_model = self._region_model(cell_vector, region_parameter, catchment_parameters) def do_clone(x): clone = x.__class__(x) clone.bounding_region = bounding_region return clone region_model.bounding_region = bounding_region region_model.clone = do_clone return region_model
def make_contours2(self, con_var='q', lats=np.arange(50, 86, 2), plot=False): """ Make contours for input into CAS. This method uses projection in to x,y coordinates to avoid problem of contour segments cut off by prime meridian. """ if os.path.isdir(self.working_dir + 'contours'): try: os.system('rm -f ' + self.working_dir + 'contours/*.in') except OSError: pass else: os.system('mkdir ' + self.working_dir + 'contours') # Only use 90 - 20 latitude d = self.ds[con_var].sel(latitude=slice(90, 20))[self.start_time, :] # Find contour levels by interpolation at lon=0 #cons = d.mean(dim = 'longitude').interp(latitude=lats).data cons = d.isel(longitude=10).interp(latitude=lats).data print(cons) pa = Proj("+proj=stere +lat_0=90", preserve_units=True) lonv, latv = np.meshgrid(d.longitude.data, d.latitude.data) x, y = pa(lonv, latv) reg_x = np.linspace(np.min(x), np.max(x), 500) reg_y = np.linspace(np.min(y), np.max(y), 500) xi, yi = np.meshgrid(reg_x, reg_y) d2 = mlab.griddata(x.flatten(), y.flatten(), d.data.flatten(), xi, yi, interp='linear') count = 0 for icon in cons: inner = False if count > 0: if cons[count - 1] > icon: inner = True fig = plt.figure(figsize=(10, 5)) ax1 = fig.add_subplot(1, 2, 1) ax1.contourf(reg_x, reg_y, d2) xycon = ax1.contour(reg_x, reg_y, d2, [icon], colors='k') latloncon = [] for iicon in range(len(xycon.allsegs[0])): ilons, ilats = pa(xycon.allsegs[0][iicon][:, 0], xycon.allsegs[0][iicon][:, 1], inverse=True) ilons = ilons % 360 latloncon.append(np.vstack((ilons, ilats)).T) if len(latloncon) == 1: a = latloncon[0] else: print('more than one contour') lens = np.zeros(len(latloncon)) for iicon in range(len(latloncon)): lens[iicon] = self.calc_con_len(latloncon[iicon]) if inner: # 2nd longest contour a = latloncon[np.where(lens == np.sort(lens)[-2])[0][0]] else: a = latloncon[np.argmax(lens)] # This bit is important! # CA algorithm requires contour starting at meridian mina = np.argmin(a[:, 0]) a = np.append(a[mina:, :], a[:mina, :], axis=0) if plot: lats = d.coords['latitude'].data lons = d.coords['longitude'].data #ax1.plot(a[:,0],a[:,1],color='red') ax2 = fig.add_subplot(1, 2, 2, projection=ccrs.NorthPolarStereo()) theta = np.linspace(0, 2 * np.pi, 100) center, radius = [0.5, 0.5], 0.5 verts = np.vstack([np.sin(theta), np.cos(theta)]).T circle = mpath.Path(verts * radius + center) ax2.set_boundary(circle, transform=ax2.transAxes) ax2.set_extent([-180, 180, 20, 90], ccrs.PlateCarree()) ax2.gridlines() cyclic_data, cyclic_lons = cartopy.util.add_cyclic_point( d.data, coord=lons) ## con1 = ax2.contourf(cyclic_lons, lats, cyclic_data, transform=ccrs.PlateCarree()) con = ax2.contour(cyclic_lons, lats, cyclic_data, [icon], colors='k', transform=ccrs.PlateCarree()) ax2.plot(a[:, 0], a[:, 1], transform=ccrs.Geodetic(), color='red') plt.show() else: plt.close() if inner: filename = self.working_dir + 'contours/%s_%.7f_tstep_%s_inner.in' % ( con_var, icon, self.start_time) else: filename = self.working_dir + 'contours/%s_%.7f_tstep_%s.in' % ( con_var, icon, self.start_time) with open(filename, "w") as csvfile: csvfile.write("Contour Advection with Surgery\n") csvfile.write("%s %.4f contour\n" % (con_var, icon)) csvfile.write("\n") csvfile.write("%s 24 %.7f %.7f 0.10000000 0.0000000\n" % (self.ndays, self.time_step, self.time_step)) csvfile.write("1 %s 0.00000\n" % a.shape[0]) csvfile.write("%s %d %d 1.00000\n" % (a.shape[0], a[0, 0], a[0, 1])) with open(filename, "a") as csvfile: writer = csv.writer(csvfile, delimiter=' ') for irow in range(a.shape[0]): writer.writerow(a[irow, :]) count += 1
def regression_subset(predictions, train, test, method): mean_error = [] if (method == 1): machine_learn = KNeighborsRegressor(n_neighbors=5, weights='distance') elif (method == 2): machine_learn = MLPRegressor(random_state=0) #for each building for i in range(3): new_train = train.loc[ train['BUILDINGID'] == i] #select for training only buildings with that label (0,1, or 2) indexes = [x for x in range(len(predictions)) if predictions[x] == i ] #get the position of the samples that have building == i if (indexes): #if list is not empty #training, samples with building == i X_train = new_train.ix[:, 0:519] Y_train = new_train[['LONGITUDE', 'LATITUDE']] machine_learn.fit(X_train, Y_train) #testing samples with prediction building == i new_test = test.iloc[indexes, :] X_test = new_test.ix[:, 0:519] Y_test = new_test[['LONGITUDE', 'LATITUDE']] #Turn into list predicts_lon_lat = machine_learn.predict(X_test).tolist() Y_test = Y_test.values.tolist() distance = [] for j in range(len(predicts_lon_lat)): #change the latitude and longitude unit myProj = Proj( "+proj=utm +zone=23K, +south +ellps=WGS84 +datum=WGS84 +units=m +no_defs" ) lon_pred, lat_pred = myProj(predicts_lon_lat[j][0], predicts_lon_lat[j][1], inverse=True) lon_Y, lat_Y = myProj(Y_test[j][0], Y_test[j][1], inverse=True) #join in a unique list Y = [] Y.append(lon_Y) Y.append(lat_Y) predict = [] predict.append(lon_pred) predict.append(lat_pred) #The distance between the two latitudes is the error distance.append(vincenty(Y, predict).meters) #If you want to use haversine distance, uncomment the line below #print haversine(lon_Y, lat_Y, lon_pred, lat_pred) mean_error.append(np.mean(distance)) #print(np.mean(distance)) return np.mean(mean_error)
from pyramid.view import view_config import requests, pyproj from pyproj import Proj p1 = Proj(init="EPSG:2169") p2 = Proj(init="EPSG:4326") def __reproject(_p): ll = pyproj.transform(p1, p2, _p[0], _p[1]) _p[0] = round(ll[0], 6) _p[1] = round(ll[1], 6) return _p @view_config(route_name='home', renderer="templates/mytemplate.pt") def home(request): describe_url = "https://opendata.vdl.lu/odaweb/index.jsp?describe=1" catalog = requests.get(describe_url).json() _catalog = [] for item in catalog["data"]: _item = {} _item["name"] = item["i18n"]["fr"]["name"] _item["id"] = item["id"] _catalog.append(_item) return {"_catalog": sorted(_catalog, key=lambda k: k['name'])} @view_config(route_name='reproject', renderer="json") def reproject(request):
def getCoords(lonlat, lon0): inProj = Proj(init='epsg:4326') # this is lon/lat outProj = Proj(proj='tmerc', lon_0=float(lon0)) coords = lonlat.split(" ") x, y = transform(inProj, outProj, float(coords[0]), float(coords[1])) return str(x) + " " + str(y) # this is UTM
def netcdf2pcrInit(self, pcr, forcing): #-define input and ouput projections if getattr(self, forcing + 'InProj') == "rotated": inProj = Proj(init="epsg:4326") else: inProj = Proj(init=getattr(self, forcing + 'InProj')) outProj = Proj(init=getattr(self, forcing + 'OutProj')) #-get the attributes of cloneMap attributeClone = getMapAttributesALL(self.clonefile) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] #-read netcdf file f = nc.Dataset(getattr(self, forcing + 'NC')) filecache[getattr(self, forcing + 'NC')] = f #-get coordinates of upper right and lower left corners of model grid xURClone = xULClone + cellsizeClone * colsClone yURClone = yULClone xLLClone = xULClone yLLClone = yULClone - cellsizeClone * rowsClone yLRClone = yLLClone xLRClone = xURClone #-transform coordinates to netcdf projection coordinates xULCloneInput, yULCloneInput = transform(outProj, inProj, xULClone, yULClone) xURCloneInput, yURCloneInput = transform(outProj, inProj, xURClone, yURClone) xLRCloneInput, yLRCloneInput = transform(outProj, inProj, xLRClone, yLRClone) xLLCloneInput, yLLCloneInput = transform(outProj, inProj, xLLClone, yLLClone) #-determine netcdf cell size and subset coordinates to model domain if getattr(self, forcing + 'InProj') == "rotated": #-get coordinates from netcdf file xrot = f.variables[getattr(self, forcing + 'VarX')][:] yrot = f.variables[getattr(self, forcing + 'VarY')][:] #-get coordinates of north pole npLat = f.variables[getattr(self, forcing + 'VarX')].grid_north_pole_latitude npLon = f.variables[getattr(self, forcing + 'VarX')].grid_north_pole_longitude #-transform x and y coordinates to grid xrot, yrot = np.meshgrid(xrot, yrot) #-transform rotated grid to lat,lon-coordinates xLatLon = xrot * 0 yLatLon = yrot * 0 for idx, row in enumerate(xrot): for idy, val in enumerate(row): x, y = rotated_grid_transform((xrot[idx, idy], yrot[idx, idy]), 2, (npLon, npLat)) xLatLon[idx, idy] = x yLatLon[idx, idy] = y #-transform x,y-coordinates to 2d array xyLatLon = [np.array(xLatLon).flatten(), np.array(yLatLon).flatten()] xyLatLon = list(map(list, zip(*xyLatLon))) #-function to find closest node from list of coordinates def closest_node(node, nodes): closest_index = distance.cdist([node], nodes).argmin() indices = np.where( np.ma.getdata(xLatLon) == nodes[closest_index][0]) indices = np.array(indices).flatten().astype('int32') return indices.tolist() #-get indices of corner points clone map from netcdf indicesUL = closest_node((xULCloneInput, yULCloneInput), xyLatLon) indicesLL = closest_node((xLLCloneInput, yLLCloneInput), xyLatLon) indicesUR = closest_node((xURCloneInput, yURCloneInput), xyLatLon) indicesLR = closest_node((xLRCloneInput, yLRCloneInput), xyLatLon) #-determine indices of the corners of netcdf grid corresponding to model grid (+ buffer) xyUL = min(indicesUL[0], indicesLL[0], indicesUR[0], indicesLR[0]) - 2 xyLL = max(indicesUL[0], indicesLL[0], indicesUR[0], indicesLR[0]) + 2 xyUR = min(indicesUL[1], indicesLL[1], indicesUR[1], indicesLR[1]) - 2 xyLR = max(indicesUL[1], indicesLL[1], indicesUR[1], indicesLR[1]) + 2 #-determine x,y-coordinates corresponding to model grid (+ buffer) from netcdf grid x = xLatLon[xyUL:(xyLL + 1), xyUR:(xyLR + 1)] y = yLatLon[xyUL:(xyLL + 1), xyUR:(xyLR + 1)] else: #-get cell size, number of rows and columns and upper left corner coordinates from netcdf grid cellsizeInput = f.variables[getattr( self, forcing + 'VarY')][1] - f.variables[getattr( self, forcing + 'VarY')][0] cellsizeInput = float(cellsizeInput) #-determine x-coordinates corresponding to model grid (+ buffer) from netcdf grid xIdxSta = np.argmin( abs(f.variables[getattr(self, forcing + 'VarX')][:] - (min(xULCloneInput, xLLCloneInput) - 2 * cellsizeInput))) xIdxEnd = np.argmin( abs(f.variables[getattr(self, forcing + 'VarX')][:] - (max(xURCloneInput, xLRCloneInput) + 2 * cellsizeInput))) x = f.variables[getattr(self, forcing + 'VarX')][xIdxSta:(xIdxEnd + 1)] #-determine y-coordinates corresponding to model grid (+ buffer) from netcdf grid yIdxEnd = np.argmin( abs(f.variables[getattr(self, forcing + 'VarY')][:] - (max(yULCloneInput, yURCloneInput) + 2 * cellsizeInput))) yIdxSta = np.argmin( abs(f.variables[getattr(self, forcing + 'VarY')][:] - (min(yLLCloneInput, yLRCloneInput) - 2 * cellsizeInput))) y = f.variables[getattr(self, forcing + 'VarY')][yIdxSta:(yIdxEnd + 1)] #-transform x and y coordinates to grid x, y = np.meshgrid(x, y) #-project x and y coordinates to model grid projection x, y = transform(inProj, outProj, x, y) #-transform x and y coordinates to arrays x = np.asarray(x).ravel() y = np.asarray(y).ravel() #-determine model grid x and y coordinates and save in grid xi = np.arange(xULClone + cellsizeClone * 0.5, (xULClone + cellsizeClone * 0.5) + colsClone * cellsizeClone, cellsizeClone) yi = np.arange( (yULClone + cellsizeClone * 0.5) - rowsClone * cellsizeClone, yULClone + cellsizeClone * 0.5, cellsizeClone) yi = np.flipud(yi) xi, yi = np.meshgrid(xi, yi) #-determine x,y-coordinates of netcdf file and model domain and indices of netcdf corresponding to model domain setattr(self, forcing + 'x', x) setattr(self, forcing + 'y', y) setattr(self, forcing + 'xi', xi) setattr(self, forcing + 'yi', yi) if getattr(self, forcing + 'InProj') == "rotated": setattr(self, forcing + 'xyUL', xyUL) setattr(self, forcing + 'xyLL', xyLL) setattr(self, forcing + 'xyUR', xyUR) setattr(self, forcing + 'xyLR', xyLR) else: setattr(self, forcing + 'xIdxSta', xIdxSta) setattr(self, forcing + 'xIdxEnd', xIdxEnd) setattr(self, forcing + 'yIdxSta', yIdxSta) setattr(self, forcing + 'yIdxEnd', yIdxEnd)
def coord_converter(x1, y1, inCoord='esri:102672', outCoord='epsg:4326'): inProj = Proj(init=inCoord, preserve_units=True) outProj = Proj(init=outCoord) lon, lat = transform(inProj, outProj, x1, y1) return (lat, lon)
def create_new_file(self): """ creates a new netcdf file, and initializes the file with positional and time variables plus the variable as described by self.ts_meta_info :return: None """ with Dataset(self.file_path, 'w') as ds: ds.Conventions = 'CF-1.6' # dimensions ds.createDimension('station', 1) ds.createDimension('time', None) # standard variables # Coordinate Reference System crs = ds.createVariable('crs', 'i4') epsg_spec = 'EPSG:{0}'.format(self.ts_meta_info.epsg_id) crs.epsg_code = epsg_spec crs.proj4 = Proj( init=epsg_spec).srs # shyft expect crs.proj4 to exist crs.grid_mapping_name = 'transverse_mercator' ts_id = ds.createVariable('series_name', 'str', ('station', )) ts_id.long_name = 'timeseries_id' ts_id.cf_role = 'timeseries_id' # ts_id.units = '' time = ds.createVariable('time', 'i8', ('time', ), least_significant_digit=1, zlib=True) time.long_name = 'time' time.units = 'seconds since 1970-01-01 00:00:00 +00:00' time.calendar = 'gregorian' x = ds.createVariable('x', 'f8', ('station', )) x.axis = 'X' x.standard_name = 'projection_x_coordinate' x.units = 'm' y = ds.createVariable('y', 'f8', ('station', )) y.axis = 'Y' y.standard_name = 'projection_y_coordinate' y.units = 'm' z = ds.createVariable('z', 'f8', ('station', )) z.axis = 'Z' z.standard_name = 'height' z.long_name = 'height above mean sea level' z.units = 'm' v = ds.createVariable(self.ts_meta_info.variable_name, 'f8', dimensions=('time', 'station'), zlib=self.ts_meta_info.zlib) v.units = self.ts_meta_info.units v.standard_name = self.ts_meta_info.standard_name v.long_name = self.ts_meta_info.long_name v.coordinates = 'y x z' v.grid_mapping = 'crs' x[0] = self.ts_meta_info.x y[0] = self.ts_meta_info.y z[0] = self.ts_meta_info.z
import csv import json import numpy as np import matplotlib.pyplot as plt from boto3.session import Session from cartopy import config as cartoconfig from datetime import datetime from os import environ, remove from PIL import Image from pyproj import Proj cartoconfig['data_dir'] = '/tmp/' # NDFD CONUS Projection p = Proj( '+units=m +a=6371200.0 +b=6371200.0 +lon_0=265.0 +proj=lcc +lat_2=25.0 +lat_1=25.0 +lat_0=25.0' ) offset_x, offset_y = p(238.445999, 20.191999) # Initialize boto3 clients aws = Session() athena = aws.client('athena') s3 = aws.resource('s3') bucket = s3.Bucket(environ['OUTPUT_BUCKET']) # Some custom exceptions for state machine logic class QueryIncompleteException(Exception): pass
# -*- coding: utf-8 -*- from shapefile import Reader from pyproj import Proj, transform lambert_93 = Proj("+init=EPSG:2154") wgs_84 = Proj("+init=EPSG:4326") # Exemple d'utilisation de la librairie pyproj # lat, lon = 656936.8, 3042238.0 # x, y = transform(lambert, wgs84, lat, lon) sf = Reader("./tipi/data/DEPARTEMENT/DEPARTEMENT") shapes = sf.shapes() shapeRecs = sf.shapeRecords() points = shapeRecs[5].shape.points print shapeRecs[5].record # print points limite_departement = [] for i in points: lon, lat = transform(lambert_93, wgs_84, i[0], i[1]) limite_departement.append([lon, lat]) departement = { "type": "Feature", "properties": {"party": "Republican"}, "style" : { "color": "#ff7800", "weight": 5,
def convert_3857_to_lonlat(x, y): p1 = Proj(init='epsg:3857') x2, y2 = p1(x, y, inverse=True) return x2, y2
def tm128_to_wgs84(x, y): return transform( Proj(**TM128), Proj(**WGS84), x, y )
def read_schedule(schedule_path, epsg): """ Read MATSim schedule :param schedule_path: path to the schedule.xml file :param epsg: 'epsg:12345' :return: list of Service objects """ services = [] transformer = Transformer.from_proj(Proj(epsg), Proj('epsg:4326')) def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode): mode = transportMode['transportMode'] service_id = transitLine['transitLine']['id'] service_routes = [] for transitRoute, transitRoute_val in transitRoutes.items(): stops = [ Stop(s['stop']['refId'], x=transit_stop_id_mapping[s['stop']['refId']]['x'], y=transit_stop_id_mapping[s['stop']['refId']]['y'], epsg=epsg, transformer=transformer) for s in transitRoute_val['stops'] ] for s in stops: s.add_additional_attributes(transit_stop_id_mapping[s.id]) arrival_offsets = [] departure_offsets = [] await_departure = [] for stop in transitRoute_val['stops']: if 'departureOffset' not in stop[ 'stop'] and 'arrivalOffset' not in stop['stop']: pass elif 'departureOffset' not in stop['stop']: arrival_offsets.append(stop['stop']['arrivalOffset']) departure_offsets.append(stop['stop']['arrivalOffset']) elif 'arrivalOffset' not in stop['stop']: arrival_offsets.append(stop['stop']['departureOffset']) departure_offsets.append(stop['stop']['departureOffset']) else: arrival_offsets.append(stop['stop']['arrivalOffset']) departure_offsets.append(stop['stop']['departureOffset']) if 'awaitDeparture' in stop['stop']: await_departure.append( str(stop['stop']['awaitDeparture']).lower() in ['true', '1']) route = [ r_val['link']['refId'] for r_val in transitRoute_val['links'] ] trips = {} for dep in transitRoute_val['departure_list']: trips[dep['departure'] ['id']] = dep['departure']['departureTime'] r = Route(route_short_name=transitLine['transitLine']['name'], mode=mode, stops=stops, route=route, trips=trips, arrival_offsets=arrival_offsets, departure_offsets=departure_offsets, id=transitRoute, await_departure=await_departure) service_routes.append(r) services.append(Service(id=service_id, routes=service_routes)) transitLine = {} transitRoutes = {} transportMode = {} transitStops = {} transit_stop_id_mapping = {} is_minimalTransferTimes = False minimalTransferTimes = { } # {'stop_id_1': {stop: 'stop_id_2', transfer_time: 0.0}} # transitLines for event, elem in ET.iterparse(schedule_path, events=('start', 'end')): if event == 'start': if elem.tag == 'stopFacility': attribs = elem.attrib if attribs['id'] not in transitStops: transitStops[attribs['id']] = attribs if attribs['id'] not in transit_stop_id_mapping: transit_stop_id_mapping[attribs['id']] = elem.attrib if elem.tag == 'minimalTransferTimes': is_minimalTransferTimes = not is_minimalTransferTimes if elem.tag == 'relation': if is_minimalTransferTimes: if not elem.attrib['toStop'] in minimalTransferTimes: attribs = elem.attrib minimalTransferTimes[attribs['fromStop']] = { 'stop': attribs['toStop'], 'transferTime': float(attribs['transferTime']) } if elem.tag == 'transitLine': if transitLine: write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode) transitLine = {"transitLine": elem.attrib} transitRoutes = {} if elem.tag == 'transitRoute': transitRoutes[elem.attrib['id']] = { 'stops': [], 'links': [], 'departure_list': [], 'attribs': elem.attrib } transitRoute = elem.attrib['id'] # doesn't have any attribs # if elem.tag == 'routeProfile': # routeProfile = {'routeProfile': elem.attrib} if elem.tag == 'stop': transitRoutes[transitRoute]['stops'].append( {'stop': elem.attrib}) # doesn't have any attribs # if elem.tag == 'route': # route = {'route': elem.attrib} if elem.tag == 'link': transitRoutes[transitRoute]['links'].append( {'link': elem.attrib}) # doesn't have any attribs # if elem.tag == 'departures': # departures = {'departures': elem.attrib} if elem.tag == 'departure': transitRoutes[transitRoute]['departure_list'].append( {'departure': elem.attrib}) elif (event == 'end') and (elem.tag == "transportMode"): transportMode = {'transportMode': elem.text} # add the last one write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode) return services, minimalTransferTimes
def tm127_to_wgs84(x, y): return transform( Proj(**TM127), Proj(**WGS84), x/2.5, y/2.5 )
from datashader.colors import Set1 from datashader.colors import Set2 from datashader.colors import Set3 from datashader.colors import Sets1to3 from pyproj import Proj, transform app = Flask(__name__) CORS(app) s3 = boto3.client('s3') cities = gpd.read_file('s3://makepath-reference/reference.gpkg', layer='cities') lngs, lats = transform(Proj(init='epsg:3857'), Proj(init='epsg:4326'), cities['geometry'].x.values, cities['geometry'].y.values) cities['latitude'] = lats cities['longitude'] = lngs cities['zoom'] = 9 cities['name'] = cities['CITY_NAME'] def load_census_demo(): print('Loading Census Data') parquet_file = fp.ParquetFile(path.expanduser('~/census.parq')) df = parquet_file.to_pandas() df.race = df.race.astype('category') return df
def deg2utm(Lon, Lat, utmzone=43): p = Proj(proj='utm', zone=utmzone, ellps='WGS84') x, y = p(Lon, Lat) return x, y
def test_itransform_no_error(): with pytest.warns(DeprecationWarning): pj = Proj(init="epsg:4555") pjx, pjy = pj(116.366, 39.867) list(itransform(pj, Proj(4326), [(pjx, pjy)], radians=True, errcheck=True))
def set_projection(self, projstring): try: return Proj(projstring) except RuntimeError: log.ODM_EXCEPTION( 'Could not set projection. Please use a proj4 string')
def find_line_of_sight (x_transmitter, y_transmitter, x_receiver, y_receiver, local_authority_ids): """ Takes transmitter and receiver locations and determines line of sight. Parameters ---------- x_transmitter: float Longitude coordinate of transmitter. y_transmitter : float Latitude coordinate of transmitter. x_receiver : float Longitude coordinate of receiver. y_receiver : float Latitude coordinate of transmitter. local_authority_ids : list Any local authority id for which the area in question intersects. """ projOSGB36 = Proj(init='epsg:27700') projWGS84 = Proj(init='epsg:4326') x_transmitter, y_transmitter = transform( projWGS84, projOSGB36, x_transmitter, y_transmitter ) x_receiver, y_receiver = transform(projWGS84, projOSGB36, x_receiver, y_receiver) x_coordinates = [] y_coordinates = [] x_coordinates.extend([x_transmitter, x_receiver]) y_coordinates.extend([y_transmitter, y_receiver]) x_min, x_max = min(x_coordinates), max(x_coordinates) y_min, y_max = min(y_coordinates), max(y_coordinates) tile_ids = find_osbg_tile(x_min, y_min, x_max, y_max) print(tile_ids) # premises_data = read_premises_data(x_min, y_min, x_max, y_max, local_authority_ids) # if len(premises_data) < 1: # line_of_sight = 'los' # else: # tile_ids = find_osbg_tile(x_min, y_min, x_max, y_max) # #get building heights # building_heights = [] # for tile_id in tile_ids: # pathlist = glob.iglob(os.path.join(DATA_RAW_INPUTS,'mastermap_building_heights_2726794', # tile_id['tile_ref_2_digit'] + '/*.csv')) # for path in pathlist: # if path[-10:-6] == tile_id['tile_ref_4_digit'].lower(): # with open(path, 'r') as system_file: # reader = csv.reader(system_file) # next(reader) # for line in reader: # building_heights.append({ # 'id': line[0], # 'max_height': line[6], # }) # else: # pass # #match premises with building heights # premises_with_heights = [] # for premises in premises_data: # for building_height in building_heights: # if premises['properties']['uid'] == building_height['id']: # premises_with_heights.append({ # 'type': "Feature", # 'geometry': { # "type": "Point", # "coordinates": [premises['geometry']['coordinates']] # }, # 'properties': { # 'uid': premises['properties']['uid'], # 'max_height': building_height['max_height'] # } # }) # # make sure viewshed files have been generated # # get list of required tile ids # Create path output_dir = os.path.abspath(os.path.join(DATA_INTERMEDIATE, 'viewshed_tiles')) if not os.path.exists(output_dir): os.makedirs(output_dir) final_tile_id_list = [] for tile_id in tile_ids: pathlist = glob.iglob(os.path.join(DATA_RAW_INPUTS,'terrain-5-dtm_2736772', tile_id['tile_ref_2_digit'] + '/*.asc')) for tile_path in pathlist: filename = os.path.basename(tile_path)[:-4] if filename == tile_id['full_tile_ref']: final_tile_id_list.append(tile_path) load_raster_files(final_tile_id_list, output_dir, x_transmitter, y_transmitter) # if len(final_tile_id_list) == 1: # filename = final_tile_id_list[0] # if not os.path.exists(os.path.join(output_dir, filename + '-viewshed.tif')): # from generate_viewshed import generate_viewshed # generate_viewshed(x_transmitter, y_transmitter, output_dir, filename, tile_path) # else: # #TODO deal with multiple tiles # for filename in final_tile_id_list: # if not os.path.exists(os.path.join(output_dir, filename + '-viewshed.tif')): # from generate_viewshed import generate_viewshed # #merge files together # #then generate viewshed # generate_viewshed(x_transmitter, y_transmitter, output_dir, filename, tile_path) # #load in raster viewshed files # for final_tile in final_tile_id_list: # path = os.path.join(output_dir,final_tile + '-viewshed.tif') # src = rasterio.open(path) # from rasterio.plot import show # show(src, cmap='terrain') # for val in src.sample([(x_receiver, y_receiver)]): # if val == 0: # line_of_sight = 'nlos' # elif val == 1: # line_of_sight = 'los' # else: # print('binary viewshed .tif returning non-conforming value') return print('complete')#line_of_sight