def WriteLineShp(lineList, outShp, lineFieldsList = None): print "Write line shapefile: %s" % outShp gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "NO") ## support for path in Chinese gdal.SetConfigOption("SHAPE_ENCODING", "") ## suppoert for field in Chinese ogr.RegisterAll() driver = ogr.GetDriverByName("ESRI Shapefile") if driver is None: print "ESRI Shapefile driver not available." sys.exit(1) if os.path.exists(outShp): driver.DeleteDataSource(outShp) fieldName = [] fieldNameIdx = 0 if lineFieldsList is not None: if len(lineList) != len(lineFieldsList): if len(lineList) + 1 == len(lineFieldsList): fieldName = lineFieldsList[0] fieldNameIdx = 1 else: sys.exit(1) else: fieldLength = len(lineFieldsList[0]) for i in range(fieldLength): name = 'lineName' + str(i) fieldName.append(name) fieldNameIdx = 0 else: fieldName = ['LineName'] fieldNameIdx = 0 ds = driver.CreateDataSource(outShp.rpartition(os.sep)[0]) if ds is None: print "ERROR Output: Creation of output file failed." sys.exit(1) lyr = ds.CreateLayer(outShp.rpartition(os.sep)[2].split('.')[0], None, ogr.wkbLineString) ## create fields for fld in fieldName: nameField = ogr.FieldDefn(fld, ogr.OFTString) lyr.CreateField(nameField) for l in lineList: idx = lineList.index(l) if len(l) > 1: line = ogr.Geometry(ogr.wkbLineString) for i in l: line.AddPoint(i[0], i[1]) templine = ogr.CreateGeometryFromJson(line.ExportToJson()) feature = ogr.Feature(lyr.GetLayerDefn()) feature.SetGeometry(templine) for fld in fieldName: idx2 = fieldName.index(fld) if lineFieldsList is not None: if fieldNameIdx == 1: fieldValue = lineFieldsList[idx + 1][idx2] else: fieldValue = lineFieldsList[idx][idx2] else: fieldValue = ' ' # print fieldValue feature.SetField(fld, fieldValue) lyr.CreateFeature(feature) feature.Destroy() ds.Destroy()
def main(ini_path, overwrite_flag=True): """Earth Engine Beamer ET Zonal Stats Args: ini_path (str): overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nEarth Engine Beamer ET Zonal Stats') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='SPATIAL') inputs.parse_section(ini, section='BEAMER') inputs.parse_section(ini, section='EXPORT') inputs.parse_section(ini, section='ZONAL_STATS') # Overwrite landsat products with Beamer specific values ini['EXPORT']['landsat_products'] = [ 'ndvi_toa', 'ndwi_toa', 'albedo_sur', 'ts', 'evi_sur', 'etstar_mean', 'etg_mean', 'etg_lpi', 'etg_upi', 'etg_lci', 'etg_uci', 'etg_mean', 'et_lpi', 'et_upi', 'et_lci', 'et_uci' ] # First row of csv is header header_list = [ 'ZONE_NAME', 'ZONE_FID', 'DATE', 'SCENE_ID', 'PLATFORM', 'PATH', 'ROW', 'YEAR', 'MONTH', 'DAY', 'DOY', 'PIXEL_COUNT', 'PIXEL_TOTAL', 'FMASK_COUNT', 'FMASK_TOTAL', 'FMASK_PCT', 'ETSTAR_COUNT', 'CLOUD_SCORE', 'QA', 'NDVI_TOA', 'NDWI_TOA', 'ALBEDO_SUR', 'TS', 'EVI_SUR', 'ETSTAR_MEAN', 'ETG_MEAN', 'ETG_LPI', 'ETG_UPI', 'ETG_LCI', 'ETG_UCI', 'ET_MEAN', 'ET_LPI', 'ET_UPI', 'ET_LCI', 'ET_UCI', 'WY_ETO', 'WY_PPT' ] int_fields = [ 'ZONE_FID', 'PATH', 'ROW', 'YEAR', 'MONTH', 'DAY', 'DOY', 'PIXEL_COUNT', 'PIXEL_TOTAL', 'FMASK_COUNT', 'FMASK_TOTAL', 'ETSTAR_COUNT' ] float_fields = list( set(header_list) - set(int_fields) - set(['ZONE_NAME', 'DATE', 'SCENE_ID', 'PLATFORM'])) # Regular expression to pull out Landsat scene_id # If RE has capturing groups, findall call below will fail to extract ID landsat_re = re.compile('L[ETC]0[4578]_\d{3}XXX_\d{4}\d{2}\d{2}') # landsat_re = re.compile('L[ETC][4578]\d{3}XXX\d{4}\d{3}') # landsat_re = re.compile('L[ETC][4578]\d{3}\d{3}\d{4}\d{3}\D{3}\d{2}') # Remove the existing CSV output_path = os.path.join(ini['ZONAL_STATS']['output_ws'], ini['BEAMER']['output_name']) if overwrite_flag and os.path.isfile(output_path): os.remove(output_path) # Create an empty CSV if not os.path.isfile(output_path): data_df = pd.DataFrame(columns=header_list) data_df[int_fields] = data_df[int_fields].astype(np.int64) data_df[float_fields] = data_df[float_fields].astype(np.float32) data_df.to_csv(output_path, index=False) # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # zone_count = len(zone_geom_list) # output_fmt = '_{0:0%sd}.csv' % str(int(math.log10(zone_count)) + 1) # Check if the zone_names are unique # Eventually support merging common zone_names if len(set([z[1] for z in zone_geom_list])) != len(zone_geom_list): logging.error( '\nERROR: There appear to be duplicate zone ID/name values.' '\n Currently, the values in "{}" must be unique.' '\n Exiting.'.format(ini['INPUTS']['zone_field'])) return False # Filter features by FID if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # Merge geometries if ini['INPUTS']['merge_geom_flag']: merge_geom = ogr.Geometry(ogr.wkbMultiPolygon) for zone in zone_geom_list: zone_multipolygon = ogr.ForceToMultiPolygon( ogr.CreateGeometryFromJson(json.dumps(zone[2]))) for zone_polygon in zone_multipolygon: merge_geom.AddGeometry(zone_polygon) # merge_json = json.loads(merge_mp.ExportToJson()) zone_geom_list = [[ 0, ini['INPUTS']['zone_filename'], json.loads(merge_geom.ExportToJson()) ]] ini['INPUTS']['zone_field'] = '' # Set all zone specific parameters into a dictionary zone = {} # Need zone_shp_path projection to build EE geometries zone['osr'] = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path']) zone['proj'] = gdc.osr_wkt(zone['osr']) # zone['proj'] = ee.Projection(zone['proj']).wkt().getInfo() # zone['proj'] = zone['proj'].replace('\n', '').replace(' ', '') # logging.debug(' Zone Projection: {}'.format(zone['proj'])) # Check that shapefile has matching spatial reference if not gdc.matching_spatref(zone['osr'], ini['SPATIAL']['osr']): logging.warning(' Zone OSR:\n{}\n'.format(zone['osr'])) logging.warning(' Output OSR:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.warning(' Zone Proj4: {}'.format( zone['osr'].ExportToProj4())) logging.warning(' Output Proj4: {}'.format( ini['SPATIAL']['osr'].ExportToProj4())) logging.warning( '\nWARNING: \n' 'The output and zone spatial references do not appear to match\n' 'This will likely cause problems!') input('Press ENTER to continue') else: logging.debug(' Zone Projection:\n{}\n'.format( zone['osr'].ExportToWkt())) logging.debug(' Output Projection:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.debug(' Output Cellsize: {}'.format( ini['SPATIAL']['cellsize'])) # Initialize Earth Engine API key logging.info('\nInitializing Earth Engine') ee.Initialize() utils.ee_request(ee.Number(1).getInfo()) # Read in ETo and PPT data from file if (ini['BEAMER']['eto_source'] == 'file' or ini['BEAMER']['ppt_source'] == 'file'): data_array = np.atleast_1d( np.genfromtxt(ini['BEAMER']['data_path'], delimiter=',', names=True, encoding=None, dtype=None)) data_fields = data_array.dtype.names logging.debug(' CSV fields: {}'.format(', '.join(data_fields))) # DEADBEEF - Compare fields names assuming all upper case data_fields = [f.upper() for f in data_fields] eto_dict = defaultdict(dict) ppt_dict = defaultdict(dict) for row in data_array: z = str(row[data_fields.index(ini['BEAMER']['data_zone_field'])]) y = row[data_fields.index(ini['BEAMER']['data_year_field'])] if ini['BEAMER']['eto_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case eto_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_eto_field'].upper())] if ini['BEAMER']['ppt_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case ppt_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_ppt_field'].upper())] # Get filtered/merged/prepped Landsat collection landsat_args = { k: v for section in ['INPUTS'] for k, v in ini[section].items() if k in [ 'landsat4_flag', 'landsat5_flag', 'landsat7_flag', 'landsat8_flag', 'fmask_flag', 'acca_flag', 'start_year', 'end_year', 'start_month', 'end_month', 'start_doy', 'end_doy', 'scene_id_keep_list', 'scene_id_skip_list', 'path_keep_list', 'row_keep_list', 'tile_geom', 'adjust_method', 'mosaic_method', 'refl_sur_method' ] } landsat_args['products'] = ini['EXPORT']['landsat_products'] landsat = ee_common.Landsat(landsat_args) # Calculate zonal stats for each feature separately for zone_fid, zone_name, zone_json in zone_geom_list: zone['fid'] = zone_fid zone['name'] = zone_name.replace(' ', '_') zone['json'] = zone_json logging.info('ZONE: {} (FID: {})'.format(zone['name'], zone['fid'])) # zone_key used for wy_ppt and wy_eto inputs from csv file if ini['INPUTS']['zone_field'] == 'FID': zone_key = str(zone['fid']) print('Using FID as zone_field') else: zone_key = zone['name'] print('Using Name as zone_field') # Build EE geometry object for zonal stats zone['geom'] = ee.Geometry(geo_json=zone['json'], opt_proj=zone['proj'], opt_geodesic=False) # logging.debug(' Centroid: {}'.format( # zone['geom'].centroid(100).getInfo()['coordinates'])) # Use feature geometry to build extent, transform, and shape zone['extent'] = gdc.Extent( ogr.CreateGeometryFromJson(json.dumps(zone['json'])).GetEnvelope()) # zone['extent'] = gdc.Extent(zone['geom'].GetEnvelope()) zone['extent'] = zone['extent'].ogrenv_swap() zone['extent'] = zone['extent'].adjust_to_snap( 'EXPAND', ini['SPATIAL']['snap_x'], ini['SPATIAL']['snap_y'], ini['SPATIAL']['cellsize']) zone['geo'] = zone['extent'].geo(ini['SPATIAL']['cellsize']) zone['transform'] = gdc.geo_2_ee_transform(zone['geo']) # zone['transform'] = '[' + ','.join(map(str, zone['transform'])) + ']' zone['shape'] = zone['extent'].shape(ini['SPATIAL']['cellsize']) logging.debug(' Zone Shape: {}'.format(zone['shape'])) logging.debug(' Zone Transform: {}'.format(zone['transform'])) logging.debug(' Zone Extent: {}'.format(zone['extent'])) # logging.debug(' Zone Geom: {}'.format(zone['geom'].getInfo())) # Assume all pixels in all 14+2 images could be reduced zone['max_pixels'] = zone['shape'][0] * zone['shape'][1] logging.debug(' Max Pixels: {}'.format(zone['max_pixels'])) # Set output spatial reference # Eventually allow user to manually set these # output_crs = zone['proj'] # ini['INPUTS']['transform'] = zone['transform'] logging.debug(' Output Projection: {}'.format(ini['SPATIAL']['crs'])) logging.debug(' Output Transform: {}'.format(zone['transform'])) # Process date range by year start_dt = datetime.datetime(ini['INPUTS']['start_year'], 1, 1) end_dt = datetime.datetime(ini['INPUTS']['end_year'] + 1, 1, 1) - datetime.timedelta(0, 1) iter_months = ini['BEAMER']['month_step'] for i, iter_start_dt in enumerate( rrule.rrule( # rrule.YEARLY, interval=interval_cnt, rrule.MONTHLY, interval=iter_months, dtstart=start_dt, until=end_dt)): iter_end_dt = ( iter_start_dt + # relativedelta.relativedelta(years=interval_cnt) - relativedelta.relativedelta(months=iter_months) - datetime.timedelta(0, 1)) if ((ini['INPUTS']['start_month'] and iter_end_dt.month < ini['INPUTS']['start_month']) or (ini['INPUTS']['end_month'] and iter_start_dt.month > ini['INPUTS']['end_month'])): logging.debug(' {} {} skipping'.format( iter_start_dt.date(), iter_end_dt.date())) continue elif ( (ini['INPUTS']['start_doy'] and int(iter_end_dt.strftime('%j')) < ini['INPUTS']['start_doy']) or (ini['INPUTS']['end_doy'] and int(iter_start_dt.strftime('%j')) > ini['INPUTS']['end_doy'])): logging.debug(' {} {} skipping'.format( iter_start_dt.date(), iter_end_dt.date())) continue else: logging.info(' {} {}'.format(iter_start_dt.date(), iter_end_dt.date())) year = iter_start_dt.year # Filter the GRIDMET collection wy_start_date = '{}-10-01'.format(year - 1) wy_end_date = '{}-10-01'.format(year) logging.debug(' WY: {} {}'.format(wy_start_date, wy_end_date)) gridmet_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \ .filterDate(wy_start_date, wy_end_date) # # PRISM collection was uploaded as an asset # if ini['BEAMER']['ppt_source'] == 'prism': # def prism_time_start(input_image): # """Set time_start property on PRISM water year PPT collection""" # # Assume year is the 4th item separated by "_" # wy = ee.String( # input_image.get('system:index')).split('_').get(3) # date_start = ee.Date(ee.String(wy).cat('-10-01')) # return input_image.select([0], ['ppt']).setMulti({ # 'system:time_start': date_start.advance(-1, 'year').millis() # }) # prism_coll = ee.ImageCollection('users/cgmorton/prism_800m_ppt_wy') # prism_coll = ee.ImageCollection(prism_coll.map(prism_time_start)) \ # .filterDate(wy_start_dt, wy_end_dt) # # prism_coll = ee.ImageCollection( # # ee_common.MapsEngineAssets.prism_ppt_wy).filterDate( # # wy_start_dt, wy_end_dt) # Get water year PPT for centroid of zone or read from file # Convert all input data to mm to match GRIDMET data if ini['BEAMER']['ppt_source'] == 'file': wy_ppt_input = ppt_dict[zone_key][year] if ini['BEAMER']['data_ppt_units'] == 'mm': pass elif ini['BEAMER']['data_ppt_units'] == 'in': wy_ppt_input *= 25.4 elif ini['BEAMER']['data_ppt_units'] == 'ft': wy_ppt_input *= (25.4 * 12) elif ini['BEAMER']['ppt_source'] == 'gridmet': wy_ppt_input = float( utils.ee_getinfo( ee.ImageCollection( gridmet_coll.select(['pr'], ['ppt']).sum()).getRegion( zone['geom'].centroid(1), 500))[1][4]) # Calculate GRIDMET zonal mean of geometry # wy_ppt_input = float(ee.ImageCollection( # gridmet_coll.select(['pr'], ['ppt'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo()['ppt'] # elif ini['BEAMER']['ppt_source'] == 'prism': # # Calculate PRISM zonal mean of geometry # wy_ppt_input = float(utils.ee_getinfo(ee.ImageCollection( # prism_coll.map(ee_common.prism_ppt_func)).sum().reduceRegion( # reducer=ee.Reducer.mean(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1))['ppt']) # Get water year ETo for centroid of zone or read from file # Convert all input data to mm for Beamer Method if ini['BEAMER']['eto_source'] == 'FILE': wy_eto_input = eto_dict[zone_key][year] if ini['BEAMER']['data_eto_units'] == 'mm': pass elif ini['BEAMER']['data_eto_units'] == 'in': wy_eto_input *= 25.4 elif ini['BEAMER']['data_eto_units'] == 'ft': wy_eto_input *= (25.4 * 12) # This assumes GRIMET data is in millimeters elif ini['BEAMER']['eto_source'] == 'gridmet': wy_eto_input = float( utils.ee_getinfo( ee.ImageCollection(gridmet_coll.select( ['eto']).sum()).getRegion(zone['geom'].centroid(1), 500))[1][4]) # wy_eto_input = float(ee.ImageCollection( # gridmet_coll.select(['eto'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo() logging.debug(' Input ETO: {} mm PPT: {} mm'.format( wy_eto_input, wy_ppt_input)) # Scale ETo & PPT wy_eto_input *= ini['BEAMER']['eto_factor'] wy_ppt_input *= ini['BEAMER']['ppt_factor'] # Convert output units from mm wy_eto_output = wy_eto_input wy_ppt_output = wy_ppt_input if ini['BEAMER']['ppt_units'] == 'mm': pass elif ini['BEAMER']['ppt_units'] == 'in': wy_ppt_output /= 25.4 elif ini['BEAMER']['ppt_units'] == 'ft': wy_ppt_output /= (25.4 * 12) if ini['BEAMER']['eto_units'] == 'mm': pass elif ini['BEAMER']['eto_units'] == 'in': wy_eto_output /= 25.4 elif ini['BEAMER']['eto_units'] == 'ft': wy_eto_output /= (25.4 * 12) logging.debug(' Output ETO: {} {} PPT: {} {}'.format( wy_eto_output, ini['BEAMER']['eto_units'], wy_ppt_output, ini['BEAMER']['ppt_units'])) # Initialize the Landsat object landsat.zone_geom = zone['geom'] landsat.start_date = iter_start_dt.strftime('%Y-%m-%d') landsat.end_date = iter_end_dt.strftime('%Y-%m-%d') landsat_coll = landsat.get_collection() if ee.Image(landsat_coll.first()).getInfo() is None: logging.info(' No images, skipping') continue # # Print the collection SCENE_ID list # logging.debug('{}'.format(', '.join([ # f['properties']['SCENE_ID'] # for f in landsat_coll.getInfo()['features']]))) # input('ENTER') # Add water year ETo and PPT values to each image def eto_ppt_func(img): """""" return ee.Image(img).setMulti({ 'wy_eto': wy_eto_input, 'wy_ppt': wy_ppt_input }) landsat_coll = ee.ImageCollection(landsat_coll.map(eto_ppt_func)) # Compute ETg image_coll = ee.ImageCollection(landsat_coll.map(landsat_etg_func)) # # Get the output image URL # output_url = ee.Image(landsat_coll.first()) \ # .select(['red', 'green', 'blue']) \ # .visualize(min=[0, 0, 0], max=[0.4, 0.4, 0.4]) \ # .getThumbUrl({'format': 'png', 'size': '600'}) # # This would load the image in your browser # import webbrowser # webbrowser.open(output_url) # # webbrowser.read(output_url) # # Show the output image # window = tk.Tk() # output_file = Image.open(io.BytesIO(urllib.urlopen(output_url).read())) # output_photo = ImageTk.PhotoImage(output_file) # label = tk.Label(window, image=output_photo) # label.pack() # window.mainloop() # Compute zonal stats of polygon def beamer_zonal_stats_func(input_image): """""" # Beamer function adds 5 ETg and 1 ET* band # Landsat collection adds 3 ancillary bands bands = len(landsat_args['products']) + 3 + 6 # .clip(zone['geom']) \ input_mean = input_image \ .reduceRegion( reducer=ee.Reducer.mean(), geometry=zone['geom'], crs=ini['SPATIAL']['crs'], crsTransform=zone['transform'], bestEffort=False, tileScale=1, maxPixels=zone['max_pixels'] * bands) fmask_img = input_image.select(['fmask']) input_count = fmask_img.gt(1) \ .addBands(fmask_img.gte(0).unmask()) \ .rename(['fmask', 'pixel']) \ .reduceRegion( reducer=ee.Reducer.sum().combine( ee.Reducer.count(), '', True), geometry=zone['geom'], crs=ini['SPATIAL']['crs'], crsTransform=zone['transform'], bestEffort=False, tileScale=1, maxPixels=zone['max_pixels'] * 3) etstar_count = input_image \ .select(['etstar_mean'], ['etstar_count']) \ .lte(ini['BEAMER']['etstar_threshold']) \ .reduceRegion( reducer=ee.Reducer.sum(), geometry=zone['geom'], crs=ini['SPATIAL']['crs'], crsTransform=zone['transform'], bestEffort=False, tileScale=1, maxPixels=zone['max_pixels'] * 2) # Save as image properties return ee.Feature( None, { 'scene_id': ee.String(input_image.get('SCENE_ID')), 'time': input_image.get('system:time_start'), 'row': input_mean.get('row'), 'pixel_count': input_count.get('pixel_sum'), 'pixel_total': input_count.get('pixel_count'), 'fmask_count': input_count.get('fmask_sum'), 'fmask_total': input_count.get('fmask_count'), 'cloud_score': input_mean.get('cloud_score'), 'etstar_count': etstar_count.get('etstar_count'), 'ndvi_toa': input_mean.get('ndvi_toa'), 'ndwi_toa': input_mean.get('ndwi_toa'), 'albedo_sur': input_mean.get('albedo_sur'), 'ts': input_mean.get('ts'), 'evi_sur': input_mean.get('evi_sur'), 'etstar_mean': input_mean.get('etstar_mean'), 'etg_mean': input_mean.get('etg_mean'), 'etg_lpi': input_mean.get('etg_lpi'), 'etg_upi': input_mean.get('etg_upi'), 'etg_lci': input_mean.get('etg_lci'), 'etg_uci': input_mean.get('etg_uci') }) # Calculate values and statistics stats_coll = ee.ImageCollection( image_coll.map(beamer_zonal_stats_func)) # # DEADBEEF - Test the function for a single image # stats_info = beamer_zonal_stats_func( # ee.Image(image_coll.first())).getInfo() # print(stats_info) # for k, v in sorted(stats_info['properties'].items()): # logging.info('{:24s}: {}'.format(k, v)) # input('ENTER') # return False # # DEADBEEF - Print the stats info to the screen # stats_info = stats_coll.getInfo() # import pprint # pp = pprint.PrettyPrinter(indent=4) # for ftr in stats_info['features']: # pp.pprint(ftr) # input('ENTER') # # return False # Get the values from EE stats_desc = utils.ee_getinfo(stats_coll) if stats_desc is None: logging.error(' Timeout error, skipping') continue # Save data for writing row_list = [] for ftr in stats_desc['features']: try: count = int(ftr['properties']['pixel_count']) except (KeyError, TypeError) as e: # logging.debug(' Exception: {}'.format(e)) continue if count == 0: logging.info(' COUNT: 0, skipping') continue # First get scene ID and time try: scene_id = landsat_re.findall( ftr['properties']['scene_id'])[0] scene_time = datetime.datetime.utcfromtimestamp( float(ftr['properties']['time']) / 1000) except: pp = pprint.PrettyPrinter(indent=4) pp.pprint(ftr) input('ENTER') # Extract and save other properties try: row_list.append({ 'ZONE_FID': zone_fid, 'ZONE_NAME': zone_name, 'SCENE_ID': scene_id, 'PLATFORM': scene_id[0:4], 'PATH': int(scene_id[5:8]), 'ROW': int(ftr['properties']['row']), # 'ROW': int(scene_id[8:11]), 'DATE': scene_time.date().isoformat(), 'YEAR': int(scene_time.year), 'MONTH': int(scene_time.month), 'DAY': int(scene_time.day), 'DOY': int(scene_time.strftime('%j')), 'PIXEL_COUNT': int(ftr['properties']['pixel_count']), 'PIXEL_TOTAL': int(ftr['properties']['pixel_total']), 'FMASK_COUNT': int(ftr['properties']['fmask_count']), 'FMASK_TOTAL': int(ftr['properties']['fmask_total']), 'CLOUD_SCORE': float(ftr['properties']['cloud_score']), 'ETSTAR_COUNT': int(ftr['properties']['etstar_count']), 'NDVI_TOA': float(ftr['properties']['ndvi_toa']), 'NDWI_TOA': float(ftr['properties']['ndwi_toa']), 'ALBEDO_SUR': float(ftr['properties']['albedo_sur']), 'TS': float(ftr['properties']['ts']), 'EVI_SUR': float(ftr['properties']['evi_sur']), 'ETSTAR_MEAN': float(ftr['properties']['etstar_mean']), 'ETG_MEAN': float(ftr['properties']['etg_mean']), 'ETG_LPI': float(ftr['properties']['etg_lpi']), 'ETG_UPI': float(ftr['properties']['etg_upi']), 'ETG_LCI': float(ftr['properties']['etg_lci']), 'ETG_UCI': float(ftr['properties']['etg_uci']), 'WY_ETO': wy_eto_output, 'WY_PPT': wy_ppt_output }) except (KeyError, TypeError) as e: logging.info(' ERROR: {}\n SCENE_ID: {}\n ' ' There may not be an SR image to join to\n' ' {}'.format(e, scene_id, ftr['properties'])) # input('ENTER') # Save all values to the dataframe (and export) if row_list: logging.debug(' Appending') data_df = data_df.append(row_list, ignore_index=True) # DEADBEEF if data_df['QA'].isnull().any(): data_df.loc[data_df['QA'].isnull(), 'QA'] = 0 fmask_mask = data_df['FMASK_TOTAL'] > 0 if fmask_mask.any(): data_df.loc[fmask_mask, 'FMASK_PCT'] = 100.0 * ( data_df.loc[fmask_mask, 'FMASK_COUNT'] / data_df.loc[fmask_mask, 'FMASK_TOTAL']) logging.debug(' Saving') data_df[int_fields] = data_df[int_fields].astype(np.int64) data_df[float_fields] = data_df[float_fields].astype( np.float32) # Compute ET from ETg and PPT offline # (must be after float conversion above) data_df['ET_MEAN'] = data_df['ETG_MEAN'] + data_df['WY_PPT'] data_df['ET_LPI'] = data_df['ETG_LPI'] + data_df['WY_PPT'] data_df['ET_UPI'] = data_df['ETG_UPI'] + data_df['WY_PPT'] data_df['ET_LCI'] = data_df['ETG_LCI'] + data_df['WY_PPT'] data_df['ET_UCI'] = data_df['ETG_UCI'] + data_df['WY_PPT'] # Convert float fields to objects, set NaN to None for field in data_df.columns.values: if field.upper() not in float_fields: continue data_df[field] = data_df[field].astype(object) null_mask = data_df[field].isnull() data_df.loc[null_mask, field] = None data_df.loc[~null_mask, field] = data_df.loc[~null_mask, field].map( lambda x: '{0:10.6f}'.format(x).strip()) # data_df.loc[~null_mask, [field]] = data_df.loc[~null_mask, [field]].apply( # lambda x: '{0:10.6f}'.format(x[0]).strip(), axis=1) # data_df = data_df.reindex_axis(header_list, axis=1) data_df = data_df.reindex(header_list, axis=1) # data_df.reset_index(drop=False, inplace=True) data_df.sort_values(['ZONE_FID', 'DATE', 'ROW'], ascending=True, inplace=True) # data_df.sort( # ['ZONE_NAME', 'DATE'], ascending=[True, True], inplace=True) data_df.to_csv(output_path, index=False) del row_list
def rcbl(parcel, start_date, end_date, bands, chipsize, filespath, quiet=True): """Get parcel raw chip images from RESTful API by location""" import os import os.path import pandas as pd from osgeo import osr, ogr import time start = time.time() api_url, api_user, api_pass = config.credentials('api') for band in bands: requrl = """{}/query/rawChipByLocation?lon={}&lat={}&start_date={}&end_date={}""" if band is not None: requrl = f"{requrl}&band={band}" if chipsize is not None: requrl = f"{requrl}&chipsize={chipsize}" # Create a valid geometry from the returned JSON withGeometry geom = ogr.CreateGeometryFromJson(parcel.get('geom')[0]) source = osr.SpatialReference() source.ImportFromEPSG(parcel.get('srid')[0]) # Assign this projection to the geometry geom.AssignSpatialReference(source) target = osr.SpatialReference() target.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source, target) # And get the lon, lat for its centroid, so that we can center the chips # on the parcel centroid = geom.Centroid() centroid.Transform(transform) # Use pid for next request # pid = parcel['pid'][0] # cropname = parcel['cropname'][0] # Set up the rawChip request cen_x, cen_y = str(centroid.GetX()), str(centroid.GetY()) response = requests.get(requrl.format(api_url, cen_y, cen_x, start_date, end_date, band, chipsize), auth=(api_user, api_pass)) if not quiet: print( "Request url:", requrl.format(api_url, cen_y, cen_x, start_date, end_date, band, chipsize)) print("Geom:", geom) print("Source:", source, ", Target:", target) print("Centroid", centroid) print("Response:", response) # Directly create a pandas DataFrame from the json response df = pd.read_json(response.content) os.makedirs(filespath, exist_ok=True) df_file = normpath(join(filespath, f'images_list.{band}.csv')) df.to_csv(df_file, index=True, header=True) # print(f"The response table is saved to: {df_file}") # Download the GeoTIFFs that were just created in the user cache for c in df.chips: url = f"{api_url}{c}" outf = normpath(join(filespath, c.split('/')[-1])) if not isfile(outf): res = requests.get(url, stream=True) if not quiet: print(f"Downloading {c.split('/')[-1]}") with open(outf, "wb") as handle: for chunk in res.iter_content(chunk_size=512): if chunk: # filter out keep-alive new chunks handle.write(chunk) if not quiet: print( f"Images for band '{band}', for the selected dates are downloaded." ) if not quiet: print("\n------Total time------") print( f"Total time required for {len(bands)} bands: {time.time() - start} seconds." )
def crop_by_geojson(file_name): # geojson = jsonify(request.get_json(force=True)) geojson = { "type": "Feature", "properties": { "id": 1 }, "geometry": { "type": "Multipolygon", "coordinates": [[[[27.397511483867362, 44.161117466010516], [27.393924221672666, 44.159751598403503], [27.393556666460618, 44.159252063395591], [27.393726740035870, 44.158373985750522], [27.392040835956994, 44.157378400690988], [27.390354358253163, 44.156239034941315], [27.390977924658255, 44.152849194060536], [27.391438333095618, 44.149298658002031], [27.386781918912796, 44.147461728155896], [27.384487250437232, 44.146859408403664], [27.382636468741264, 44.156671855578281], [27.383891699721374, 44.156645049015140], [27.384649769913505, 44.157388133683327], [27.385547083122507, 44.160232076255667], [27.387997850095061, 44.160722084482430], [27.390672446485077, 44.161638147279866], [27.395361188085396, 44.163429614137918], [27.396513835695238, 44.162325787855522], [27.397511483867362, 44.161117466010516]]]] } } min_x, max_x, min_y, max_y = bbox(geojson) # Register Imagine driver and open file driver = gdal.GetDriverByName('GTiff') driver.Register() dataset = gdal.Open('{}/{}.tif'.format(image_dir, file_name)) if dataset is None: print 'Could not open ' + file_name + '.tif' sys.exit(1) # Getting image dimensions cols = dataset.RasterXSize rows = dataset.RasterYSize bands = dataset.RasterCount # Getting georeference info transform = dataset.GetGeoTransform() projection = dataset.GetProjection() xOrigin = transform[0] yOrigin = transform[3] pixelWidth = transform[1] pixelHeight = -transform[5] # Getting spatial reference of input raster srs = osr.SpatialReference() srs.ImportFromWkt(projection) # WGS84 projection reference OSR_WGS84_REF = osr.SpatialReference() OSR_WGS84_REF.ImportFromEPSG(4326) # OSR transformation wgs84_to_image_trasformation = osr.CoordinateTransformation( OSR_WGS84_REF, srs) XYmin = wgs84_to_image_trasformation.TransformPoint(min_x, max_y) XYmax = wgs84_to_image_trasformation.TransformPoint(max_x, min_y) # Computing Point1(i1,j1), Point2(i2,j2) i1 = int((XYmin[0] - xOrigin) / pixelWidth) j1 = int((yOrigin - XYmin[1]) / pixelHeight) i2 = int((XYmax[0] - xOrigin) / pixelWidth) j2 = int((yOrigin - XYmax[1]) / pixelHeight) new_cols = i2 - i1 + 1 new_rows = j2 - j1 + 1 # New upper-left X,Y values new_x = xOrigin + i1 * pixelWidth new_y = yOrigin - j1 * pixelHeight new_transform = (new_x, transform[1], transform[2], new_y, transform[4], transform[5]) wkt_geom = ogr.CreateGeometryFromJson("""{}""".format(geojson['geometry'])) wkt_geom.Transform(wgs84_to_image_trasformation) target_ds = gdal.GetDriverByName('MEM').Create('', new_cols, new_rows, bands, gdal.GDT_Byte) target_ds.SetGeoTransform(new_transform) target_ds.SetProjection(projection) # Create a memory layer to rasterize from. driver = ogr.GetDriverByName('Memory') memds = driver.CreateDataSource('tmpmemds') lyr = memds.CreateLayer('poly', geom_type=ogr.wkbMultiPolygon) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetGeometryDirectly(ogr.Geometry(wkt=wkt_geom.ExportToWkt())) lyr.CreateFeature(feat) gdal.RasterizeLayer(target_ds, [1, 2, 3], lyr, burn_values=[1, 1, 1]) # Create output file output_file = '{}/{}_cutted_by_geojson.tif'.format(image_dir, file_name) driver = gdal.GetDriverByName('GTiff') outds = driver.Create(output_file, new_cols, new_rows, bands, gdal.GDT_Byte) # Read in bands and store all the data in bandList mask_array = [] band_list = [] for i in range(bands): band_list.append( dataset.GetRasterBand(i + 1).ReadAsArray(i1, j1, new_cols, new_rows)) mask_array.append(target_ds.GetRasterBand(i + 1).ReadAsArray()) for j in range(bands): data = numpy.where(mask_array[j] == 1, band_list[j], 0) outds.GetRasterBand(j + 1).WriteArray(data) outds.SetProjection(projection) outds.SetGeoTransform(new_transform) dataset = None outds = None return send_file(output_file, as_attachment=True)
def cut_by_geojson(input_file, output_file, shape_geojson): # Get coords for bounding box x, y = zip(*gj.utils.coords(gj.loads(shape_geojson))) min_x, max_x, min_y, max_y = min(x), max(x), min(y), max(y) # Open original data as read only dataset = gdal.Open(input_file, gdal.GA_ReadOnly) bands = dataset.RasterCount # Getting georeference info transform = dataset.GetGeoTransform() projection = dataset.GetProjection() xOrigin = transform[0] yOrigin = transform[3] pixelWidth = transform[1] pixelHeight = -transform[5] # Getting spatial reference of input raster srs = osr.SpatialReference() srs.ImportFromWkt(projection) # WGS84 projection reference OSR_WGS84_REF = osr.SpatialReference() OSR_WGS84_REF.ImportFromEPSG(4326) # OSR transformation wgs84_to_image_trasformation = osr.CoordinateTransformation( OSR_WGS84_REF, srs) XYmin = wgs84_to_image_trasformation.TransformPoint(min_x, max_y) XYmax = wgs84_to_image_trasformation.TransformPoint(max_x, min_y) # Computing Point1(i1,j1), Point2(i2,j2) i1 = int((XYmin[0] - xOrigin) / pixelWidth) j1 = int((yOrigin - XYmin[1]) / pixelHeight) i2 = int((XYmax[0] - xOrigin) / pixelWidth) j2 = int((yOrigin - XYmax[1]) / pixelHeight) new_cols = i2 - i1 + 1 new_rows = j2 - j1 + 1 # New upper-left X,Y values new_x = xOrigin + i1 * pixelWidth new_y = yOrigin - j1 * pixelHeight new_transform = (new_x, transform[1], transform[2], new_y, transform[4], transform[5]) wkt_geom = ogr.CreateGeometryFromJson(str(shape_geojson)) wkt_geom.Transform(wgs84_to_image_trasformation) target_ds = GDAL_MEMORY_DRIVER.Create('', new_cols, new_rows, 1, gdal.GDT_Byte) target_ds.SetGeoTransform(new_transform) target_ds.SetProjection(projection) # Create a memory layer to rasterize from. ogr_dataset = OGR_MEMORY_DRIVER.CreateDataSource('shapemask') ogr_layer = ogr_dataset.CreateLayer('shapemask', srs=srs) ogr_feature = ogr.Feature(ogr_layer.GetLayerDefn()) ogr_feature.SetGeometryDirectly(ogr.Geometry(wkt=wkt_geom.ExportToWkt())) ogr_layer.CreateFeature(ogr_feature) gdal.RasterizeLayer(target_ds, [1], ogr_layer, burn_values=[1], options=["ALL_TOUCHED=TRUE"]) # Create output file driver = gdal.GetDriverByName('GTiff') outds = driver.Create(output_file, new_cols, new_rows, bands, gdal.GDT_Float32) # Read in bands and store all the data in bandList mask_array = target_ds.GetRasterBand(1).ReadAsArray() band_list = [] for i in range(bands): band_list.append( dataset.GetRasterBand(i + 1).ReadAsArray(i1, j1, new_cols, new_rows)) for j in range(bands): data = np.where(mask_array == 1, band_list[j], mask_array) outds.GetRasterBand(j + 1).SetNoDataValue(0) outds.GetRasterBand(j + 1).WriteArray(data) outds.SetProjection(projection) outds.SetGeoTransform(new_transform) target_ds = None dataset = None outds = None ogr_dataset = None
def btn_calculate(self): ret = super(DlgCalculateForestFire, self).btn_calculate() if not ret: return self.close() if self.radio_landsat8.isChecked(): prod_mode = 'L8' else: prod_mode = 'S2' crosses_180th, geojsons = self.aoi.bounding_box_gee_geojson() val = [] n = 1 if self.area_tab.area_fromfile.isChecked(): for f in self.aoi.get_layer_wgs84().getFeatures(): # Get an OGR geometry from the QGIS geometry geom = f.geometry() val.append(geom) n += 1 # stringify json object val_string = '{}'.format(json.loads(val[0].asJson())) # create ogr geometry val_geom = ogr.CreateGeometryFromJson(val_string) # simplify polygon to tolerance of 0.003 val_geom_simplified = val_geom.Simplify(0.003) # fetch coordinates from json coords = json.loads( val_geom_simplified.ExportToJson())['coordinates'] geometries = json.dumps([{ "coordinates": coords, "type": "Polygon" }]) elif self.area_tab.area_fromadmin.isChecked(): geometries = json.dumps([{ "coordinates": self.get_admin_poly_geojson()['geometry']['coordinates'][0], "type": "Polygon" }]) elif self.area_tab.area_frompoint.isChecked(): point = QgsPointXY( float(self.area_tab.area_frompoint_point_x.text()), float(self.area_tab.area_frompoint_point_y.text())) crs_src = QgsCoordinateReferenceSystem( self.area_tab.canvas.mapSettings().destinationCrs().authid()) point = QgsCoordinateTransform( crs_src, self.aoi.crs_dst, QgsProject.instance()).transform(point) geometries = json.dumps( json.loads(QgsGeometry.fromPointXY(point).asJson())) # area = self.aoi.get_area()/(1000 * 1000) # log('{0}'.format(poly)) date_format = '{0}-{1}-{2}' prefire_start = date_format.format( self.prefire_start_btn.date().year(), self.prefire_start_btn.date().month(), self.prefire_start_btn.date().day()) prefire_end = date_format.format(self.prefire_end_btn.date().year(), self.prefire_end_btn.date().month(), self.prefire_end_btn.date().day()) postfire_start = date_format.format( self.postfire_start_btn.date().year(), self.postfire_start_btn.date().month(), self.postfire_start_btn.date().day()) postfire_end = date_format.format(self.postfire_end_btn.date().year(), self.postfire_end_btn.date().month(), self.postfire_end_btn.date().day()) payload = { 'prod_mode': prod_mode, # 'area':area, 'prefire_start_btn': prefire_start, 'prefire_end_btn': prefire_end, 'postfire_start_btn': postfire_start, 'postfire_end_btn': postfire_end, 'geojsons': geometries, # 'geojsons':json.dumps(geojsons), 'crs': self.aoi.get_crs_dst_wkt(), 'crosses_180th': crosses_180th, # 'og_simple':'{}'.format(og_simple), 'task_name': self.options_tab.task_name.text(), 'task_notes': self.options_tab.task_notes.toPlainText() } resp = run_script(get_script_slug('forest-fire'), payload) if resp: mb.pushMessage( QtWidgets.QApplication.translate("MISLAND", "Submitted"), QtWidgets.QApplication.translate( "MISLAND", "Forest Fire task submitted to Google Earth Engine."), level=0, duration=5) else: mb.pushMessage( QtWidgets.QApplication.translate("MISLAND", "Error"), QtWidgets.QApplication.translate( "MISLAND", "Unable to submit forest fire task to Google Earth Engine." ), level=0, duration=5)
def do_run(fgmj): startup = { 'ffmc': {'value': 85.0}, 'dmc': {'value': 6.0}, 'dc': {'value': 15.0}, 'precipitation': {'value': 0.0}, } region = os.path.basename(os.path.dirname(os.path.dirname(fgmj))) job_name = os.path.basename(os.path.dirname(fgmj)) job_time = job_name[job_name.rindex('_') + 1:-4] job_date = job_time[:8] fire_name = job_name[:job_name.index('_')] out_dir = os.path.join(ROOT_DIR, job_date, region, fire_name, job_time) done_already = os.path.exists(out_dir) if not done_already: common.ensure_dir(out_dir) with open(fgmj) as f: data = json.load(f) MSG_DEFAULT_STARTUP = 'using default startup indices' project = data['project'] stn = try_read_first(project['stations'], 'stations', MSG_DEFAULT_STARTUP) if stn is not None: stream = try_read_first(stn['station'], 'streams', MSG_DEFAULT_STARTUP) if stream is not None: startup = stream['condition']['startingCodes'] unnest_values(startup) logging.info("Startup indices are: {}".format(startup)) ffmc = startup['ffmc'] dmc = startup['dmc'] dc = startup['dc'] apcp_0800 = float(startup['precipitation']) if np.isnan(apcp_0800): apcp_0800 = 0 pt = None ignition = try_read_first(project['ignitions'], 'ignitions', is_fatal=True) ign = try_read_first(ignition['ignition']['ignitions'], 'ignitions', is_fatal=True) perim = None poly = ign['polygon'] if poly['units'] != 'LAT_LON': logging.fatal("Only lat/long coordinates are currently supported") sys.exit(-1) if ign['polyType'] != 'POINT': if ign['polyType'] == 'POLYGON_OUT': pts = poly['polygon']['points'] pts = list(map(unnest_values, pts)) pts = [list(map(lambda v: [v['x'], v['y']], pts))] lat = statistics.mean(list(map(lambda v: v[1], pts[0]))) long = statistics.mean(list(map(lambda v: v[0], pts[0]))) # print(long) orig_zone = 15 orig_long = -93 diff = long - orig_long # print(diff) ZONE_SIZE = 6 zone_diff = round(diff / ZONE_SIZE) # print(zone_diff) meridian = orig_long + (zone_diff * ZONE_SIZE) # print(meridian) zone = orig_zone + zone_diff # print(pts) p = '''{"type": "Polygon", "coordinates": ''' + str(pts) + ''', }''' # print(p) g = ogr.CreateGeometryFromJson(p) # print(g) # print("Hi! I'm a %s with an Area %s" % (g.GetGeometryName(), g.Area())) # print("I have inside me %s feature(s)!\n" % g.GetGeometryCount()) # for idx, f in enumerate(g): # print("I'm feature n.%s and I am a %s.\t I have an Area of %s - You can get my json repr with f.ExportToJson()" % (idx, f.GetGeometryName(),f.Area())) source = osr.SpatialReference() source.ImportFromEPSG(4269) target = osr.SpatialReference() target.ImportFromEPSG(3159) z = target.ExportToWkt() z = z[:z.rindex(",AUTHORITY")] + "]" z = z.replace('UTM zone 15N', 'UTM zone {}N') z = z.replace('"central_meridian",-93', '"central_meridian",{}') z = z.format(zone, meridian) # print(z) # print(target) target.ImportFromWkt(z) transform = osr.CoordinateTransformation(source, target) g.Transform(transform) #print(g) # logging.debug("Hi! I'm a %s with an Area %s" % (g.GetGeometryName(), g.Area())) # logging.debug("I have inside me %s feature(s)!\n" % g.GetGeometryCount()) # for idx, f in enumerate(g): # logging.debug("I'm feature n.%s and I am a %s.\t I have an Area of %s - You can get my json repr with f.ExportToJson()" % (idx, f.GetGeometryName(),f.Area())) out_name = '{}.shp'.format(fire_name) out_file = os.path.join(out_dir, out_name) driver = ogr.GetDriverByName("Esri Shapefile") ds = driver.CreateDataSource(out_file) layr1 = ds.CreateLayer('', None, ogr.wkbPolygon) # create the field layr1.CreateField(ogr.FieldDefn('id', ogr.OFTInteger)) # Create the feature and set values defn = layr1.GetLayerDefn() feat = ogr.Feature(defn) feat.SetField('id', 1) feat.SetGeometry(g) layr1.CreateFeature(feat) # close the shapefile ds.Destroy() target.MorphToESRI() with open(os.path.join(out_dir, '{}.prj'.format(fire_name)), 'w') as file: file.write(target.ExportToWkt()) YEAR = 2021 perim = firestarr_gis.rasterize_perim(out_dir, out_file, YEAR, fire_name)[1] else: logging.fatal("Unsupported ignition type {}".format(ign['polyType'])) if perim is None: sys.exit(-1) else: pt = try_read_first(poly['polygon'], 'points', is_fatal=True) if pt is None: # should have already exited but check logging.fatal("Ignition point not initialized") sys.exit(-1) unnest_values(pt) lat = pt['y'] long = pt['x'] logging.info("Startup coordinates are {}, {}".format(lat, long)) scenario = try_read_first(project['scenarios'], 'scenarios', is_fatal=True)['scenario'] start_time = scenario['startTime']['time'] start_time = pd.to_datetime(start_time) logging.info("Scenario start time is: {}".format(start_time)) hour = start_time.hour minute = start_time.minute tz = (start_time.tz._minutes) / 60.0 if math.floor(tz) != tz: logging.fatal("Currently not set up to deal with partial hour timezones") sys.exit(-1) tz = int(tz) logging.info("Timezone offset is {}".format(tz)) date_offset = 0 start_date = datetime.date.today() start_date = start_time.date() if start_date != datetime.date.today(): date_offset = (start_date - datetime.date.today()).days logging.warning("Simulation does not start today - date offset set to {}".format(date_offset)) url = r"http://wxshield:80/wxshield/getWx.php?model=geps&lat={}&long={}&dateOffset={}&tz={}&mode=daily".format(lat, long, date_offset, tz) logging.debug(url) try: csv = common.download(url).decode("utf-8") except: logging.fatal("Unable to download weather") sys.exit(-3) data = [x.split(',') for x in csv.splitlines()] df = pd.DataFrame(data[1:], columns=data[0]) # print(df) # supposed to be really picky about inputs #"Scenario,Date,APCP,TMP,RH,WS,WD,FFMC,DMC,DC,ISI,BUI,FWI"; df = df[['MEMBER', 'DAILY', 'PREC', 'TEMP', 'RH', 'WS', 'WD']] df.columns = ['Scenario', 'Date', 'APCP', 'TMP', 'RH', 'WS', 'WD'] # for some reason scenario numbers are negative right now? df['Scenario'] = df['Scenario'].apply(lambda x: -1 - int(x)) df['Date'] = df['Date'].apply(lambda x: x + " 13:00:00") for col in ['FFMC', 'DMC', 'DC', 'ISI', 'BUI', 'FWI']: df[col] = 0 df.to_csv('wx.csv', index=False) cmd = "./FireSTARR" args = "{} {} {} {} {}:{:02d} -v --wx wx.csv --ffmc {} --dmc {} --dc {} --apcp_0800 {}".format(out_dir, start_date, lat, long, hour, minute, ffmc, dmc, dc, apcp_0800) if perim is not None: args = args + " --perim {}".format(perim) # run generated command for parsing data run_what = [cmd] + shlex.split(args.replace('\\', '/')) logging.info("Running: " + ' '.join(run_what)) t0 = timeit.default_timer() stdout, stderr = common.finish_process(common.start_process(run_what, "/FireGUARD/FireSTARR")) t1 = timeit.default_timer() logging.info("Took {}s to run simulations".format(t1 - t0)) log_name = os.path.join(out_dir, "log.txt") with open(log_name, 'w') as log_file: log_file.write(stdout.decode('utf-8')) outputs = sorted(os.listdir(out_dir)) extent = None probs = [x for x in outputs if x.endswith('asc') and x.startswith('wxshield')] if len(probs) > 0: prob = probs[-1] extent = firestarr_gis.project_raster(os.path.join(out_dir, prob), os.path.join(PROB_DIR, job_date, region, fire_name + '.tif')) perims = [x for x in outputs if x.endswith('tif')] if len(perims) > 0: perim = perims[0] firestarr_gis.project_raster(os.path.join(out_dir, perim), os.path.join(PERIM_DIR, job_date, region, fire_name + '.tif'), outputBounds=extent) else: return None return log_name
def ee_beamer_et(ini_path=None, overwrite_flag=False): """Earth Engine Beamer ET Image Download Args: ini_path (str): overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nEarth Engine Beamer Annual Mean ETg Image Download') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='SPATIAL') inputs.parse_section(ini, section='IMAGES') inputs.parse_section(ini, section='BEAMER') ini['IMAGES']['download_bands'] = [ 'etg_mean', 'etg_lci', 'etg_uci', 'etg_lpi', 'etg_upi' ] stat_list = ['mean', 'median'] nodata_value = -9999 zips_folder = 'zips' annuals_folder = 'annuals' # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # zone_count = len(zone_geom_list) # output_fmt = '_{0:0%sd}.csv' % str(int(math.log10(zone_count)) + 1) # Check if the zone_names are unique # Eventually support merging common zone_names if len(set([z[1] for z in zone_geom_list])) != len(zone_geom_list): logging.error( '\nERROR: There appear to be duplicate zone ID/name values.' '\n Currently, the values in "{}" must be unique.' '\n Exiting.'.format(ini['INPUTS']['zone_field'])) return False # Filter features by FID if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # Merge geometries if ini['INPUTS']['merge_geom_flag']: merge_geom = ogr.Geometry(ogr.wkbMultiPolygon) for zone in zone_geom_list: zone_multipolygon = ogr.ForceToMultiPolygon( ogr.CreateGeometryFromJson(json.dumps(zone[2]))) for zone_polygon in zone_multipolygon: merge_geom.AddGeometry(zone_polygon) # merge_json = json.loads(merge_mp.ExportToJson()) zone_geom_list = [[ 0, ini['INPUTS']['zone_filename'], json.loads(merge_geom.ExportToJson()) ]] ini['INPUTS']['zone_field'] = '' # Set all zone specific parameters into a dictionary zone = {} # Need zone_shp_path projection to build EE geometries zone['osr'] = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path']) zone['proj'] = gdc.osr_wkt(zone['osr']) # zone['proj'] = ee.Projection(zone['proj']).wkt().getInfo() # zone['proj'] = zone['proj'].replace('\n', '').replace(' ', '') # logging.debug(' Zone Projection: {}'.format(zone['proj'])) # Check that shapefile has matching spatial reference if not gdc.matching_spatref(zone['osr'], ini['SPATIAL']['osr']): logging.warning(' Zone OSR:\n{}\n'.format(zone['osr'])) logging.warning(' Output OSR:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.warning(' Zone Proj4: {}'.format( zone['osr'].ExportToProj4())) logging.warning(' Output Proj4: {}'.format( ini['SPATIAL']['osr'].ExportToProj4())) logging.warning( '\nWARNING: \n' 'The output and zone spatial references do not appear to match\n' 'This will likely cause problems!') input('Press ENTER to continue') else: logging.debug(' Zone Projection:\n{}\n'.format( zone['osr'].ExportToWkt())) logging.debug(' Output Projection:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.debug(' Output Cellsize: {}'.format( ini['SPATIAL']['cellsize'])) # Initialize Earth Engine API key logging.info('\nInitializing Earth Engine') ee.Initialize() utils.ee_request(ee.Number(1).getInfo()) # Get list of path/row strings to centroid coordinates if ini['INPUTS']['tile_keep_list']: ini['INPUTS']['tile_geom'] = [ wrs2.tile_centroids[tile] for tile in ini['INPUTS']['tile_keep_list'] if tile in wrs2.tile_centroids.keys() ] ini['INPUTS']['tile_geom'] = ee.Geometry.MultiPoint( ini['INPUTS']['tile_geom'], 'EPSG:4326') else: ini['INPUTS']['tile_geom'] = None # Read in ETo and PPT data from file if (ini['BEAMER']['eto_source'] == 'file' or ini['BEAMER']['ppt_source'] == 'file'): data_array = np.genfromtxt(ini['BEAMER']['data_path'], delimiter=',', names=True, dtype=None) data_fields = data_array.dtype.names logging.debug(' CSV fields: {}'.format(', '.join(data_fields))) # DEADBEEF - Compare fields names assuming all upper case data_fields = [f.upper() for f in data_fields] eto_dict = defaultdict(dict) ppt_dict = defaultdict(dict) for row in data_array: z = str(row[data_fields.index(ini['BEAMER']['data_zone_field'])]) y = int(row[data_fields.index(ini['BEAMER']['data_year_field'])]) if ini['BEAMER']['eto_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case eto_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_eto_field'].upper())] if ini['BEAMER']['ppt_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case ppt_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_ppt_field'].upper())] # Get filtered/merged/prepped Landsat collection landsat_args = { k: v for section in ['INPUTS'] for k, v in ini[section].items() if k in [ 'landsat4_flag', 'landsat5_flag', 'landsat7_flag', 'landsat8_flag', 'fmask_flag', 'acca_flag', 'start_year', 'end_year', 'start_month', 'end_month', 'start_doy', 'end_doy', 'scene_id_keep_list', 'scene_id_skip_list', 'path_keep_list', 'row_keep_list', 'tile_geom', 'adjust_method', 'mosaic_method', 'refl_sur_method' ] } landsat_args['products'] = ['evi_sur'] landsat = ee_common.Landsat(landsat_args) # Download images for each feature separately for zone_fid, zone_name, zone_json in zone_geom_list: zone['fid'] = zone_fid zone['name'] = zone_name.replace(' ', '_') zone['json'] = zone_json logging.info('ZONE: {} (FID: {})'.format(zone['name'], zone['fid'])) # Build EE geometry object for zonal stats zone['geom'] = ee.Geometry(geo_json=zone['json'], opt_proj=zone['proj'], opt_geodesic=False) # logging.debug(' Centroid: {}'.format( # zone['geom'].centroid(100).getInfo()['coordinates'])) # Use feature geometry to build extent, transform, and shape zone['extent'] = gdc.Extent( ogr.CreateGeometryFromJson(json.dumps(zone['json'])).GetEnvelope()) # zone['extent'] = gdc.Extent(zone['geom'].GetEnvelope()) zone['extent'] = zone['extent'].ogrenv_swap() zone['extent'] = zone['extent'].adjust_to_snap( 'EXPAND', ini['SPATIAL']['snap_x'], ini['SPATIAL']['snap_y'], ini['SPATIAL']['cellsize']) zone['geo'] = zone['extent'].geo(ini['SPATIAL']['cellsize']) zone['transform'] = gdc.geo_2_ee_transform(zone['geo']) # zone['transform'] = '[' + ','.join(map(str, zone['transform'])) + ']' zone['shape'] = zone['extent'].shape(ini['SPATIAL']['cellsize']) logging.debug(' Zone Shape: {}'.format(zone['shape'])) logging.debug(' Zone Transform: {}'.format(zone['transform'])) logging.debug(' Zone Extent: {}'.format(zone['extent'])) # logging.debug(' Zone Geom: {}'.format(zone['geom'].getInfo())) # Assume all pixels in all 14+2 images could be reduced zone['max_pixels'] = zone['shape'][0] * zone['shape'][1] logging.debug(' Max Pixels: {}'.format(zone['max_pixels'])) # Set output spatial reference # Eventually allow user to manually set these # output_crs = zone['proj'] logging.debug(' Image Projection: {}'.format(ini['SPATIAL']['crs'])) # output_transform = zone['transform'][:] output_transform = '[' + ','.join(map(str, zone['transform'])) + ']' output_shape = '{1}x{0}'.format(*zone['shape']) logging.debug(' Image Transform: {}'.format(output_transform)) logging.debug(' Image Shape: {}'.format(output_shape)) zone_output_ws = os.path.join(ini['IMAGES']['output_ws'], zone_name) zone_zips_ws = os.path.join(zone_output_ws, zips_folder) zone_annuals_ws = os.path.join(zone_output_ws, annuals_folder) if not os.path.isdir(zone_zips_ws): os.makedirs(zone_zips_ws) if not os.path.isdir(zone_annuals_ws): os.makedirs(zone_annuals_ws) # Process date range by year interval_cnt = 1 start_dt = datetime.datetime(ini['INPUTS']['start_year'], 1, 1) end_dt = datetime.datetime(ini['INPUTS']['end_year'] + 1, 1, 1) - datetime.timedelta(0, 1) for i, iter_start_dt in enumerate( rrule.rrule(rrule.YEARLY, interval=interval_cnt, dtstart=start_dt, until=end_dt)): iter_end_dt = (iter_start_dt + relativedelta.relativedelta(years=interval_cnt) - datetime.timedelta(0, 1)) if ((ini['INPUTS']['start_month'] and iter_end_dt.month < ini['INPUTS']['start_month']) or (ini['INPUTS']['end_month'] and iter_start_dt.month > ini['INPUTS']['end_month'])): logging.debug(' {} {} skipping'.format( iter_start_dt.date(), iter_end_dt.date())) continue elif ( (ini['INPUTS']['start_doy'] and int(iter_end_dt.strftime('%j')) < ini['INPUTS']['start_doy']) or (ini['INPUTS']['end_doy'] and int(iter_start_dt.strftime('%j')) > ini['INPUTS']['end_doy'])): logging.debug(' {} {} skipping'.format( iter_start_dt.date(), iter_end_dt.date())) continue else: logging.info('{} {}'.format(iter_start_dt.date(), iter_end_dt.date())) year = iter_start_dt.year # image_id = 'etg_{}_{}'.format( image_id = '{}_{}'.format(zone_name.lower().replace(' ', '_'), year) zip_path = os.path.join(zone_zips_ws, image_id + '.zip') # median_path = os.path.join( # zone_output_ws, image_id + '.img') logging.debug(' Zip: {}'.format(zip_path)) if os.path.isfile(zip_path) and overwrite_flag: logging.debug(' Output already exists, removing zip') os.remove(zip_path) elif os.path.isfile(zip_path) and not overwrite_flag: # Check that existing ZIP files can be opened try: with zipfile.ZipFile(zip_path, 'r') as z: pass except Exception as e: logging.warning(' Zip file error, removing'.format(i)) os.remove(zip_path) # Filter the GRIDMET collection wy_start_date = '{}-10-01'.format(year - 1) wy_end_date = '{}-10-01'.format(year) logging.debug(' WY: {} {}'.format(wy_start_date, wy_end_date)) gridmet_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \ .filterDate(wy_start_date, wy_end_date) # # PRISM collection was uploaded as an asset # if ini['BEAMER']['ppt_source'] == 'prism': # def prism_time_start(input_image): # """Set time_start property on PRISM water year PPT collection""" # # Assume year is the 4th item separated by "_" # water_year = ee.String(input_image.get('system:index')).split('_').get(3) # date_start = ee.Date(ee.String(water_year).cat('-10-01')) # return input_image.select([0], ['ppt']).set({ # 'system:time_start': date_start.millis() # }) # prism_coll = ee.ImageCollection('users/cgmorton/prism_800m_ppt_wy') # prism_coll = prism_coll.map(prism_time_start) \ # .filterDate(wy_start_date, wy_end_date) # Get water year PPT from file # Convert all input data to mm to match GRIDMET data if ini['BEAMER']['ppt_source'] == 'file': wy_ppt_input = ppt_dict[zone_name][year] if ini['BEAMER']['data_ppt_units'] == 'mm': pass elif ini['BEAMER']['data_ppt_units'] == 'in': wy_ppt_input *= 25.4 elif ini['BEAMER']['data_ppt_units'] == 'ft': wy_ppt_input *= (25.4 * 12) elif ini['BEAMER']['ppt_source'] == 'gridmet': # GET GRIDMET value at centroid of geometry wy_ppt_input = float( utils.ee_getinfo( ee.ImageCollection( gridmet_coll.select(['pr'], ['ppt']).sum()).getRegion( zone['geom'].centroid(1), 500))[1][4]) # Calculate GRIDMET zonal mean of geometry # wy_ppt_input = float(ee.ImageCollection( # gridmet_coll.select(['pr'], ['ppt'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo()['ppt'] # elif ini['BEAMER']['ppt_source'] == 'prism': # # Calculate PRISM zonal mean of geometry # wy_ppt_input = float(ee.ImageCollection( # prism_coll.map(ee_common.prism_ppt_func)).sum().reduceRegion( # reducer=ee.Reducer.mean(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo()['ppt']) # Get water year ETo read from file # Convert all input data to mm for Beamer Method if ini['BEAMER']['eto_source'] == 'file': wy_eto_input = eto_dict[zone_name][year] if ini['BEAMER']['data_eto_units'] == 'mm': pass elif ini['BEAMER']['data_eto_units'] == 'in': wy_eto_input *= 25.4 elif ini['BEAMER']['data_eto_units'] == 'ft': wy_eto_input *= (25.4 * 12) # This assumes GRIMET data is in millimeters elif ini['BEAMER']['eto_source'] == 'gridmet': wy_eto_input = float( utils.ee_getinfo( ee.ImageCollection(gridmet_coll.select( ['eto']).sum()).getRegion(zone['geom'].centroid(1), 500))[1][4]) # wy_eto_input = float(ee.ImageCollection( # gridmet_coll.select(['eto'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo() logging.debug(' Input ETO: {} mm PPT: {} mm'.format( wy_eto_input, wy_ppt_input)) # Scale ETo & PPT wy_eto_input *= ini['BEAMER']['eto_factor'] wy_ppt_input *= ini['BEAMER']['ppt_factor'] # Convert output units from mm wy_ppt_output = wy_ppt_input wy_eto_output = wy_eto_input if ini['IMAGES']['ppt_units'] == 'mm': pass elif ini['IMAGES']['ppt_units'] == 'in': wy_ppt_output /= 25.4 elif ini['IMAGES']['ppt_units'] == 'ft': wy_ppt_output /= (25.4 * 12) if ini['IMAGES']['eto_units'] == 'mm': pass elif ini['IMAGES']['eto_units'] == 'in': wy_eto_output /= 25.4 elif ini['IMAGES']['eto_units'] == 'ft': wy_eto_output /= (25.4 * 12) logging.debug(' Output ETO: {} {} PPT: {} {}'.format( wy_eto_output, ini['IMAGES']['eto_units'], wy_ppt_output, ini['IMAGES']['ppt_units'])) # Initialize the Landsat object for target zone and iteration landsat.zone_geom = zone['geom'] landsat.start_date = iter_start_dt.strftime('%Y-%m-%d') landsat.end_date = iter_end_dt.strftime('%Y-%m-%d') landsat_coll = landsat.get_collection() # print(sorted(utils.ee_getinfo( # landsat_coll.aggregate_histogram('SCENE_ID')))) # input('ENTER') # Skip if Landsat collection is empty if not utils.ee_getinfo( landsat_coll.aggregate_histogram('SCENE_ID')): logging.info(' Empty Landsat collection, skipping') continue # Add water year ETo and PPT values to each image def eto_ppt_func(img): """""" return ee.Image(img).setMulti({ 'wy_eto': wy_eto_output, 'wy_ppt': wy_ppt_output }) landsat_coll = ee.ImageCollection(landsat_coll.map(eto_ppt_func)) # Build each collection separately then merge etg_coll = ee.ImageCollection(landsat_coll.map( ee_common.beamer_func)) \ .select(ini['IMAGES']['download_bands']) # Clip using the feature geometry # Set the masked values to a nodata value # so that the TIF can have a nodata value other than 0 set etg_image = ee.Image(etg_coll.mean()) \ .clip(zone['geom']) \ .unmask(nodata_value, False) if not os.path.isfile(zip_path): # Get the download URL logging.debug(' Requesting URL') zip_url = utils.ee_request( etg_image.getDownloadURL({ 'name': image_id, 'crs': ini['SPATIAL']['crs'], 'crs_transform': output_transform, 'dimensions': output_shape })) # Try downloading a few times logging.info(' Downloading') for i in range(1, 10): try: response = urlrequest.urlopen(zip_url) with open(zip_path, 'wb') as output_f: shutil.copyfileobj(response, output_f) break except Exception as e: logging.info(' Resending query') logging.debug(' {}'.format(e)) sleep(i**2) os.remove(zip_path) # Try extracting the files try: logging.info(' Extracting') with zipfile.ZipFile(zip_path, 'r') as z: z.extractall(zone_annuals_ws) except Exception as e: logging.warning(' Error: could not extract'.format(i)) logging.debug(' {}'.format(e)) try: os.remove(zip_path) except Exception as e: pass # Set nodata value for item in os.listdir(zone_annuals_ws): if item.startswith(image_id) and item.endswith('.tif'): gdc.raster_path_set_nodata( os.path.join(zone_annuals_ws, item), nodata_value) raster_statistics(os.path.join(zone_annuals_ws, item)) logging.info('\nComputing composite rasters from annual means') for stat in stat_list: logging.info(' Stat: {}'.format(stat)) for band in ini['IMAGES']['download_bands']: logging.info(' {}'.format(band)) image_band_list = [ os.path.join(zone_annuals_ws, item) for item in os.listdir(zone_annuals_ws) if item.endswith('.{}.tif'.format(band.lower())) ] # for image_path in image_band_list: # raster_path_set_nodata(image_path, nodata_value) output_path = os.path.join( # zone_output_ws, 'etg_{}_{}.{}.tif'.format( zone_output_ws, '{}_{}.{}.tif'.format(zone_name.lower().replace(' ', '_'), stat.lower(), band.lower())) logging.debug(' {}'.format(output_path)) # Use GDAL to compute the composite raster cell_statistics(image_band_list, output_path, stat.lower()) raster_statistics(output_path)
def get_lat_index(location): """Get latitude index min and max.""" geom = ogr.CreateGeometryFromJson(json.dumps(location)) env = geom.GetEnvelope() return int(math.floor(env[2] * 10)), int(math.ceil(env[3] * 10))
def _to_ogr(source, srid=None): geom = None if isinstance(source, dict): # GeoJSON if not 'coordinates' in source: if 'features' in source: raise ValueError( f'FeatureCollection {repr(source)} cannot be converted to a single OGR Geometry' ) if 'geometry' in source: source = source['geometry'] else: raise ValueError( f'{repr(source)} doesn\'t look like a valid GeoJSON') geom_str = json.dumps(source) geom = ogr.CreateGeometryFromJson(geom_str) if geom is None: raise ValueError( f'Failed to convert {repr(source)} to OGR Geometry') elif isinstance(source, str): try: jsn = json.loads(source) except json.JSONDecodeError: pass else: # GeoJSON string return OgrWrapper._to_ogr(jsn, srid) wkt = source m = re.match(r'\s*SRID=(\d*)\s*;(.*)$', wkt) # Extended WKT if m: srid = int(m.group(1)) wkt = m.group(2) # Try WKT geom = ogr.CreateGeometryFromWkt(wkt) if geom is None: geom = ogr.CreateGeometryFromGML(source) if geom is None: raise ValueError( f'{repr(source)} doesn\'t look like neither of GeoJSON, WKT nor GML' ) elif isinstance(source, ET.Element): xml = ET.tostring(source, encoding='utf-8').decode() try: return OgrWrapper._to_ogr(xml) except ValueError: raise ValueError( f'{repr(source)} is not representing a valid GML') else: raise TypeError( f'{repr(source)} cannot be converted to OGR Geometry') if srid is not None: OgrWrapper._set_srs(geom, srid) return geom
def GeoJSONtoQgsGeomentry(_json: Union[dict, str]) -> QgsGeometry: if isinstance(_json, dict): _json = json.dumps(_json) ret = ogr.CreateGeometryFromJson(_json) return QgsGeometry.fromWkt(ret.ExportToWkt())
def _save(self, tweet, hashtag=None): # Obtain the event from the database. event = self.__session.query(DBEONet).filter(DBEONet.hashtag == hashtag).one_or_none() # No need to continue if the tweet is already in the database. db_tweet = self.__session.query(DBTweets).filter(DBTweets.tweet_id == tweet.id_str).one_or_none() if db_tweet is not None: return # the data fields to save. name = tweet.user.name create_date = tweet.created_at place = tweet.place.name if tweet.place else None text = tweet.text profile_pic = tweet.user.profile_image_url screen_name = tweet.user.screen_name try: tweet.extended_entities except AttributeError: media_url = None else: media_url = tweet.extended_entities['media'][0]['media_url'] if tweet.extended_entities['media'] and tweet.extended_entities['media'][0]['type'] == 'photo' else None # The coordinates of the tweet are either the specific long, lat # provided in the tweet, or the centroid of the place's bounding # box. # # The coordinates is a list that must be saved as a string. However # the list needs to be reconstituted when it's read. The clients # consuming the REST API need the JSON version of the coordinates. if tweet.coordinates is not None: coordinates = PythonDBObject(tweet.coordinates) json_coordinates = json.dumps(tweet.coordinates) # Second choice is the centroid of the place elif tweet.place is not None: center = place_centroid(tweet) # Build the dictionary. l_coords = [center.GetX(), center.GetY()] d_coords = {u'type': 'Point', u'coordinates': l_coords} coordinates = PythonDBObject(d_coords) json_coordinates = json.dumps(d_coords) # Use the centroid of the event if we have that. elif hashtag is not None: e_json = event.json_geometries f_json = str(e_json) g_json = f_json[1:-1] ogr_geometry = ogr.CreateGeometryFromJson(g_json) geotype = ogr_geometry.GetGeometryName if 'Point' == geotype: coordinates = PythonDBObject(g_json) json_coordinates =g_json else: center = ogr_geometry.Centroid() # Build the dictionary. l_coords = [center.GetX(), center.GetY()] d_coords = {u'type': 'Point', u'coordinates': l_coords} coordinates = PythonDBObject(d_coords) json_coordinates = json.dumps(d_coords) # Commit to the database. tweet_record = DBTweets(tweet_id=tweet.id_str, eonet_id=event.eonet_id, name=name, createDate=create_date, place=place, msg=text, coordinates=coordinates.encode(), hashtag=hashtag, media_url=media_url, screen_name=screen_name, profile_pic=profile_pic, json_coordinates=json_coordinates) self.__session.add(tweet_record) self.__session.commit() # Write the same to standard out to monitor the progress. The terminal # requires the text to be encoded as utf8. text = text.encode('utf8', 'replace') name = name.encode('utf8', 'replace') place = place.encode('utf8', 'replace') if place else None media_url = media_url.encode('utf8', 'replace') if media_url else None print('Written to the database: [Name: %s][Place: %s][At: %s] %s %s.' % (name, place, create_date, text, media_url))
def pintarCapasReferencia(self, uriFigura, value, posicion): consulta = self.root.findGroup('referencia') if consulta == None: raise AttributeError('Capas no detectadas') return None #Nombre de la capa de acuerdo al valor del ComboBox de capas a cargar nameCapa = value egName = list(self.capasReferencia.keys())[list( self.capasReferencia.values()).index(value)] #Obtenemos los datos data = self.obtenerCapasDeReferencia(egName) type(data) srid = 32614 inSpatialRef = osr.SpatialReference() inSpatialRef.ImportFromEPSG(int(srid)) outSpatialRef = osr.SpatialReference() outSpatialRef.ImportFromEPSG(int(srid)) coordTrans = osr.CoordinateTransformation(inSpatialRef, outSpatialRef) if not bool(data): raise Exception('Error') keys = list(data['features'][0]['properties'].keys()) properties = [] geoms = [] for feature in data['features']: geom = feature['geometry'] property = feature['properties'] geom = json.dumps(geom) geometry = ogr.CreateGeometryFromJson(geom) geometry.Transform(coordTrans) geoms.append(geometry.ExportToWkt()) l = [] for i in range(0, len(keys)): l.append(property[keys[i]]) properties.append(l) fields = "" for k in keys: fields = fields + "&field=" + k + ":string(15)" uri = str(uriFigura) + "?crs=epsg:" + str(srid) + fields + "&index=yes" mem_layer = QgsVectorLayer(uri, nameCapa, 'memory') mem_layer.setReadOnly(True) prov = mem_layer.dataProvider() feats = [QgsFeature() for i in range(len(geoms))] for i, feat in enumerate(feats): feat.setAttributes(properties[i]) feat.setGeometry(QgsGeometry.fromWkt(geoms[i])) prov.addFeatures(feats) settings = QgsPalLayerSettings() settings.fieldName = 'clave' settings.centroidWhole = True textFormat = QgsTextFormat() textFormat.setColor(QColor(0, 0, 0)) textFormat.setSize(8) textFormat.setNamedStyle('Bold') settings.setFormat(textFormat) labeling = QgsVectorLayerSimpleLabeling(settings) mem_layer.setLabeling(labeling) mem_layer.setLabelsEnabled(True) QgsProject.instance().addMapLayers([mem_layer], False) mzaNL = QgsLayerTreeLayer(mem_layer) consulta.insertChildNode(0, mzaNL)
def pintarVolumenConstrucciones(self): mem_layer = QgsProject.instance().mapLayersByName('volumenes') if len(mem_layer) != 1: raise AttributeError mem_layer = mem_layer[0] if mem_layer == None: return data = self.obtenerVolConstrucciones() type(data) srid = 32614 inSpatialRef = osr.SpatialReference() inSpatialRef.ImportFromEPSG(int(srid)) outSpatialRef = osr.SpatialReference() outSpatialRef.ImportFromEPSG(int(srid)) coordTrans = osr.CoordinateTransformation(inSpatialRef, outSpatialRef) if not bool(data): raise Exception('Error') if data['features'] == []: return keys = list(data['features'][0]['properties'].keys()) properties = [] geoms = [] for feature in data['features']: geom = feature['geometry'] property = feature['properties'] geom = json.dumps(geom) geometry = ogr.CreateGeometryFromJson(geom) geometry.Transform(coordTrans) geoms.append(geometry.ExportToWkt()) l = [] for i in range(0, len(keys)): l.append(property[keys[i]]) properties.append(l) prov = mem_layer.dataProvider() feats = [QgsFeature() for i in range(len(geoms))] for i, feat in enumerate(feats): feat.setAttributes(properties[i]) feat.setGeometry(QgsGeometry.fromWkt(geoms[i])) prov.addFeatures(feats) settings = QgsPalLayerSettings() settings.fieldName = 'clave' settings.centroidWhole = True textFormat = QgsTextFormat() textFormat.setColor(QColor(0, 0, 255)) textFormat.setSize(8) textFormat.setNamedStyle('Bold') settings.setFormat(textFormat) labeling = QgsVectorLayerSimpleLabeling(settings) mem_layer.setLabeling(labeling) mem_layer.setLabelsEnabled(True) mem_layer.triggerRepaint() mem_layer.commitChanges() mem_layer.triggerRepaint()
def init_xy_with_subsets(self, lon_min, lat_min, lon_max, lat_max, target_cellsize_meters, subset_grid_shp, subset_grid_field_name=None): """Create & initialize x/y dimensions/coordinate vars and subset vars. Args: lon_min: Minimum longitude of domain. lat_min: Minimum latitude of domain. lon_max: Maximum longitude of domain. lat_max: Maximum latitude of domain. target_cellsize_meters: Target cell size, in meters. Actual calculated cell sizes will be approximations of this. subset_grid_shp: Path to subset grid polygon shapefile used to define subgrid domains. subset_grid_field_name: Optional, default None) Shapefile field name to be stored in the index file. Raises: Exception when given subset grid shapefile does not exist or does not include any grid polygons intersecting with given extent. Returns: Instance of `RegularGrid` representing the extended generated grid whose extent matches the union of all intersecting subset grid polygons. """ shp = ogr.Open(subset_grid_shp) layer = shp.GetLayer() # Create OGR Geometry from ocean model grid extent ring = ogr.Geometry(ogr.wkbLinearRing) ring.AddPoint(lon_min, lat_max) ring.AddPoint(lon_min, lat_min) ring.AddPoint(lon_max, lat_min) ring.AddPoint(lon_max, lat_max) ring.AddPoint(lon_min, lat_max) # Create polygon ofs_poly = ogr.Geometry(ogr.wkbPolygon) ofs_poly.AddGeometry(ring) # Get the EPSG value from the import shapefile and transform to WGS84 spatial_ref = layer.GetSpatialRef() shp_srs = spatial_ref.GetAttrValue('AUTHORITY', 1) source = osr.SpatialReference() source.ImportFromEPSG(int(shp_srs)) target = osr.SpatialReference() target.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source, target) ofs_poly.Transform(transform) # Find the intersection between grid polygon and ocean model grid extent subset_polys = {} fids = [] fields = {} fid = 0 for feature in layer: geom = feature.GetGeometryRef() if ofs_poly.Intersects(geom): subset_polys[fid] = geom.ExportToJson() if subset_grid_field_name is not None: field_name = feature.GetField(str(subset_grid_field_name)) fields.update({fid: field_name}) fids.append(fid) else: fids.append(fid) fid += 1 if len(fids) == 0: raise Exception( 'Given subset grid shapefile contains no polygons that intersect with model domain; cannot proceed.' ) # Use a single subset polygon to calculate x/y cell sizes. This ensures # that cells do not fall on the border between two grid polygons. single_polygon = ogr.Geometry(ogr.wkbMultiPolygon) single_polygon.AddGeometry( ogr.CreateGeometryFromJson(subset_polys[fids[0]])) sp_x_min, sp_x_max, sp_y_min, sp_y_max = single_polygon.GetEnvelope() cellsize_x, cellsize_y = RegularGrid.calc_cellsizes( sp_x_min, sp_y_min, sp_x_max, sp_y_max, target_cellsize_meters) # Combine identified subset grid polygons into single multipolygon to # calculate full extent of all combined subset grids multipolygon = ogr.Geometry(ogr.wkbMultiPolygon) for fid in fids: multipolygon.AddGeometry( ogr.CreateGeometryFromJson(subset_polys[fid])) (x_min, x_max, y_min, y_max) = multipolygon.GetEnvelope() full_reg_grid = RegularGrid(x_min, y_min, x_max, y_max, cellsize_x, cellsize_y) # Create NetCDF dimensions & coordinate variables using dimension sizes # from regular grid self.create_dims_coord_vars(len(full_reg_grid.y_coords), len(full_reg_grid.x_coords)) # Populate NetCDF coordinate variables using regular grid coordinates self.var_x[:] = full_reg_grid.x_coords[:] self.var_y[:] = full_reg_grid.y_coords[:] self.nc_file.gridSpacingLongitude = full_reg_grid.cellsize_x self.nc_file.gridSpacingLatitude = full_reg_grid.cellsize_y # Create subgrid dimension/variables self.create_subgrid_dims_vars(len(subset_polys), subset_grid_field_name) # Calculate subgrid mask ranges, populate subgrid ID for subgrid_index, fid in enumerate(fids): self.var_subgrid_id[subgrid_index] = fid if subset_grid_field_name is not None: self.var_subgrid_name[subgrid_index] = fields[fid] # Convert OGR geometry to shapely geometry subset_poly_shape = shape(json.loads(subset_polys[fid])) min_x_coord = subset_poly_shape.bounds[0] max_x_coord = subset_poly_shape.bounds[2] min_y_coord = subset_poly_shape.bounds[1] max_y_coord = subset_poly_shape.bounds[3] subgrid_x_min = None subgrid_x_max = None subgrid_y_min = None subgrid_y_max = None for i, x in enumerate(self.var_x): if x >= min_x_coord: subgrid_x_min = i break count_x = round( ((max_x_coord - min_x_coord) / full_reg_grid.cellsize_x)) for i, y in enumerate(self.var_y): if y >= min_y_coord: subgrid_y_min = i break count_y = round( ((max_y_coord - min_y_coord) / full_reg_grid.cellsize_y)) subgrid_x_max = subgrid_x_min + count_x - 1 subgrid_y_max = subgrid_y_min + count_y - 1 self.var_subgrid_x_min[subgrid_index] = subgrid_x_min self.var_subgrid_x_max[subgrid_index] = subgrid_x_max self.var_subgrid_y_min[subgrid_index] = subgrid_y_min self.var_subgrid_y_max[subgrid_index] = subgrid_y_max return full_reg_grid
def run_crophealth_app(ds, lat, lon, buffer): """ Plots an interactive map of the crop health case-study area and allows the user to draw polygons. This returns a plot of the average NDVI value in the polygon area. Last modified: January 2020 Parameters ---------- ds: xarray.Dataset data set containing combined, masked data Masked values are set to 'nan' lat: float The central latitude corresponding to the area of loaded ds lon: float The central longitude corresponding to the area of loaded ds buffer: The number of square degrees to load around the central latitude and longitude. For reasonable loading times, set this as `0.1` or lower. """ # Suppress warnings warnings.filterwarnings('ignore') # Update plotting functionality through rcParams mpl.rcParams.update({'figure.autolayout': True}) # Define polygon bounds latitude = (lat - buffer, lat + buffer) longitude = (lon - buffer, lon + buffer) # Define the bounding box that will be overlayed on the interactive map # The bounds are hard-coded to match those from the loaded data geom_obj = { "type": "Feature", "properties": { "style": { "stroke": True, "color": 'red', "weight": 4, "opacity": 0.8, "fill": True, "fillColor": False, "fillOpacity": 0, "showArea": True, "clickable": True } }, "geometry": { "type": "Polygon", "coordinates": [ [ [ longitude[0], latitude[0] ], [ longitude[1], latitude[0] ], [ longitude[1], latitude[1] ], [ longitude[0], latitude[1] ], [ longitude[0], latitude[0] ] ] ] } } # Create a map geometry from the geom_obj dictionary # center specifies where the background map view should focus on # zoom specifies how zoomed in the background map should be loadeddata_geometry = ogr.CreateGeometryFromJson(str(geom_obj['geometry'])) loadeddata_center = [ loadeddata_geometry.Centroid().GetY(), loadeddata_geometry.Centroid().GetX() ] loadeddata_zoom = 16 # define the study area map studyarea_map = Map( center=loadeddata_center, zoom=loadeddata_zoom, basemap=basemaps.Esri.WorldImagery ) # define the drawing controls studyarea_drawctrl = DrawControl( polygon={"shapeOptions": {"fillOpacity": 0}}, marker={}, circle={}, circlemarker={}, polyline={}, ) # add drawing controls and data bound geometry to the map studyarea_map.add_control(studyarea_drawctrl) studyarea_map.add_layer(GeoJSON(data=geom_obj)) # Index to count drawn polygons polygon_number = 0 # Define widgets to interact with instruction = widgets.Output(layout={'border': '1px solid black'}) with instruction: print("Draw a polygon within the red box to view a plot of " "average NDVI over time in that area.") info = widgets.Output(layout={'border': '1px solid black'}) with info: print("Plot status:") fig_display = widgets.Output(layout=widgets.Layout( width="50%", # proportion of horizontal space taken by plot )) with fig_display: plt.ioff() fig, ax = plt.subplots(figsize=(8, 6)) ax.set_ylim([0, 1]) colour_list = plt.rcParams['axes.prop_cycle'].by_key()['color'] # Function to execute each time something is drawn on the map def handle_draw(self, action, geo_json): nonlocal polygon_number # Execute behaviour based on what the user draws if geo_json['geometry']['type'] == 'Polygon': info.clear_output(wait=True) # wait=True reduces flicker effect # Save geojson polygon to io temporary file to be rasterized later jsonData = json.dumps(geo_json) binaryData = jsonData.encode() io = BytesIO(binaryData) io.seek(0) # Read the polygon as a geopandas dataframe gdf = gpd.read_file(io) gdf.crs = "EPSG:4326" # Convert the drawn geometry to pixel coordinates xr_poly = xr_rasterize(gdf, ds.NDVI.isel(time=0), crs='EPSG:6933') # Construct a mask to only select pixels within the drawn polygon masked_ds = ds.NDVI.where(xr_poly) masked_ds_mean = masked_ds.mean(dim=['x', 'y'], skipna=True) colour = colour_list[polygon_number % len(colour_list)] # Add a layer to the map to make the most recently drawn polygon # the same colour as the line on the plot studyarea_map.add_layer( GeoJSON( data=geo_json, style={ 'color': colour, 'opacity': 1, 'weight': 4.5, 'fillOpacity': 0.0 } ) ) # add new data to the plot xr.plot.plot( masked_ds_mean, marker='*', color=colour, ax=ax ) # reset titles back to custom ax.set_title("Average NDVI from Landsat 8") ax.set_xlabel("Date") ax.set_ylabel("NDVI") # refresh display fig_display.clear_output(wait=True) # wait=True reduces flicker effect with fig_display: display(fig) with info: print("Plot status: polygon sucessfully added to plot.") # Iterate the polygon number before drawing another polygon polygon_number = polygon_number + 1 else: info.clear_output(wait=True) with info: print("Plot status: this drawing tool is not currently " "supported. Please use the polygon tool.") # call to say activate handle_draw function on draw studyarea_drawctrl.on_draw(handle_draw) with fig_display: # TODO: update with user friendly something display(widgets.HTML("")) # Construct UI: # +-----------------------+ # | instruction | # +-----------+-----------+ # | map | plot | # | | | # +-----------+-----------+ # | info | # +-----------------------+ ui = widgets.VBox([instruction, widgets.HBox([studyarea_map, fig_display]), info]) display(ui)
def export_to_shp(jobs, file_name, output_folder): """Exports results to a shapefile. :param jobs: list of jobs (a job contains the result information) :param file_name: the output file name :param output_folder: the output task folder """ global exported_count global errors_count from osgeo import ogr from osgeo import osr # os.environ['GDAL_DATA'] = r'C:\voyager\server_2-1381\app\gdal' driver = ogr.GetDriverByName("ESRI Shapefile") for job in jobs: try: geometry_type = None if '[geo]' not in job: errors_count += 1 status_writer.send_state(status.STAT_WARNING, 'No Geometry field') status_writer.send_state(status.STAT_WARNING) continue geo_json = job['[geo]'] if geo_json['type'].lower() == 'polygon': geometry_type = ogr.wkbPolygon elif geo_json['type'].lower() == 'geometrycollection': geom = ogr.CreateGeometryFromJson("{0}".format(job['[geo]'])) if geom.GetDimension() == 0: geometry_type = ogr.wkbPoint elif geom.GetDimension() == 1: geometry_type = ogr.wkbLineString else: geometry_type = ogr.wkbPolygon elif geo_json['type'].lower() == 'multipolygon': geometry_type = ogr.wkbMultiPolygon elif geo_json['type'].lower() == 'linestring': geometry_type = ogr.wkbLineString elif geo_json['type'].lower() == 'multilinestring': geometry_type = ogr.wkbMultiLineString elif geo_json['type'].lower() == 'point': geometry_type = ogr.wkbPoint elif geo_json['type'].lower() == 'multipoint': geometry_type = ogr.wkbMultiPoint except KeyError as ke: errors_count += 1 status_writer.send_state(status.STAT_WARNING, 'No Geometry field') continue except TypeError as te: errors_count += 1 status_writer.send_state(status.STAT_WARNING, 'No Geometry field') status_writer.send_state(status.STAT_WARNING) continue if os.path.exists( os.path.join(output_folder, '{0}_{1}.shp'.format(file_name, geo_json['type']))): shape_file = ogr.Open( os.path.join(output_folder, '{0}_{1}.shp'.format(file_name, geo_json['type'])), 1) layer = shape_file.GetLayer() else: shape_file = driver.CreateDataSource( os.path.join(output_folder, '{0}_{1}.shp'.format(file_name, geo_json['type']))) epsg_code = 4326 srs = osr.SpatialReference() srs.ImportFromEPSG(epsg_code) layer = shape_file.CreateLayer( '{0}_{1}'.format(file_name, geo_json['type']), srs, geometry_type) for name in jobs[0].keys(): if not name == '[geo]': name = str(name) if name.startswith('fu_'): new_field = ogr.FieldDefn(name, ogr.OFTReal) elif name.startswith('fi_'): new_field = ogr.FieldDefn(name, ogr.OFTInteger) elif name.startswith('fl_'): new_field = ogr.FieldDefn(name, ogr.OFTInteger64) elif name.startswith('fd_'): new_field = ogr.FieldDefn(name, ogr.OFTDateTime) else: new_field = ogr.FieldDefn(name, ogr.OFTString) layer.CreateField(new_field) try: layer_def = layer.GetLayerDefn() feature = ogr.Feature(layer_def) geom = ogr.CreateGeometryFromJson("{0}".format(job['[geo]'])) if not geom: geom = ogr.CreateGeometryFromJson("{0}".format( json.dumps(job['[geo]']))) feature.SetGeometry(geom) except KeyError: feature.SetGeometry(None) pass try: job.pop('[geo]') except KeyError: pass try: for field, value in job.items(): field, value = str(field), str(value) i = feature.GetFieldIndex(field[0:10]) feature.SetField(i, value) layer.CreateFeature(feature) exported_count += 1 shape_file.Destroy() except Exception as ex: errors_count += 1 status_writer.send_state(status.STAT_WARNING, 'No Geometry field') shape_file.Destroy() continue
from osgeo import ogr from osgeo import osr source = osr.SpatialReference() source.ImportFromEPSG(4326) target = osr.SpatialReference() target.ImportFromEPSG(3857) transform = osr.CoordinateTransformation(source, target) geojson = """{ "type": "LineString", "coordinates": [ [ -75.313585, 43.069271 ], [ -75.269426, 43.114027 ], [ -75.158731, 43.114325 ] ] }""" geom = ogr.CreateGeometryFromJson(geojson) geom.Transform(transform) print "Length = %d" % geom.Length() + " meters"
def splitAtDateline(geom, preferredEpsg): """ Given a Polygon Geometry object in lat/long, determine whether it crosses the date line, and if so, split it into a multipolygon with a part on either side. Use the given preferred EPSG to perform calculations. Return a new Geometry in lat/long. """ crosses = crossesDateline(geom, preferredEpsg) if crosses: (projTr, llTr) = makeTransformations(4326, preferredEpsg) coords = getCoords(geom) (x, y) = (coords[:, 0], coords[:, 1]) (yMin, yMax) = (y.min(), y.max()) xMinPositive = None xMaxNegative = None xGe0 = (x >= 0) xLt0 = (x < 0) if xGe0.any() > 0 and xLt0.any() > 0: xMaxNegative = x[xLt0].max() xMinPositive = x[xGe0].min() # Create rectangles for the east and west hemispheres, constrained by the # extent of this polygon. Note that this assumes that we do not # cross both the date line, and also the prime (zero) meridian. This may not # always be true, notably when we are close to the pole. eastHemiRectCoords = [[xMinPositive, yMax], [xMinPositive, yMin], [180, yMin], [180, yMax], [xMinPositive, yMax]] eastHemiRectJson = repr({ 'type': 'Polygon', 'coordinates': [eastHemiRectCoords] }) westHemiRectCoords = [[-180, yMax], [-180, yMin], [xMaxNegative, yMin], [xMaxNegative, yMax], [-180, yMax]] westHemiRectJson = repr({ 'type': 'Polygon', 'coordinates': [westHemiRectCoords] }) eastHemiRect = ogr.CreateGeometryFromJson(eastHemiRectJson) westHemiRect = ogr.CreateGeometryFromJson(westHemiRectJson) geomProj = copyGeom(geom) geomProj.Transform(projTr) eastHemiRect.Transform(projTr) westHemiRect.Transform(projTr) eastHemiPart = geomProj.Intersection(eastHemiRect) westHemiPart = geomProj.Intersection(westHemiRect) eastHemiPart.Transform(llTr) westHemiPart.Transform(llTr) # Put these together as a single multipolygon eastPartCoords = getCoords(eastHemiPart) westPartCoords = getCoords(westHemiPart) # Discard any vertices which are still no the wrong side of the 180 line. I # do not understand what is going on here, but I have invested far more of # my valuable time than I should, and this kludge will be a reasonable approximation. eastPartCoords = eastPartCoords[eastPartCoords[:, 0] > 0, :] westPartCoords = westPartCoords[westPartCoords[:, 0] < 0, :] # Convert to lists eastPartCoords = eastPartCoords.tolist() westPartCoords = westPartCoords.tolist() # Discard anything left with only 2 points if len(eastPartCoords) < 3: eastPartCoords = [] if len(westPartCoords) < 3: westPartCoords = [] # Close polygons. What a kludge..... if len(eastPartCoords) > 2: if eastPartCoords[-1][0] != eastPartCoords[0][ 0] or eastPartCoords[-1][1] != eastPartCoords[0][1]: eastPartCoords.append(eastPartCoords[0]) if len(westPartCoords) > 2: if westPartCoords[-1][0] != westPartCoords[0][ 0] or westPartCoords[-1][1] != westPartCoords[0][1]: westPartCoords.append(westPartCoords[0]) # Make a multi-polygon from the two parts coordsMulti = [[eastPartCoords], [westPartCoords]] jsonStr = repr({ 'type': 'MultiPolygon', 'coordinates': coordsMulti }) newGeom = ogr.CreateGeometryFromJson(jsonStr) else: # It didn't really cross the date line, but seems to due to rounding # error in crossesDateline(). newGeom = copyGeom(geom) else: newGeom = copyGeom(geom) return newGeom
def calculate_on_GEE(self): self.close() crosses_180th, geojsons = self.aoi.bounding_box_gee_geojson() val = [] n = 1 if self.area_tab.area_fromfile.isChecked(): for f in self.aoi.get_layer_wgs84().getFeatures(): # Get an OGR geometry from the QGIS geometry geom = f.geometry() val.append(geom) n += 1 # stringify json object val_string = '{}'.format(json.loads(val[0].asJson())) # create ogr geometry val_geom = ogr.CreateGeometryFromJson(val_string) # simplify polygon to tolerance of 0.003 val_geom_simplified = val_geom.Simplify(0.003) # fetch coordinates from json coords = json.loads( val_geom_simplified.ExportToJson())['coordinates'] geometries = json.dumps([{ "coordinates": coords, "type": "Polygon" }]) elif self.area_tab.area_fromadmin.isChecked(): geometries = json.dumps([{ "coordinates": self.get_admin_poly_geojson()['geometry']['coordinates'][0], "type": "Polygon" }]) elif self.area_tab.area_frompoint.isChecked(): point = QgsPointXY( float(self.area_tab.area_frompoint_point_x.text()), float(self.area_tab.area_frompoint_point_y.text())) crs_src = QgsCoordinateReferenceSystem( self.area_tab.canvas.mapSettings().destinationCrs().authid()) point = QgsCoordinateTransform( crs_src, self.aoi.crs_dst, QgsProject.instance()).transform(point) geometries = json.dumps( json.loads(QgsGeometry.fromPointXY(point).asJson())) payload = { 'year_start': self.lc_setup_tab.use_esa_bl_year.date().year(), 'year_end': self.lc_setup_tab.use_esa_tg_year.date().year(), 'fl': self.get_fl(), 'download_annual_lc': self.download_annual_lc.isChecked(), # 'geojsons':json.dumps(geojsons), 'geojsons': geometries, 'crs': self.aoi.get_crs_dst_wkt(), 'crosses_180th': crosses_180th, 'remap_matrix': self.lc_setup_tab.dlg_esa_agg.get_agg_as_list(), 'task_name': self.options_tab.task_name.text(), 'task_notes': self.options_tab.task_notes.toPlainText() } resp = run_script(get_script_slug('soil-organic-carbon'), payload) if resp: mb.pushMessage( QtWidgets.QApplication.translate("MISLAND", "Submitted"), QtWidgets.QApplication.translate( "MISLAND", "Soil organic carbon submitted to Google Earth Engine."), level=0, duration=5) else: mb.pushMessage( QtWidgets.QApplication.translate("MISLAND", "Error"), QtWidgets.QApplication.translate( "MISLAND", "Unable to submit soil organic carbon task to Google Earth Engine." ), level=0, duration=5)
def get_stitch_cfgs(context_file): """Return all possible stitch interferogram configurations.""" # get context with open(context_file) as f: context = json.load(f) # get args project = context['project'] direction = context.get('direction', 'along') subswaths = [int(i) for i in context.get('subswaths', "1 2 3").split()] subswaths.sort() min_stitch_count = int(context['min_stitch_count']) extra_products = [i.strip() for i in context.get('extra_products', 'los.rdr.geo').split()] orig_query = context['query'] logger.info("orig_query: %s" % json.dumps(orig_query, indent=2)) # cleanse query of ids from triggered rules query = clean_query(orig_query) logger.info("clean query: %s" % json.dumps(query, indent=2)) # log enumerator params logger.info("project: %s" % project) logger.info("direction: %s" % direction) logger.info("subswaths: %s" % subswaths) logger.info("min_stitch_count: %s" % min_stitch_count) logger.info("extra_products: %s" % extra_products) # get bbox from query coords = None bbox = [-90., 90., -180., 180.] if 'and' in query.get('query', {}).get('filtered', {}).get('filter', {}): filts = query['query']['filtered']['filter']['and'] elif 'geo_shape' in query.get('query', {}).get('filtered', {}).get('filter', {}): filts = [ { "geo_shape": query['query']['filtered']['filter']['geo_shape'] } ] else: filts = [] for filt in filts: if 'geo_shape' in filt: coords = filt['geo_shape']['location']['shape']['coordinates'] roi = { 'type': 'Polygon', 'coordinates': coords, } logger.info("query filter ROI: %s" % json.dumps(roi)) roi_geom = ogr.CreateGeometryFromJson(json.dumps(roi)) roi_x_min, roi_x_max, roi_y_min, roi_y_max = roi_geom.GetEnvelope() bbox = [ roi_y_min, roi_y_max, roi_x_min, roi_x_max ] logger.info("query filter bbox: %s" % bbox) break # query docs uu = UU() logger.info("rest_url: {}".format(uu.rest_url)) logger.info("dav_url: {}".format(uu.dav_url)) logger.info("version: {}".format(uu.version)) logger.info("grq_index_prefix: {}".format(uu.grq_index_prefix)) # get normalized rest url rest_url = uu.rest_url[:-1] if uu.rest_url.endswith('/') else uu.rest_url # get index name and url url = "{}/{}/_search?search_type=scan&scroll=60&size=100".format(rest_url, uu.grq_index_prefix) logger.info("idx: {}".format(uu.grq_index_prefix)) logger.info("url: {}".format(url)) # query hits query.update({ "partial_fields" : { "partial" : { "exclude" : "city", } } }) #logger.info("query: {}".format(json.dumps(query, indent=2))) r = requests.post(url, data=json.dumps(query)) r.raise_for_status() scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] hits = [] while True: r = requests.post('%s/_search/scroll?scroll=60m' % rest_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] if len(res['hits']['hits']) == 0: break hits.extend(res['hits']['hits']) # remove partial fields and reformat metadata as expected by stitcher_utils #hits = remove_partials(hits) # extract reference ids ids = { h['_id']: True for h in hits } logger.info("ids: {}".format(json.dumps(ids, indent=2))) logger.info("hits count: {}".format(len(hits))) # dump metadata valid_meta_ts_out_file = "valid_meta_ts_out.json" with open(valid_meta_ts_out_file, 'w') as f: json.dump(hits, f, indent=2) # group frames by track and date pairs grouped = group_frames_by_track_date(hits) logger.info("grouped: %s" % json.dumps(grouped, indent=2)) # enumerate configs projects = [] directions = [] extra_products_list = [] filenames = [] filename_urls = [] ifg_ids = [] base_products = ['filt_topophase.unw.geo', 'filt_topophase.unw.conncomp.geo', 'phsig.cor.geo'] base_products.extend(extra_products) for track in sorted(grouped['grouped']): for dt_pair in sorted(grouped['grouped'][track]): stitch_count = 0 # filter scenes without all requested subswaths swath_check = {} for swath in subswaths: if swath not in grouped['grouped'][track][dt_pair]: raise RuntimeError("Did not find singlescene IFGs for subswath %s for track %s dates %s. Check your query results." % (swath, track, dt_pair)) for tr, id in grouped['grouped'][track][dt_pair][swath]: swath_check.setdefault(tr, {})[swath] = id skip_tr = {} for tr in sorted(swath_check): for swath in subswaths: if swath not in swath_check[tr]: skip_tr[tr] = True furls = [] swathnums = [] ifg_sts = set() ifg_ets = set() fnames_tr = {} for swath in subswaths: swathnums.append(swath) for tr, id in grouped['grouped'][track][dt_pair][swath]: if tr in skip_tr: logger.warning("Skipping %s for scene %s since only subswaths %s exist." % (id, tr, sorted(swath_check[tr].keys()))) continue bisect.insort(fnames_tr.setdefault(tr, []), os.path.join(id, 'merged', 'filt_topophase.unw.geo')) for prod_file in base_products: furls.append({ 'url': "%s/merged/%s" % (grouped['hits'][id], prod_file), 'local_path': "%s/merged/" % id, }) furls.append({ 'url': "%s/merged/%s.xml" % (grouped['hits'][id], prod_file), 'local_path': "%s/merged/" % id, }) furls.append({ 'url': "%s/fine_interferogram.xml" % grouped['hits'][id], 'local_path': "%s/" % id, }) furls.append({ 'url': "%s/%s.dataset.json" % (grouped['hits'][id], id), 'local_path': "%s/_%s.dataset.json" % (id, id), }) furls.append({ 'url': "%s/%s.met.json" % (grouped['hits'][id], id), 'local_path': "%s/_%s.met.json" % (id, id), }) stitch_count += 1 st, et = tr.split('_') ifg_sts.add(st) ifg_ets.add(et) ifg_sts = list(ifg_sts) ifg_sts.sort() ifg_ets = list(ifg_ets) ifg_ets.sort() # check minimum stitch count met if stitch_count < min_stitch_count: logger.warning("Failed to find minimum stitch count of %s for track %s date pair %s: %s" % (min_stitch_count, track, dt_pair, stitch_count)) continue # build job params projects.append(project) directions.append(direction) extra_products_list.append(extra_products) filenames.append([fnames_tr[tr] for tr in sorted(fnames_tr)]) filename_urls.append(furls) ifg_hash = hashlib.md5(json.dumps([ projects[-1], directions[-1], extra_products_list[-1], filenames[-1], filename_urls[-1], ], sort_keys=True)).hexdigest() ifg_ids.append(ID_TMPL.format(int(track), ifg_sts[0], ifg_ets[-1], ''.join(map(str, swathnums)), direction, ifg_hash[0:4])) logger.info("projects: %s" % projects) logger.info("directions: %s" % directions) logger.info("extra_products: %s" % extra_products_list) logger.info("filenames: %s" % json.dumps(filenames, indent=2)) logger.info("filename_urls: %s" % json.dumps(filename_urls, indent=2)) logger.info("ifg_ids: %s" % ifg_ids) return ( projects, directions, extra_products_list, filenames, filename_urls, ifg_ids )
def ee_image_download(ini_path=None, overwrite_flag=False): """Earth Engine Annual Mean Image Download Parameters ---------- ini_path : str overwrite_flag : bool, optional If True, overwrite existing files (the default is False). """ logging.info('\nEarth Engine EDDI Image Download') # 12 month EDDI aggregation_days = 365 export_name = 'eddi_12month' output_name = 'eddi.12month' eddi_date_list = [ '0131', '0228', '0331', '0430', '0531', '0630', '0731', '0831', '0930', '1031', '1130', '1231' ] # eddi_date_list = ['0930', '1231'] # eddi_date_list = ['{:02d}01'.format(m) for m in range(1, 13)] # eddi_date_list = [] eddi_folder = 'eddi' # Do we need to support separate EDDI years? # start_year = 1984 # end_year = 2016 # climo_year_start = 1979 climo_year_end = 2017 # Read config file # ini = inputs.ini_parse(ini_path, section='IMAGE') ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='SPATIAL') inputs.parse_section(ini, section='EXPORT') inputs.parse_section(ini, section='IMAGES') nodata_value = -9999 # Manually set output spatial reference logging.info('\nHardcoding GRIDMET snap, cellsize and spatial reference') ini['output_x'], ini['output_y'] = -124.79299639209513, 49.41685579737572 ini['SPATIAL']['cellsize'] = 0.041666001963701 # ini['SPATIAL']['cellsize'] = [0.041666001963701, 0.041666001489718] # ini['output_x'] = -124.79166666666666666667 # ini['output_y'] = 25.04166666666666666667 # ini['SPATIAL']['cellsize'] = 1. / 24 ini['SPATIAL']['osr'] = gdc.epsg_osr(4326) # ini['SPATIAL']['osr'] = gdc.epsg_osr(4269) ini['SPATIAL']['crs'] = 'EPSG:4326' logging.debug(' Snap: {} {}'.format(ini['output_x'], ini['output_y'])) logging.debug(' Cellsize: {}'.format(ini['SPATIAL']['cellsize'])) logging.debug(' OSR: {}'.format(ini['SPATIAL']['osr'])) # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # Filter features by FID before merging geometries if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # Merge geometries if ini['INPUTS']['merge_geom_flag']: merge_geom = ogr.Geometry(ogr.wkbMultiPolygon) for zone in zone_geom_list: zone_multipolygon = ogr.ForceToMultiPolygon( ogr.CreateGeometryFromJson(json.dumps(zone[2]))) for zone_polygon in zone_multipolygon: merge_geom.AddGeometry(zone_polygon) # merge_json = json.loads(merge_mp.ExportToJson()) zone_geom_list = [[ 0, ini['INPUTS']['zone_filename'], json.loads(merge_geom.ExportToJson()) ]] ini['INPUTS']['zone_field'] = '' # Need zone_shp_path projection to build EE geometries zone_osr = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path']) zone_proj = gdc.osr_wkt(zone_osr) # zone_proj = ee.Projection(zone_proj).wkt().getInfo() # zone_proj = zone_proj.replace('\n', '').replace(' ', '') logging.debug(' Zone Projection: {}'.format(zone_proj)) # Initialize Earth Engine API key logging.info('\nInitializing Earth Engine') ee.Initialize() utils.ee_request(ee.Number(1).getInfo()) # Get current running tasks tasks = utils.get_ee_tasks() # Download images for each feature separately for zone_fid, zone_name, zone_json in zone_geom_list: zone_name = zone_name.replace(' ', '_') logging.info('ZONE: {} (FID: {})'.format(zone_name, zone_fid)) # Build EE geometry object for zonal stats zone_geom = ee.Geometry(zone_json, zone_proj, False) # Project the zone_geom to the GRIDMET projection # if zone_proj != output_proj: zone_geom = zone_geom.transform(ini['SPATIAL']['crs'], 0.001) # Get the extent from the Earth Engine geometry object? zone_extent = zone_geom.bounds().getInfo()['coordinates'][0] zone_extent = gdc.Extent([ min(zip(*zone_extent)[0]), min(zip(*zone_extent)[1]), max(zip(*zone_extent)[0]), max(zip(*zone_extent)[1]) ]) # # Use GDAL and geometry json to build extent, transform, and shape # zone_extent = gdc.Extent( # ogr.CreateGeometryFromJson(json.dumps(zone_json)).GetEnvelope()) # # zone_extent = gdc.Extent(zone_geom.GetEnvelope()) # zone_extent.ymin, zone_extent.xmax = zone_extent.xmax, zone_extent.ymin # Adjust extent to match raster zone_extent = zone_extent.adjust_to_snap('EXPAND', ini['output_x'], ini['output_y'], ini['SPATIAL']['cellsize']) zone_geo = zone_extent.geo(ini['SPATIAL']['cellsize']) zone_transform = gdc.geo_2_ee_transform(zone_geo) zone_transform = '[' + ','.join(map(str, zone_transform)) + ']' zone_shape = zone_extent.shape(ini['SPATIAL']['cellsize']) logging.debug(' Zone Shape: {}'.format(zone_shape)) logging.debug(' Zone Transform: {}'.format(zone_transform)) logging.debug(' Zone Extent: {}'.format(zone_extent)) # logging.debug(' Geom: {}'.format(zone_geom.getInfo())) # output_transform = zone_transform[:] output_transform = '[' + ','.join(map(str, zone_transform)) + ']' output_shape = '[{1}x{0}]'.format(*zone_shape) logging.debug(' Output Projection: {}'.format(ini['SPATIAL']['crs'])) logging.debug(' Output Transform: {}'.format(output_transform)) logging.debug(' Output Shape: {}'.format(output_shape)) zone_eddi_ws = os.path.join(ini['IMAGES']['output_ws'], zone_name, eddi_folder) if not os.path.isdir(zone_eddi_ws): os.makedirs(zone_eddi_ws) # GRIDMET PDSI # Process each image in the collection by date export_list = [] export_list = list( date_range(datetime.datetime(ini['INPUTS']['start_year'], 1, 1), datetime.datetime(ini['INPUTS']['end_year'], 12, 31), skip_leap_days=True)) # Filter list to only keep last dekad of October and December if eddi_date_list: export_list = [ tgt_dt for tgt_dt in export_list if tgt_dt.strftime('%m%d') in eddi_date_list ] for tgt_dt in export_list: date_str = tgt_dt.strftime('%Y%m%d') logging.info('{} {}'.format(tgt_dt.strftime('%Y-%m-%d'), output_name)) if tgt_dt >= datetime.datetime.today(): logging.info(' Date after current date, skipping') continue # Rename to match naming style from getDownloadURL # image_name.band.tif export_id = '{}_{}_{}'.format(ini['INPUTS']['zone_filename'], date_str, export_name.lower()) output_id = '{}_{}'.format(date_str, output_name) export_path = os.path.join(ini['EXPORT']['export_ws'], export_id + '.tif') output_path = os.path.join(zone_eddi_ws, output_id + '.tif') logging.debug(' Export: {}'.format(export_path)) logging.debug(' Output: {}'.format(output_path)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) del tasks[export_id] if os.path.isfile(export_path): logging.debug(' Export image already exists, removing') utils.remove_file(export_path) # os.remove(export_path) if os.path.isfile(output_path): logging.debug(' Output image already exists, removing') utils.remove_file(output_path) # os.remove(output_path) else: if os.path.isfile(export_path): logging.debug(' Export image already exists, moving') shutil.move(export_path, output_path) gdc.raster_path_set_nodata(output_path, nodata_value) # DEADBEEF - should raster stats be computed? # gdc.raster_statistics(output_path) continue elif os.path.isfile(output_path): logging.debug(' Output image already exists, skipping') continue elif export_id in tasks.keys(): logging.debug(' Task already submitted, skipping') continue eddi_image = ee_eddi_image(tgt_dt.strftime('%Y-%m-%d'), agg_days=aggregation_days, variable='eddi', year_start=climo_year_start, year_end=climo_year_end) logging.debug(' Building export task') # if ini['EXPORT']['export_dest'] == 'gdrive': task = ee.batch.Export.image.toDrive( image=eddi_image, description=export_id, # folder=ini['EXPORT']['export_folder'], fileNamePrefix=export_id, dimensions=output_shape, crs=ini['SPATIAL']['crs'], crsTransform=output_transform) # elif ini['EXPORT']['export_dest'] == 'gdrive': # task = ee.batch.Export.image.toCloudStorage( # image=eddi_image, # description=export_id, # bucket=ini['EXPORT']['export_folder'], # fileNamePrefix=export_id, # dimensions=output_shape, # crs=ini['SPATIAL']['crs'], # crsTransform=output_transform) logging.debug(' Starting export task') utils.ee_request(task.start())
def kml_geom(place): geojson_geom = place.get('location').get('geometry') json_geom = ogr.CreateGeometryFromJson(str(geojson_geom)) kml_geom = json_geom.ExportToKML() return kml_geom
prop, value)) else: raise else: return feature with open(os.path.join(pwd, '..', 'raw', 'chicago_community_areas.geojson'), 'r') as f: ca_geojson = json.load(f) ca_feature = feature_from_array(ca_geojson['features'], 'community', community_area) ca_geom = ogr.CreateGeometryFromJson(json.dumps(ca_feature['geometry'])) intersections = [] with open(os.path.join(pwd, '..', 'raw', 'chicago_wards.geojson'), 'r') as f: ward_geojson = json.load(f) for ward in wards.split(','): ward_feature = feature_from_array(ward_geojson['features'], 'ward', ward) ward_geom = ogr.CreateGeometryFromJson( json.dumps(ward_feature['geometry'])) intersection = json.loads( ca_geom.Intersection(ward_geom).ExportToJson())
def checkGeos(): point = """{"type":"Point","coordinates":[-90.02001032734194,35.127405562848075]}""" geom = ogr.CreateGeometryFromJson(point) print(geom.GetX()) print(geom.GetY())
def export_shapefile(vectorlayer): file_path = vectorlayer.file_path file_name = vectorlayer.file_name full_path = join(file_path, file_name) encoding = vectorlayer.encoding srs_wkt = vectorlayer.srs_wkt geom_type = vectorlayer.geom_type f = open(full_path, "r", encoding=encoding) geojson = json.loads(f.read()) f.close() dst_dir = tempfile.mkdtemp() shapefile_name = file_name.replace(".json", ".shp") dst_file = join(dst_dir, shapefile_name) #sistema de coordenadas 4326 src_spatial_ref = osr.SpatialReference() src_spatial_ref.ImportFromEPSG(4326) #sistema de coordenadas de la capa original dst_spatial_ref = osr.SpatialReference() dst_spatial_ref.ImportFromWkt(srs_wkt) coord_transform = osr.CoordinateTransformation(src_spatial_ref, dst_spatial_ref) driver = ogr.GetDriverByName("ESRI shapefile") datasource = driver.CreateDataSource(dst_file) layer = datasource.CreateLayer(name=shapefile_name, srs=dst_spatial_ref, geom_type=geom_type) for i, attr in enumerate(geojson["attributes"]): field = ogr.FieldDefn(attr["name"], attr["type"]) field.SetWidth(attr["width"]) field.SetPrecision(attr["precision"]) layer.CreateField(field) for i, feature in enumerate(geojson["features"]): geometry = feature.get("geometry", None) if (geometry == None): continue dst_geometry = ogr.CreateGeometryFromJson(json.dumps(geometry)) dst_feature = ogr.Feature(layer.GetLayerDefn()) dst_feature.SetGeometry(dst_geometry) dst_geometry.Transform(coord_transform) properties = feature.get("properties", None) if properties == None: continue for field, value in properties.items(): field = field.replace("\"", "") field = field.replace("'", "") dst_feature.SetField(field, value) layer.CreateFeature(dst_feature) dst_feature.Destroy() datasource.Destroy() zip_dst = file_name.replace(".json", "_zip") zip_dst = shutil.make_archive(zip_dst, 'zip', dst_dir) shutil.rmtree(dst_dir) zip_file = open(zip_dst, "rb") f = FileWrapper(zip_file) response = HttpResponse(f, content_type="application/zip") zip_name = file_name.replace(".json", ".zip") response['Content-Disposition'] = "attachment; filename=" + zip_name return response
# It’s known as "Cylindrical Equal-Area”, but I guess it’s called the other thing in EPSG because that agency was the first # one to add it under their own product name. </sigh> # https://nsidc.org/data/atlas/epsg_3410.html from osgeo import ogr from osgeo import osr source = osr.SpatialReference() source.ImportFromEPSG(4326) target = osr.SpatialReference() target.ImportFromEPSG(3410) transform = osr.CoordinateTransformation(source, target) poly = ogr.CreateGeometryFromJson(geojson.dumps(f['geometry'])) poly.Transform(transform) sq_m = format(poly.GetArea(), 'f') props['geom:area_square_m'] = float(sq_m) except Exception, e: logging.warning( "failed to calculate area in square meters, because %s" % e) # end of osgeo stuff f['bbox'] = bbox # ensure that all properties are prefixed
def _calc_route_info(_route_data): link_collection = pd.DataFrame(columns=[ 'WVK_ID', 'WEGNUMMER', 'JTE_ID_BEG', 'JTE_ID_END', 'POS_TV_WOL', 'BST_CODE', 'geometry', 'DutchRD_X', 'DutchRD_Y', 'latitude', 'longitude', 'route_id', 'link_offset', 'distance_offset' ]) route_id = _route_data['route_id'] route_link_collection = _route_data['link_collection'] XY = [] latlon = [] wvk_id = [] link_begin_ind = [] link_distance = [] lnstr = ogr.Geometry(ogr.wkbLineString) distance_offset = 0 for link_ind, link in enumerate(route_link_collection): link_info = { 'route_id': route_id, 'link_offset': link_ind, 'JTE_ID_BEG': link[0], 'JTE_ID_END': link[1], **link[2] } # DutchRD X,Y link_geo = ogr.CreateGeometryFromJson(link[2]['geometry']) pts = link_geo.GetPoints() link_info['DutchRD_X'] = np.array([p[0] for p in pts]) link_info['DutchRD_Y'] = np.array([p[1] for p in pts]) link_info['distance_offset'] = distance_offset XY += pts wvk_id.append(link[2]['WVK_ID']) link_begin_ind.append(len(latlon) + 1) segment_length = np.zeros(len(pts)) p_pre = np.array(pts[0]) for p_ind, p in enumerate(pts): p_wgs = DutchRDtoWGS84(*p) lnstr.AddPoint_2D(*p_wgs) latlon.append(p_wgs) p = np.array(p) segment_length[p_ind] = np.linalg.norm(p - p_pre) p_pre = p link_distance.append(np.sum(segment_length)) link_info['segment_distances'] = segment_length link_info['distance_along_route'] = np.cumsum( segment_length) + distance_offset link_collection = link_collection.append(link_info, ignore_index=True) distance_offset = link_info['distance_along_route'][-1] route_info = { 'id': route_id, 'XY': XY, 'latlon': latlon, 'link': { 'wvk_id': wvk_id, 'distance': link_distance, 'beginpointind': link_begin_ind }, 'geojson': { 'type': 'Feature', 'geometry': json.loads(lnstr.ExportToJson()), 'properties': [] } } return route_info, link_collection
def get(self, request, *args, **kwargs): ext_lc = kwargs['extension'].lower() ## data = None if settings.GEODASH_CACHE_DATA: client = provision_memcached_client() if client: key = self._build_key(request, *args, **kwargs) print "Checking cache with key ", key data = None try: data = client.get(key) except socket_error as serr: data = None print "Error getting data from in-memory cache." if serr.errno == errno.ECONNREFUSED: print "Memcached is likely not running. Start memcached with supervisord." raise serr if not data: print "Data not found in cache." data = self._build_data(request, *args, **kwargs) if ext_lc == "geodash": data = [int(x) for x in data] try: client.set(key, data) except socket_error as serr: print "Error saving data to in-memory cache." if serr.errno == errno.ECONNREFUSED: print "Memcached is likely not running or the data exceeds memcached item size limit. Start memcached with supervisord." raise serr else: print "Data found in cache." else: print "Could not connect to memcached client. Bypassing..." data = self._build_data(request, *args, **kwargs) else: print "Not caching data (settings.geodash_CACHE_DATA set to False)." data = self._build_data(request, *args, **kwargs) #content = json.dumps(data, default=jdefault) #content = re.sub( # settings.GEODASH_REGEX_CLIP_COORDS_PATTERN, # settings.GEODASH_REGEX_CLIP_COORDS_REPL, # content, # flags=re.IGNORECASE) root = self._build_root(request, *args, **kwargs) attributes = self._build_attributes(request, *args, **kwargs) if attributes: data = grep(obj=data, root=root, attributes=attributes, filters=getRequestParameters(request, "grep", None)) if ext_lc == "json": return HttpResponse(json.dumps(data, default=jdefault), content_type="application/json") elif ext_lc == "yml" or ext_lc == "yaml": response = yaml.safe_dump(data, encoding="utf-8", allow_unicode=True, default_flow_style=False) return HttpResponse(response, content_type="text/plain") elif ext_lc == "csv" or ext_lc == "csv": writer = GeoDashDictWriter("", attributes) writer.writeheader() writer.writerows(extract(root, data, [])) response = writer.getvalue() return HttpResponse(response, content_type="text/csv") elif ext_lc == "zip": # See the following for how to create zipfile in memory, mostly. # https://newseasandbeyond.wordpress.com/2014/01/27/creating-in-memory-zip-file-with-python/ tempDirectory = tempfile.mkdtemp() print "Temp Directory:", tempDirectory if tempDirectory: geometryType = self._build_geometry_type( request, *args, **kwargs) ########### Create Files ########### os.environ['SHAPE_ENCODING'] = "utf-8" # See following for how to create shapefiles using OGR python bindings # https://pcjericks.github.io/py-gdalogr-cookbook/vector_layers.html#filter-and-select-input-shapefile-to-new-output-shapefile-like-ogr2ogr-cli basepath, out_filename, ext = parse_path(request.path) out_shapefile = os.path.join(tempDirectory, out_filename + ".shp") out_driver = ogr.GetDriverByName("ESRI Shapefile") if os.path.exists(out_shapefile): out_driver.DeleteDataSource(out_shapefile) out_datasource = out_driver.CreateDataSource(out_shapefile) out_layer = out_datasource.CreateLayer( (out_filename + ".shp").encode('utf-8'), geom_type=geometryType) ########### Create Fields ########### out_layer.CreateField(ogr.FieldDefn( "id", ogr.OFTInteger)) # Create ID Field for attribute in attributes: label = attribute.get('label_shp') or attribute.get( 'label') out_layer.CreateField( ogr.FieldDefn( label, ATTRIBUTE_TYPE_TO_OGR.get( attribute.get('type', 'string')))) ########### Create Features ########### features = extract(root, data, []) for i in range(len(features)): feature = features[i] out_feature = ogr.Feature(out_layer.GetLayerDefn()) geom = extract( self._build_geometry(request, *args, **kwargs), feature, None) out_feature.SetGeometry( ogr.CreateGeometryFromJson( json.dumps(geom, default=jdefault))) out_feature.SetField("id", i) for attribute in attributes: label = attribute.get('label_shp') or attribute.get( 'label') out_value = extract(attribute.get('path'), feature, None) out_feature.SetField( (attribute.get('label_shp') or attribute.get('label')), out_value.encode('utf-8') if isinstance( out_value, basestring) else out_value) out_layer.CreateFeature(out_feature) out_datasource.Destroy() ########### Create Projection ########### spatialRef = osr.SpatialReference() spatialRef.ImportFromEPSG(4326) spatialRef.MorphToESRI() with open(os.path.join(tempDirectory, out_filename + ".prj"), 'w') as f: f.write(spatialRef.ExportToWkt()) f.close() ########### Create Zipfile ########### buff = StringIO.StringIO() zippedShapefile = zipfile.ZipFile(buff, mode='w') #memoryFiles = [] component_filenames = os.listdir(tempDirectory) #for i in range(len(componentFiles)): # memoryFiles.append(StringIO.StringIO()) for i in range(len(component_filenames)): with open( os.path.join(tempDirectory, component_filenames[i]), 'r') as f: contents = f.read() zippedShapefile.writestr(component_filenames[i], contents) zippedShapefile.close() print "zippedShapefile.printdir()", zippedShapefile.printdir() ########### Delete Temporary Directory ########### shutil.rmtree(tempDirectory) ########### Response ########### return HttpResponse(buff.getvalue(), content_type="application/zip") #for i in range(len(componentFiles)): # with open(componentFiles[i], 'w') as componentFile: # memoryFiles[i].write(componentFile.read()) else: raise Http404( "Could not acquire temporary directory for building shapefile." ) elif ext_lc == "geodash": response = HttpResponse(content_type='application/octet-stream') # Need to do by bytes(bytearray(x)) to properly translate integers to 1 byte each # If you do bytes(data) it will give 4 bytes to each integer. response.write(bytes(bytearray(data))) return response else: raise Http404("Unknown config format.")
def _create_base_map(self, ): """ Deal with different types way to define the AOI, if none is specified, then the image bound is used. """ gdal.UseExceptions() ogr.UseExceptions() if self.aoi is not None: if os.path.exists(self.aoi): try: g = gdal.Open(self.aoi) # subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir + '/AOI.json', self.aoi]) geojson = get_boundary(self.aoi)[0] with open(self.toa_dir + "/AOI.json", "wb") as f: f.write(geojson.encode()) except: try: gr = ogr.Open(str(self.aoi)) l = gr.GetLayer(0) f = l.GetFeature(0) g = f.GetGeometryRef() except: raise IOError( "AOI file cannot be opened by gdal, please check it or transform into format can be opened by gdal" ) else: try: g = ogr.CreateGeometryFromJson(self.aoi) except: try: g = ogr.CreateGeometryFromGML(self.aoi) except: try: g = ogr.CreateGeometryFromWkt(self.aoi) except: try: g = ogr.CreateGeometryFromWkb(self.aoi) except: raise IOError( "The AOI has to be one of GeoJSON, GML, Wkt or Wkb." ) gjson_str = ( """{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}""" % g.ExportToJson()) with open(self.toa_dir + "/AOI.json", "wb") as f: f.write(gjson_str.encode()) ogr.DontUseExceptions() gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + "/AOI.json"): g = gdal.Open(self.toa_bands[0]) proj = g.GetProjection() if "WGS 84" in proj: # subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) geojson = get_boundary(self.toa_bands[0], to_wgs84=False) with open(self.toa_dir + "/AOI.json", "wb") as f: f.write(geojson.encode()) else: # subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]]) geojson = get_boundary(self.toa_bands[0])[0] with open(self.toa_dir + "/AOI.json", "wb") as f: f.write(geojson.encode()) self.logger.warning( "AOI is not created and full band extend is used") self.aoi = self.toa_dir + "/AOI.json" else: self.aoi = self.toa_dir + "/AOI.json" if self.pixel_res is None: self.pixel_res = abs( gdal.Open(self.toa_bands[0]).GetGeoTransform()[1]) self.psf_xstd = 260 / self.pixel_res self.psf_ystd = 340 / self.pixel_res