def __init__(self, minX, maxX, minY, maxY, logFile=None): ''' :param minX: :param maxX: :param minY: :param maxY: :param logFile: if logfile is specified logger will write into file instead of the terminal ''' if logFile is None: logging.basicConfig( format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) else: logging.basicConfig( filename=logFile, filemode='a', format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) sys.excepthook = self.uncaughtErrorHandler self.logger = logging.getLogger(__name__) self.logger.info('Process started') self.client = MalardClient(notebook=False) self.minX = minX self.maxX = maxX self.minY = minY self.maxY = maxY self.parentDsName = self.config('parentDsName') self.outputDataSet = self.config('outputDataSet') self.inputDataSet = self.config('inputDataSet') self.region = self.config('region') self.maskDataSet = self.config('maskDataSet') self.withinDataSets = self.config('withinDataSets') self.withinDataSetTypes = self.config('withinDataSetTypes') self.runName = self.config('runName') assert (self.maxX - self.minX) == (self.maxY - self.minY) self.size = maxX - minX self.dataSet = DataSet(parentDs=self.config('parentDsName'), dataSet=self.config('inputDataSet'), region=self.config('region'))
def __init__(self, logFile=None): ''' :param logFile: if logfile is specified logger will write into file instead of the terminal ''' if logFile is None: logging.basicConfig( format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) else: logging.basicConfig( filename=logFile, filemode='a', format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) sys.excepthook = self.uncaughtErrorHandler self.logger = logging.getLogger(__name__) self.logger.info('Process started') #self.parentDsName = self.config('parentDsName') self.inputDataSet = DataSet(parentDs=self.config('parentDsName'), dataSet=self.config('inputDataSet'), region=self.config('region')) #self.region = self.config('region') self.runName = self.config('runName') self.client = MalardClient(notebook=False) self.query_sync = DataSetQuery(self.config('malardSyncURL'), self.config('malardEnvironmentName')) #self.query_async = AsyncDataSetQuery.AsyncDataSetQuery(self.config('malardAsyncURL'), self.config('malardEnvironmentName'), False) # get projection #self.projection = json.loads(self.client.getProjection(self.parentDsName, self.region))['proj4'] self.projection = self.client.getProjection(self.inputDataSet).proj4
# remove xyz file if deleteTemporaryFiles: os.remove(out_xyz) return xy, values if __name__ == '__main__': logging.basicConfig(level=logging.INFO) client = MalardClient() parentDataSet = 'cryotempo' dataSet = 'GRIS_BaselineC_Q2' region = 'greenland' inputDs = DataSet(parentDataSet, dataSet, region) proj4 = client.getProjection(inputDs).proj4 print(proj4) bb = client.boundingBox(inputDs) gridCells = client.gridCells(inputDs, bb) minT = datetime.datetime(2011, 3, 1, 0, 0, 0) maxT = datetime.datetime(2011, 3, 31, 23, 59, 59) mask = '/data/puma1/scratch/cryotempo/masks/ice.shp' tmp = gp.read_file(mask)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Wed Nov 20 09:33:33 2019 @author: jon """ from MalardClient.MalardClient import MalardClient from MalardClient.DataSet import DataSet from MalardClient.BoundingBox import BoundingBox client = MalardClient() ds = DataSet("cryotempo","poca","greenland" ) dsSwath = DataSet("cryotempo","GRIS_BaselineC_Q2","greenland" ) bb = client.boundingBox(ds) gcs = client.gridCells(ds, bb) minX=-1600000 maxX=-1500000 minY=-2600000 maxY=-2500000 minT=1298912551 maxT=1298912551 bb = BoundingBox( minX, maxX, minY, maxY, minT, maxT )
def main(pub_month, pub_year, loadConfig): region = loadConfig["region"] parentDataSet = loadConfig["parentDataSet"] uncertainty_threshold = loadConfig[ "uncertainty_threshold"] if "uncertainty_threshold" in loadConfig else None powerdB = loadConfig["powerdB"] coh = loadConfig["coh"] dataSetName = loadConfig["resultsetName"] pocaParentDataSet = loadConfig["pocaParentDataSet"] pocaDataSetName = loadConfig["pocaDataSet"] pocaDemDiff = loadConfig["pocaDemDiff"] output_path = os.path.join(loadConfig["resultPath"], "pointProduct") ensure_dir(output_path) malardEnv = loadConfig["MalardEnvironment"] client = MalardClient(malardEnv) uncDatasetName = "{}_unc".format( dataSetName) if uncertainty_threshold is not None else dataSetName uncDataSet = DataSet(parentDataSet, uncDatasetName, region) dataSet = DataSet(parentDataSet, dataSetName, region) pocaDataSet = DataSet(pocaParentDataSet, pocaDataSetName, region) pocaDataSet_noDemDiff = DataSet(pocaParentDataSet, pocaDataSetName.replace("_demDiff", ""), region) projections = [ 'x', 'y', 'time', 'elev', 'powerdB', 'coh', 'demDiff', 'demDiffMad', 'swathFileId', 'Q_uStd' ] filters = [{ 'column': 'Q_uStd', 'op': 'lte', 'threshold': uncertainty_threshold }, { 'column': 'powerdB', 'op': 'gte', 'threshold': powerdB }, { 'column': 'coh', 'op': 'gte', 'threshold': coh }, { 'column': 'inRegionMask', 'op': 'eq', 'threshold': 1.0 }] filters_poca = [{ "column": "demDiff", "op": "lte", "threshold": pocaDemDiff }, { "column": "demDiff", "op": "gte", "threshold": -pocaDemDiff }, { 'column': 'inRegionMask', 'op': 'eq', 'threshold': 1.0 }] from_dt = datetime(pub_year, pub_month, 1, 0, 0, 0) to_dt = from_dt + relativedelta(months=1) - timedelta(seconds=1) bb = client.boundingBox(uncDataSet) gridcells = client.gridCells( uncDataSet, BoundingBox(bb.minX, bb.maxX, bb.minY, bb.maxY, from_dt, to_dt)) proj4 = client.getProjection(uncDataSet).proj4 print("Number of Gridcells found to process {}".format(len(gridcells))) process_start = datetime.now() print("MinT={} MaxT={}".format(from_dt, to_dt)) #Create a shapefile index for each month index = s.ShapeFileIndex(output_path, "THEM_POINT", proj4, uncDataSet.region, from_dt) for i, gc in enumerate(gridcells): gc_start = datetime.now() month_gc = BoundingBox(gc.minX, gc.maxX, gc.minY, gc.maxY, from_dt, to_dt) queryInfo = client.executeQuery(uncDataSet, month_gc, projections=projections, filters=filters) if queryInfo.status == "Success" and not queryInfo.resultFileName.startswith( "Error"): data = queryInfo.to_df dataSwathStr = np.array(len(data), "S5") dataSwathStr.fill("swath") data["swathPoca"] = dataSwathStr swath_file_ids = data['swathFileId'].unique() pocaInfo = client.executeQuery(pocaDataSet, gc, filters=filters_poca) pocaDf = pd.DataFrame() if pocaInfo.status == "Success" and not pocaInfo.resultFileName.startswith( "Error"): pocaDf = pocaInfo.to_df if len(pocaDf) > 0: pocaStr = np.empty(len(pocaDf), "S5") pocaStr.fill("poca") pocaDf["swathPoca"] = pocaStr poca_file_ids = pocaDf['swathFileId'].unique() print("Poca points to include {}".format(len(pocaDf))) data = pd.concat([data, pocaDf], sort=False) print("Found {} data rows".format(len(data))) if len(data) > 0: results = client.getSwathNamesFromIds(dataSet, swath_file_ids) if len(pocaDf) > 0: try: results.update( client.getSwathNamesFromIds( pocaDataSet_noDemDiff, poca_file_ids)) except KeyError as ex: print( "Exception caught while retrieving swathIds for data set {} file ids {}" .format(pocaDataSet_noDemDiff, poca_file_ids)) raise KeyError(ex) writePointProduct(output_path, dataSet, month_gc, data, proj4, results, index) client.releaseCacheHandle(pocaInfo.resultFileName) else: print("Grid Cells skipped X=[{}] Y=[{}] with message [{}] ".format( gc.minX, gc.minY, queryInfo.status)) client.releaseCacheHandle(queryInfo.resultFileName) index.close() gc_elapsed = (datetime.now() - gc_start).total_seconds() print('Processed [{}] grid cells. Took=[{}]s'.format(i + 1, gc_elapsed)) process_elapsed = (datetime.now() - process_start).total_seconds() print("Took [{}s] to process".format(process_elapsed))
Created on Wed Nov 13 09:55:03 2019 @author: jon """ import MalardGDAL as mg from MalardClient.MalardClient import MalardClient from MalardClient.DataSet import DataSet from MalardClient.BoundingBox import BoundingBox from MalardClient.MaskFilter import MaskFilter from datetime import datetime client = MalardClient() ds = DataSet("cryotempo", "swath_c", "greenland") proj4 = client.getProjection(ds).proj4 print(proj4) minX = 700000 minY = -2200000 cell_size = 130000 bbox = BoundingBox(minX, minX + cell_size, minY, minY + cell_size, datetime(2011, 2, 1, 0, 0), datetime(2011, 5, 1, 0, 0)) ## TODO: These need to be stored in Malard by DataSet and Type. maskFilterIce = MaskFilter( p_shapeFile="/data/puma1/scratch/cryotempo/masks/icesheets.shp")
def getDataSets(self, parentDataSet): listOfDatasets = self.query.getDataSets(parentDataSet) return [DataSet(parentDataSet, ds['name'], ds['region'] ) for ds in json.loads(listOfDatasets) ]
def __init__(self, minX, maxX, minY, maxY, logFile=None, notebook=False): ''' :param minX: :param maxX: :param minY: :param maxY: :param logFile: if logfile is specified logger will write into file instead of the terminal ''' if logFile is None: logging.basicConfig( format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) else: logging.basicConfig( filename=logFile, filemode='a', format= '%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) sys.excepthook = self.uncaughtErrorHandler self.logger = logging.getLogger(__name__) self.logger.info('Process started') self.client = MalardClient(notebook=notebook) self.query_async = AsyncDataSetQuery( self.config('malardAsyncURL'), self.config('malardEnvironmentName'), False) self.minX = minX self.maxX = maxX self.minY = minY self.maxY = maxY assert (self.maxX - self.minX) == (self.maxY - self.minY) self.size = maxX - minX self.inputDataSet = DataSet(parentDs=self.config('parentDsName'), dataSet=self.config('inputDataSet'), region=self.config('region')) self.parentDsName = self.config('parentDsName') self.outputDataSet = self.config('outputDataSet') self.region = self.config('region') self.buffer = self.config('buffer') self.projection = self.client.getProjection(self.inputDataSet).proj4 bbx = self.client.boundingBox(self.inputDataSet) self.minT = bbx.minT self.maxT = bbx.maxT # masks maskDataSet = self.config('maskDataSet') query_sync = DataSetQuery(self.config('malardSyncURL'), self.config('malardEnvironmentName')) mGla = query_sync.getGridCellMask(self.parentDsName, maskDataSet, 'Glacier', self.region, self.minX, self.minY, self.size) self.maskDataSetFile = json.loads(mGla)['fileName']
df['x'] = x df['y'] = y return df environmentName = 'DEVv2' gridCellSize = 100000 resolution = 500 mask_prefix = "SARIN" client = MalardClient(environmentName, True) dataSet = DataSet('cryotempo', 'GRIS_BaselineC_Q2', 'greenland') proj4 = client.getProjection(dataSet).proj4 mask = '/data/puma1/scratch/cryotempo/masks/icesheets.shp' if mask_prefix == "ICE" else '/data/puma1/scratch/cryotempo/sarinmasks/{}_Greenland.shp'.format( mask_prefix) tmpPath = '/home/jon/data/masks/' bbox = client.boundingBox(dataSet) gridCells = client.gridCells(dataSet, bbox) for gc in gridCells: data = interpolationGrid(gc.minX, gc.minY, gridCellSize, resolution)