def __init__(self, minX, maxX, minY, maxY, logFile=None):
'''
:param minX:
:param maxX:
:param minY:
:param maxY:
:param logFile: if logfile is specified logger will write into file instead of the terminal
'''
if logFile is None:
logging.basicConfig(
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
else:
logging.basicConfig(
filename=logFile,
filemode='a',
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
sys.excepthook = self.uncaughtErrorHandler
self.logger = logging.getLogger(__name__)
self.logger.info('Process started')
self.client = MalardClient(notebook=False)
self.minX = minX
self.maxX = maxX
self.minY = minY
self.maxY = maxY
self.parentDsName = self.config('parentDsName')
self.outputDataSet = self.config('outputDataSet')
self.inputDataSet = self.config('inputDataSet')
self.region = self.config('region')
self.maskDataSet = self.config('maskDataSet')
self.withinDataSets = self.config('withinDataSets')
self.withinDataSetTypes = self.config('withinDataSetTypes')
self.runName = self.config('runName')
assert (self.maxX - self.minX) == (self.maxY - self.minY)
self.size = maxX - minX
self.dataSet = DataSet(parentDs=self.config('parentDsName'),
dataSet=self.config('inputDataSet'),
region=self.config('region'))
def __init__(self, logFile=None):
'''
:param logFile: if logfile is specified logger will write into file instead of the terminal
'''
if logFile is None:
logging.basicConfig(
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
else:
logging.basicConfig(
filename=logFile,
filemode='a',
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
sys.excepthook = self.uncaughtErrorHandler
self.logger = logging.getLogger(__name__)
self.logger.info('Process started')
#self.parentDsName = self.config('parentDsName')
self.inputDataSet = DataSet(parentDs=self.config('parentDsName'),
dataSet=self.config('inputDataSet'),
region=self.config('region'))
#self.region = self.config('region')
self.runName = self.config('runName')
self.client = MalardClient(notebook=False)
self.query_sync = DataSetQuery(self.config('malardSyncURL'),
self.config('malardEnvironmentName'))
#self.query_async = AsyncDataSetQuery.AsyncDataSetQuery(self.config('malardAsyncURL'), self.config('malardEnvironmentName'), False)
# get projection
#self.projection = json.loads(self.client.getProjection(self.parentDsName, self.region))['proj4']
self.projection = self.client.getProjection(self.inputDataSet).proj4
# remove xyz file
if deleteTemporaryFiles:
os.remove(out_xyz)
return xy, values
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
client = MalardClient()
parentDataSet = 'cryotempo'
dataSet = 'GRIS_BaselineC_Q2'
region = 'greenland'
inputDs = DataSet(parentDataSet, dataSet, region)
proj4 = client.getProjection(inputDs).proj4
print(proj4)
bb = client.boundingBox(inputDs)
gridCells = client.gridCells(inputDs, bb)
minT = datetime.datetime(2011, 3, 1, 0, 0, 0)
maxT = datetime.datetime(2011, 3, 31, 23, 59, 59)
mask = '/data/puma1/scratch/cryotempo/masks/ice.shp'
tmp = gp.read_file(mask)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 20 09:33:33 2019
@author: jon
"""
from MalardClient.MalardClient import MalardClient
from MalardClient.DataSet import DataSet
from MalardClient.BoundingBox import BoundingBox
client = MalardClient()
ds = DataSet("cryotempo","poca","greenland" )
dsSwath = DataSet("cryotempo","GRIS_BaselineC_Q2","greenland" )
bb = client.boundingBox(ds)
gcs = client.gridCells(ds, bb)
minX=-1600000
maxX=-1500000
minY=-2600000
maxY=-2500000
minT=1298912551
maxT=1298912551
bb = BoundingBox( minX, maxX, minY, maxY, minT, maxT )
def main(pub_month, pub_year, loadConfig):
region = loadConfig["region"]
parentDataSet = loadConfig["parentDataSet"]
uncertainty_threshold = loadConfig[
"uncertainty_threshold"] if "uncertainty_threshold" in loadConfig else None
powerdB = loadConfig["powerdB"]
coh = loadConfig["coh"]
dataSetName = loadConfig["resultsetName"]
pocaParentDataSet = loadConfig["pocaParentDataSet"]
pocaDataSetName = loadConfig["pocaDataSet"]
pocaDemDiff = loadConfig["pocaDemDiff"]
output_path = os.path.join(loadConfig["resultPath"], "pointProduct")
ensure_dir(output_path)
malardEnv = loadConfig["MalardEnvironment"]
client = MalardClient(malardEnv)
uncDatasetName = "{}_unc".format(
dataSetName) if uncertainty_threshold is not None else dataSetName
uncDataSet = DataSet(parentDataSet, uncDatasetName, region)
dataSet = DataSet(parentDataSet, dataSetName, region)
pocaDataSet = DataSet(pocaParentDataSet, pocaDataSetName, region)
pocaDataSet_noDemDiff = DataSet(pocaParentDataSet,
pocaDataSetName.replace("_demDiff", ""),
region)
projections = [
'x', 'y', 'time', 'elev', 'powerdB', 'coh', 'demDiff', 'demDiffMad',
'swathFileId', 'Q_uStd'
]
filters = [{
'column': 'Q_uStd',
'op': 'lte',
'threshold': uncertainty_threshold
}, {
'column': 'powerdB',
'op': 'gte',
'threshold': powerdB
}, {
'column': 'coh',
'op': 'gte',
'threshold': coh
}, {
'column': 'inRegionMask',
'op': 'eq',
'threshold': 1.0
}]
filters_poca = [{
"column": "demDiff",
"op": "lte",
"threshold": pocaDemDiff
}, {
"column": "demDiff",
"op": "gte",
"threshold": -pocaDemDiff
}, {
'column': 'inRegionMask',
'op': 'eq',
'threshold': 1.0
}]
from_dt = datetime(pub_year, pub_month, 1, 0, 0, 0)
to_dt = from_dt + relativedelta(months=1) - timedelta(seconds=1)
bb = client.boundingBox(uncDataSet)
gridcells = client.gridCells(
uncDataSet,
BoundingBox(bb.minX, bb.maxX, bb.minY, bb.maxY, from_dt, to_dt))
proj4 = client.getProjection(uncDataSet).proj4
print("Number of Gridcells found to process {}".format(len(gridcells)))
process_start = datetime.now()
print("MinT={} MaxT={}".format(from_dt, to_dt))
#Create a shapefile index for each month
index = s.ShapeFileIndex(output_path, "THEM_POINT", proj4,
uncDataSet.region, from_dt)
for i, gc in enumerate(gridcells):
gc_start = datetime.now()
month_gc = BoundingBox(gc.minX, gc.maxX, gc.minY, gc.maxY, from_dt,
to_dt)
queryInfo = client.executeQuery(uncDataSet,
month_gc,
projections=projections,
filters=filters)
if queryInfo.status == "Success" and not queryInfo.resultFileName.startswith(
"Error"):
data = queryInfo.to_df
dataSwathStr = np.array(len(data), "S5")
dataSwathStr.fill("swath")
data["swathPoca"] = dataSwathStr
swath_file_ids = data['swathFileId'].unique()
pocaInfo = client.executeQuery(pocaDataSet,
gc,
filters=filters_poca)
pocaDf = pd.DataFrame()
if pocaInfo.status == "Success" and not pocaInfo.resultFileName.startswith(
"Error"):
pocaDf = pocaInfo.to_df
if len(pocaDf) > 0:
pocaStr = np.empty(len(pocaDf), "S5")
pocaStr.fill("poca")
pocaDf["swathPoca"] = pocaStr
poca_file_ids = pocaDf['swathFileId'].unique()
print("Poca points to include {}".format(len(pocaDf)))
data = pd.concat([data, pocaDf], sort=False)
print("Found {} data rows".format(len(data)))
if len(data) > 0:
results = client.getSwathNamesFromIds(dataSet, swath_file_ids)
if len(pocaDf) > 0:
try:
results.update(
client.getSwathNamesFromIds(
pocaDataSet_noDemDiff, poca_file_ids))
except KeyError as ex:
print(
"Exception caught while retrieving swathIds for data set {} file ids {}"
.format(pocaDataSet_noDemDiff, poca_file_ids))
raise KeyError(ex)
writePointProduct(output_path, dataSet, month_gc, data, proj4,
results, index)
client.releaseCacheHandle(pocaInfo.resultFileName)
else:
print("Grid Cells skipped X=[{}] Y=[{}] with message [{}] ".format(
gc.minX, gc.minY, queryInfo.status))
client.releaseCacheHandle(queryInfo.resultFileName)
index.close()
gc_elapsed = (datetime.now() - gc_start).total_seconds()
print('Processed [{}] grid cells. Took=[{}]s'.format(i + 1, gc_elapsed))
process_elapsed = (datetime.now() - process_start).total_seconds()
print("Took [{}s] to process".format(process_elapsed))
Created on Wed Nov 13 09:55:03 2019
@author: jon
"""
import MalardGDAL as mg
from MalardClient.MalardClient import MalardClient
from MalardClient.DataSet import DataSet
from MalardClient.BoundingBox import BoundingBox
from MalardClient.MaskFilter import MaskFilter
from datetime import datetime
client = MalardClient()
ds = DataSet("cryotempo", "swath_c", "greenland")
proj4 = client.getProjection(ds).proj4
print(proj4)
minX = 700000
minY = -2200000
cell_size = 130000
bbox = BoundingBox(minX, minX + cell_size, minY, minY + cell_size,
datetime(2011, 2, 1, 0, 0), datetime(2011, 5, 1, 0, 0))
## TODO: These need to be stored in Malard by DataSet and Type.
maskFilterIce = MaskFilter(
p_shapeFile="/data/puma1/scratch/cryotempo/masks/icesheets.shp")
def getDataSets(self, parentDataSet):
listOfDatasets = self.query.getDataSets(parentDataSet)
return [DataSet(parentDataSet, ds['name'], ds['region'] ) for ds in json.loads(listOfDatasets) ]
def __init__(self, minX, maxX, minY, maxY, logFile=None, notebook=False):
'''
:param minX:
:param maxX:
:param minY:
:param maxY:
:param logFile: if logfile is specified logger will write into file instead of the terminal
'''
if logFile is None:
logging.basicConfig(
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
else:
logging.basicConfig(
filename=logFile,
filemode='a',
format=
'%(asctime)s, %(threadName)s %(thread)d: %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
sys.excepthook = self.uncaughtErrorHandler
self.logger = logging.getLogger(__name__)
self.logger.info('Process started')
self.client = MalardClient(notebook=notebook)
self.query_async = AsyncDataSetQuery(
self.config('malardAsyncURL'),
self.config('malardEnvironmentName'), False)
self.minX = minX
self.maxX = maxX
self.minY = minY
self.maxY = maxY
assert (self.maxX - self.minX) == (self.maxY - self.minY)
self.size = maxX - minX
self.inputDataSet = DataSet(parentDs=self.config('parentDsName'),
dataSet=self.config('inputDataSet'),
region=self.config('region'))
self.parentDsName = self.config('parentDsName')
self.outputDataSet = self.config('outputDataSet')
self.region = self.config('region')
self.buffer = self.config('buffer')
self.projection = self.client.getProjection(self.inputDataSet).proj4
bbx = self.client.boundingBox(self.inputDataSet)
self.minT = bbx.minT
self.maxT = bbx.maxT
# masks
maskDataSet = self.config('maskDataSet')
query_sync = DataSetQuery(self.config('malardSyncURL'),
self.config('malardEnvironmentName'))
mGla = query_sync.getGridCellMask(self.parentDsName, maskDataSet,
'Glacier', self.region, self.minX,
self.minY, self.size)
self.maskDataSetFile = json.loads(mGla)['fileName']
df['x'] = x
df['y'] = y
return df
environmentName = 'DEVv2'
gridCellSize = 100000
resolution = 500
mask_prefix = "SARIN"
client = MalardClient(environmentName, True)
dataSet = DataSet('cryotempo', 'GRIS_BaselineC_Q2', 'greenland')
proj4 = client.getProjection(dataSet).proj4
mask = '/data/puma1/scratch/cryotempo/masks/icesheets.shp' if mask_prefix == "ICE" else '/data/puma1/scratch/cryotempo/sarinmasks/{}_Greenland.shp'.format(
mask_prefix)
tmpPath = '/home/jon/data/masks/'
bbox = client.boundingBox(dataSet)
gridCells = client.gridCells(dataSet, bbox)
for gc in gridCells:
data = interpolationGrid(gc.minX, gc.minY, gridCellSize, resolution)