class Monitor:
def __init__(self, preconditions, usecase, hosts, types, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
types,
logdir=log_dir,
simulation=simulation,
polling="job_status_callback.sh")
def execute(self, job):
"""
:type job: Job
"""
self.pm.execute(job.get_call(),
job.get_preconditions(),
job.get_postconditions(),
job.get_parameters(),
logprefix=job.get_name())
def wait_for_completion(self):
self.pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-idepix-meris', logdir='log', hosts=hosts, types=types)
for year in years:
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
(minDate, maxDate) = getMinMaxDate(year, month)
idepixName = 'meris-idepix-' + year + '-' + month
params = ['meris-idepix-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName)
#======================================================
pm.wait_for_completion()
class Monitor:
def __init__(self, preconditions, usecase, hosts, calls, log_dir, simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions, usecase, hosts, calls, log_dir=log_dir, simulation=simulation)
def execute(self, job):
"""
:type job: Job
"""
self.pm.execute(job.get_call(), job.get_preconditions(), job.get_postconditions(), job.get_parameters(),
log_prefix=job.get_name())
def wait_for_completion(self):
self.pm.wait_for_completion()
def wait_for_completion_and_terminate(self):
self.pm.wait_for_completion_and_terminate()
def run(self):
pm = PMonitor(inputs, request='round-robin2-matchups', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
sensorPointData = pointDataLocal[sensor]
calvalusPointData = calvalusPointDataRoot + '/' + sensor + '-' + basename(sensorPointData)
pm.execute('ingest-point-data.sh', [sensorPointData], [calvalusPointData], logprefix="ingest-point-data-" + sensor)
eodata = sensorEodata[sensor]
for processor in processors:
if sensor in processorConfig[processor]['processorName']:
params = ['round-robin-\${processor}-\${sensor}-matchup.xml',
'sensor', sensor,
'processor', processor,
'eodataRoot', eodata['root'],
'eodataSuffix', eodata['suffix'],
'processorBundleName', processorConfig[processor]['processorBundleName'],
'processorBundleVersion', processorConfig[processor]['processorBundleVersion'],
'processorName', processorConfig[processor]['processorName'][sensor],
'pointData', calvalusPointData,
'namenode', namenode
]
matchupName = 'round-robin-matchup-' + processor + '-' + sensor
pm.execute('template-step.py', [calvalusPointData], [matchupName], parameters=params, logprefix=matchupName)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-modis', logdir='log', hosts=hosts, types=types)
modisBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
modisFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
modisDailyBSName = 'modis-daily-bs-' + str(singleDay)
params = ['modis-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'day', '%02d' % (singleDay.day) ]
pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName)
modisFormatInputs.append(modisDailyBSName)
if year >= '2003' and year <= '2007':
modisBiasInputs.append(modisDailyBSName)
modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-modis-daily-bs' ]
# debug only daily binned band-shifted
pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName)
modisBiasMapName = 'modis-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'modis', \
'sensorMarker', '11' ]
pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName)
modisBiasFormatName = 'modis-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'modis/bias-map-parts/part-*', \
'outputPath', 'modis/bias-map-netcdf-geo', \
'prefix', 'OC-modis-bias' ]
# debug only bias map
pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName)
#======================================================
pm.wait_for_completion()
def run(self):
allLocalPointData = []
for sensor in sensors:
for station in sensorStations[sensor]:
allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor][station])
print allLocalPointData
pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
for station in sensorStations[sensor]:
localPointData = localPointDataRoot + "/" + pointData[sensor][station]
calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor][station]
handle = "ingest-point-data-" + sensor + "-" + station
pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle)
#======================================================
for vc in gainsWithVC[sensor].keys():
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', station,
'sensor', sensor,
'vc', vc,
'calib', '"'+gainsWithVC[sensor][vc]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc
]
handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle)
#======================================================
vc = 'withoutvc'
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', station,
'sensor', sensor,
'vc', vc,
'calib', '"'+gainsWithoutVC[sensor]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc
]
handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-merging', logdir='log', hosts=hosts, types=types)
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
formatMergedInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
mergedName = 'merged-daily-' + str(singleDay)
params = ['sensor-merging-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['input'], [mergedName], parameters=params, logprefix=mergedName)
formatMergedInputs.append(mergedName)
mergedFormatName = 'merged-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \
'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-merged-daily' ]
pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='round-robin2-area', logdir='log', hosts=hosts, types=types)
for region in regions:
for processor in processors:
nameProcess = 'round-robin-process-' + processor + '-' + region
params = ['round-robin-\${processor}-\${region}-process.xml',
'processor', processor,
'region', region,
'processorBundleName', processorConfig[processor]['processorBundleName'],
'processorBundleVersion', processorConfig[processor]['processorBundleVersion'],
'processorName', processorConfig[processor]['processorName']
]
pm.execute('template-step.py', inputs, [nameProcess], parameters=params, logprefix=nameProcess)
if processorConfig[processor]['needsFormatting']:
nameFormat = 'round-robin-format-' + processor + '-' + region
params = ['round-robin-\${processor}-\${region}-format.xml',
'processor', processor,
'region', region
]
pm.execute('template-step.py', [nameProcess], [nameFormat], parameters=params, logprefix=nameFormat)
#======================================================
pm.wait_for_completion()
# cloud product e.g. MOD35_L2.A2015196.1855.061.2017321064215.hdf
modisCloudMaskFiles = fnmatch.filter(
os.listdir(modisCloudMaskRootDir +
'/' + year + '/' +
month + '/' +
str(day).zfill(2)),
fileFilter)
modisCloudMaskPath = modisCloudMaskRootDir + '/' + year + '/' + month + '/' + str(
day).zfill(
2
) + '/' + modisCloudMaskFiles[0]
if os.path.exists(modisCloudMaskPath):
#print( 'Daily product: ' + str(is_daily_product(modisCloudMaskPath)) + ' --> ' + modisCloudMaskPath)
# =============== Merge MODIS L1b with ERA-INTERIM, then TCWV from Idepix-ERA-INTERIM merge product =======================
l1bEraFile = l1bFileBase + '_l1b-era-interim.nc'
m.execute(
'wvcci-l2-tcwv-modis-step_4.sh',
['dummy'], [l1bEraFile],
parameters=[
l1bPath, l1bFiles[index],
modisCloudMaskPath, year,
month, day, hhmm,
wvcciRootDir
])
m.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request="oc-seawifs", logdir="log", hosts=hosts, types=types)
seawifsBiasInputs = []
for year in years:
months = monthsAll
if year == "2002":
months = months2002
elif year == "2012":
months = months2012
for month in months:
seawifsFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
seawifsDailyBSName = "seawifs-daily-bs-" + str(singleDay)
params = [
"seawifs-daily-bs-\${date}.xml",
"date",
str(singleDay),
"year",
year,
"month",
month,
"day",
"%02d" % (singleDay.day),
]
pm.execute(
"template-step.py",
["SEAWIFS_L3_daily"],
[seawifsDailyBSName],
parameters=params,
logprefix=seawifsDailyBSName,
)
seawifsFormatInputs.append(seawifsDailyBSName)
if year >= "2003" and year <= "2007":
seawifsBiasInputs.append(seawifsDailyBSName)
seawifsFormatBSName = "seawifs-daily-bs-format-" + month + "-" + year
params = [
"l3format-\${prefix}-\${date}.xml",
"date",
month + "-" + year,
"inputPath",
"seawifs/daily-bs/" + year + "/" + month + "/????-??-??/part-*",
"outputPath",
"seawifs/daily-bs/" + year + "/" + month + "/netcdf-geo",
"prefix",
"OC-seawifs-daily-bs",
]
# debug only daily binned band-shifted
pm.execute(
"template-step.py",
seawifsFormatInputs,
[seawifsFormatBSName],
parameters=params,
logprefix=seawifsFormatBSName,
)
seawifsBiasMapName = "seawifs-bias-map"
params = ["bias-map-\${sensor}.xml", "sensor", "seawifs", "sensorMarker", "12"]
pm.execute(
"template-step.py", seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName
)
seawifsBiasFormatName = "seawifs-bias-map-format"
params = [
"l3format-\${prefix}-\${date}.xml",
"date",
"5years",
"inputPath",
"seawifs/bias-map-parts/part-*",
"outputPath",
"seawifs/bias-map-netcdf-geo",
"prefix",
"OC-seawifs-bias",
]
# debug only bias map
pm.execute(
"template-step.py",
[seawifsBiasMapName],
[seawifsBiasFormatName],
parameters=params,
logprefix=seawifsBiasFormatName,
)
# ======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-complete', logdir='log', hosts=hosts, types=types)
#########################################################################################
merisBiasInputs = []
for year in years:
merisDailyTemplate = 'meris-daily-useIdepix-QAA-\${date}.xml'
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-QAA-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
merisDailyFormatInputs = []
merisDailyBSFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
polymerName = 'polymer-' + str(minDate)
params = ['meris-polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
merisDailyName = 'meris-daily-' + str(singleDay)
params = [merisDailyTemplate, \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName)
merisDailyFormatInputs.append(merisDailyName)
merisDailyBSName = 'meris-daily-bs-' + str(singleDay)
params = ['meris-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName)
merisDailyBSFormatInputs.append(merisDailyBSName)
if year >= '2003' and year <= '2007':
merisBiasInputs.append(merisDailyBSName)
merisFormatName = 'meris-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \
'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily' ]
#pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName)
merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily-bs' ]
#pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName)
merisBiasMapName = 'meris-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'meris', \
'sensorMarker', '10' ]
pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName)
merisBiasFormatName = 'meris-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'meris/bias-map-parts/part-*', \
'outputPath', 'meris/bias-map-netcdf-geo', \
'prefix', 'OC-meris-bias' ]
#pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName)
######################################################################
modisBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
modisFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
modisDailyBSName = 'modis-daily-bs-' + str(singleDay)
params = ['modis-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'doy', '%03d' % (singleDay.timetuple().tm_yday) ]
pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName)
modisFormatInputs.append(modisDailyBSName)
if year >= '2003' and year <= '2007':
modisBiasInputs.append(modisDailyBSName)
modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-modis-daily-bs' ]
#pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName)
modisBiasMapName = 'modis-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'modis', \
'sensorMarker', '11' ]
pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName)
modisBiasFormatName = 'modis-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'modis/bias-map-parts/part-*', \
'outputPath', 'modis/bias-map-netcdf-geo', \
'prefix', 'OC-modis-bias' ]
#pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName)
######################################################################
seawifsBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
seawifsFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
seawifsDailyBSName = 'seawifs-daily-bs-' + str(singleDay)
params = ['seawifs-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'doy', '%03d' % (singleDay.timetuple().tm_yday) ]
pm.execute('template-step.py', ['SEAWIFS_L3_daily'], [seawifsDailyBSName], parameters=params, logprefix=seawifsDailyBSName)
seawifsFormatInputs.append(seawifsDailyBSName)
if year >= '2003' and year <= '2007':
seawifsBiasInputs.append(seawifsDailyBSName)
seawifsFormatBSName = 'seawifs-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'seawifs/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'seawifs/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-seawifs-daily-bs' ]
#pm.execute('template-step.py', seawifsFormatInputs, [seawifsFormatBSName], parameters=params, logprefix=seawifsFormatBSName)
seawifsBiasMapName = 'seawifs-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'seawifs', \
'sensorMarker', '12' ]
pm.execute('template-step.py', seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName)
seawifsBiasFormatName = 'seawifs-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'seawifs/bias-map-parts/part-*', \
'outputPath', 'seawifs/bias-map-netcdf-geo', \
'prefix', 'OC-seawifs-bias' ]
#pm.execute('template-step.py', [seawifsBiasMapName], [seawifsBiasFormatName], parameters=params, logprefix=seawifsBiasFormatName)
######################################################################
mergedInput = [merisBiasMapName, modisBiasMapName, seawifsBiasMapName]
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
formatMergedInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
mergedName = 'merged-daily-' + str(singleDay)
params = ['sensor-merging-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', mergedInput, [mergedName], parameters=params, logprefix=mergedName)
formatMergedInputs.append(mergedName)
mergedFormatName = 'merged-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \
'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-merged-daily' ]
pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName)
#======================================================
pm.wait_for_completion()
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1'
brdfMosaicDir = gaRootDir + '/Mosaic/brdf'
#### Upscaling/Mosaicing ####
m = PMonitor([gaRootDir],
request='ga-l3-mosaic',
logdir='log',
hosts=[('localhost',16)],
types=[('ga-l3-mosaic-step.sh',16)])
## BRDF mosaicking
for year in years:
for doy in doys:
for snowMode in snowModes:
for resolution in resolutions:
brdfMosaicDir = gaRootDir + '/Mosaic/brdf/' + snowMode + '/' + resolution
m.execute('ga-l3-mosaic-step.sh', [gaRootDir], [brdfMosaicDir], parameters=[year,doy,snowMode,resolution,gaRootDir,beamDir])
## BRDF mosaic --> Albedo mosaic
# TODO
# wait for processing to complete
m.wait_for_completion()
#####################################################################################################
#os.system('date')
def run(self):
allLocalPointData = []
for sensor in sensors:
allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor])
pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
localPointData = localPointDataRoot + "/" + pointData[sensor]
calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor]
handle = "ingest-point-data-" + sensor
pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle)
#======================================================
for vc in gains[sensor].keys():
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', 'global',
'sensor', sensor,
'vc', vc,
'calib', '"'+gains[sensor][vc]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis3/' + sensor + "-" + vc
]
handleMA = 'vc-analysis-ana-' + sensor + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA)
params = ['/calvalus/projects/vc-analysis3/' + sensor + "-" + vc,
'vc-analysis-' + sensor + "-" + vc
]
handleCP = 'vc-analysis-ana-' + sensor + "-" + vc + '-copy'
pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP)
params = ['ftp.brockmann-consult.de',
'oc-cci',
'vc-analysis-' + sensor + "-" + vc + '.tar.gz',
ftpDirs[sensor][vc]
]
handleFTP = 'vc-analysis-ana-' + sensor + "-" + vc + '-ftp'
pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP)
#======================================================
params = ['ma_vc-analysis-ana-idepix-' + sensor + '.xml',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis3/' + sensor + "-idepix"
]
handleMA = 'vc-analysis-ana-idepix-' + sensor
pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA)
params = ['/calvalus/projects/vc-analysis3/' + sensor + "-idepix",
'vc-analysis-' + sensor + "-idepix"
]
handleCP = 'vc-analysis-ana-idepix-' + sensor + '-copy'
pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP)
params = ['ftp.brockmann-consult.de',
'oc-cci',
'vc-analysis-' + sensor + '-idepix.tar.gz',
'VC/Gain_validation_2014-11-05'
]
handleFTP = 'vc-analysis-ana-' + sensor + '-ftp'
pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='idepix-comparison', logdir='log', hosts=hosts, types=types)
for (month, firstDay, lastDay) in TIME_RANGES:
minDate = datetime.date(year, month, firstDay)
maxDate = datetime.date(year, month, lastDay)
polymerTag = 'polymer-' + str(minDate)
params = ['polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerTag], parameters=params, logprefix=polymerTag)
idepixOldTag = 'idepix-old-' + str(minDate)
params = ['idepix-\${variant}-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'old', \
'processorBundleName', 'beam-idepix-cc', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixOldTag], parameters=params, logprefix=idepixOldTag)
idepixNewTag = 'idepix-new-' + str(minDate)
params = ['idepix-\${variant}-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'new', \
'processorBundleName', 'beam-idepix-oc', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixNewTag], parameters=params, logprefix=idepixNewTag)
#idepixNewQlTag = 'idepix-new-ql-' + str(minDate)
#params = ['idepix-ql-\${variant}-\${year}-\${month}.xml', \
# 'minDate', str(minDate), \
# 'maxDate', str(maxDate), \
# 'variant', 'new', \
# 'year', str(year), \
# 'month', str(month) ]
#pm.execute('template-step.py', [idepixNewTag, polymerTag], [idepixNewQlTag], parameters=params, logprefix=idepixNewQlTag)
mergedL2Tag = 'merged-l2-' + str(minDate)
params = ['merge-polymer-and-two-idepix-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'new', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', [idepixNewTag, idepixOldTag, polymerTag], [mergedL2Tag], parameters=params, logprefix=mergedL2Tag)
mergedL2FormatTag = 'merge-l2-format-' + str(minDate)
params = ['l2-format-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', [mergedL2Tag], [mergedL2FormatTag], parameters=params, logprefix=mergedL2FormatTag)
for singleDay in dateRange(minDate, maxDate):
merisDailyNewName = 'meris-daily-new-' + str(singleDay)
merisDailyParams = ['daily-new-\${date}.xml', \
'date', str(singleDay), \
'year', '%4d' % (singleDay.year), \
'month', str(singleDay.month) ]
pm.execute('template-step.py', [polymerTag], [merisDailyNewName], parameters=merisDailyParams, logprefix=merisDailyNewName)
merisDailyName = 'meris-daily-old-' + str(singleDay)
merisDailyParams = ['daily-old-\${date}.xml', \
'date', str(singleDay), \
'year', '%4d' % (singleDay.year), \
'month', str(singleDay.month) ]
pm.execute('template-step.py', [polymerTag], [merisDailyName], parameters=merisDailyParams, logprefix=merisDailyName)
# quicklooks L2 polymer + old idepix ?)
# DONE # quicklooks L2 polymer + new idepix
# DONE # merge L2 polymer + old + new idepix --> Format to DIMAP
# daily L3 polymer + new idepix + quicklooks
# daily L3 polymer + old idepix + quicklooks)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-meris', logdir='log', hosts=hosts, types=types)
merisBiasInputs = []
for year in years:
merisDailyTemplate = 'meris-daily-useIdepix-\${date}.xml'
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
merisDailyFormatInputs = []
merisDailyBSFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
idepixName = 'idepix-' + str(minDate)
params = ['meris-idepix-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName)
polymerName = 'polymer-' + str(minDate)
params = ['meris-polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
merisDailyName = 'meris-daily-' + str(singleDay)
params = [merisDailyTemplate, \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [idepixName, polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName)
merisDailyFormatInputs.append(merisDailyName)
merisDailyBSName = 'meris-daily-bs-' + str(singleDay)
params = ['meris-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName)
merisDailyBSFormatInputs.append(merisDailyBSName)
if year >= '2003' and year <= '2007':
merisBiasInputs.append(merisDailyBSName)
merisFormatName = 'meris-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \
'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily' ]
# (debug only / user product ?) daily binned
pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName)
merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily-bs' ]
# debug only daily binned band-shifted
pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName)
merisBiasMapName = 'meris-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'meris', \
'sensorMarker', '10' ]
pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName)
merisBiasFormatName = 'meris-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'meris/bias-map-parts/part-*', \
'outputPath', 'meris/bias-map-netcdf-geo', \
'prefix', 'OC-meris-bias' ]
# debug only bias map
pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName)
#======================================================
pm.wait_for_completion()
