def run(self):
pm = PMonitor(inputs, request='oc-idepix-meris', logdir='log', hosts=hosts, types=types)
for year in years:
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
(minDate, maxDate) = getMinMaxDate(year, month)
idepixName = 'meris-idepix-' + year + '-' + month
params = ['meris-idepix-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName)
#======================================================
pm.wait_for_completion()
class Monitor:
def __init__(self, preconditions, usecase, hosts, types, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
types,
logdir=log_dir,
simulation=simulation,
polling="job_status_callback.sh")
def execute(self, job):
"""
:type job: Job
"""
self.pm.execute(job.get_call(),
job.get_preconditions(),
job.get_postconditions(),
job.get_parameters(),
logprefix=job.get_name())
def wait_for_completion(self):
self.pm.wait_for_completion()
def __init__(self, parameters, types):
PMonitor.__init__(self, ['none', parameters['data_root']],
request=parameters['requestName'],
hosts=[('localhost', 10)],
types=types,
logdir=parameters['log_dir'],
simulation='simulation' in parameters
and parameters['simulation'])
self._tasks_progress = {}
self._lower_script_progress = {}
self._upper_script_progress = {}
self._processor_logs = {}
self._pids = {}
self._to_be_cancelled = []
self._cancelled = []
def __init__(self, preconditions, usecase, hosts, calls, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
calls,
log_dir=log_dir,
simulation=simulation)
def run(self):
pm = PMonitor(inputs, request='round-robin2-matchups', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
sensorPointData = pointDataLocal[sensor]
calvalusPointData = calvalusPointDataRoot + '/' + sensor + '-' + basename(sensorPointData)
pm.execute('ingest-point-data.sh', [sensorPointData], [calvalusPointData], logprefix="ingest-point-data-" + sensor)
eodata = sensorEodata[sensor]
for processor in processors:
if sensor in processorConfig[processor]['processorName']:
params = ['round-robin-\${processor}-\${sensor}-matchup.xml',
'sensor', sensor,
'processor', processor,
'eodataRoot', eodata['root'],
'eodataSuffix', eodata['suffix'],
'processorBundleName', processorConfig[processor]['processorBundleName'],
'processorBundleVersion', processorConfig[processor]['processorBundleVersion'],
'processorName', processorConfig[processor]['processorName'][sensor],
'pointData', calvalusPointData,
'namenode', namenode
]
matchupName = 'round-robin-matchup-' + processor + '-' + sensor
pm.execute('template-step.py', [calvalusPointData], [matchupName], parameters=params, logprefix=matchupName)
#======================================================
pm.wait_for_completion()
def __init__(self, preconditions, usecase, hosts, types, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
types,
logdir=log_dir,
simulation=simulation,
polling="job_status_callback.sh")
def __init__(self, preconditions, usecase, hosts, calls, log_dir, simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions, usecase, hosts, calls, log_dir=log_dir, simulation=simulation)
class Monitor:
def __init__(self, preconditions, usecase, hosts, calls, log_dir, simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions, usecase, hosts, calls, log_dir=log_dir, simulation=simulation)
def execute(self, job):
"""
:type job: Job
"""
self.pm.execute(job.get_call(), job.get_preconditions(), job.get_postconditions(), job.get_parameters(),
log_prefix=job.get_name())
def wait_for_completion(self):
self.pm.wait_for_completion()
def wait_for_completion_and_terminate(self):
self.pm.wait_for_completion_and_terminate()
def _run_step(self, task_id, host, command, output_paths, log_prefix,
async_):
"""
Executes command on host, collects output paths if any, returns exit code
"""
wd = self._prepare_working_dir(task_id)
process = PMonitor._start_processor(command, host, wd)
self._pids[command] = process.pid
self._trace_processor_output(output_paths, process, task_id, command,
wd, log_prefix, async_)
process.stdout.close()
code = process.wait()
return code
def run(self):
allLocalPointData = []
for sensor in sensors:
for station in sensorStations[sensor]:
allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor][station])
print allLocalPointData
pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
for station in sensorStations[sensor]:
localPointData = localPointDataRoot + "/" + pointData[sensor][station]
calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor][station]
handle = "ingest-point-data-" + sensor + "-" + station
pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle)
#======================================================
for vc in gainsWithVC[sensor].keys():
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', station,
'sensor', sensor,
'vc', vc,
'calib', '"'+gainsWithVC[sensor][vc]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc
]
handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle)
#======================================================
vc = 'withoutvc'
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', station,
'sensor', sensor,
'vc', vc,
'calib', '"'+gainsWithoutVC[sensor]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc
]
handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='oc-merging', logdir='log', hosts=hosts, types=types)
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
formatMergedInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
mergedName = 'merged-daily-' + str(singleDay)
params = ['sensor-merging-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['input'], [mergedName], parameters=params, logprefix=mergedName)
formatMergedInputs.append(mergedName)
mergedFormatName = 'merged-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \
'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-merged-daily' ]
pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName)
#======================================================
pm.wait_for_completion()
def run(self):
pm = PMonitor(inputs, request='round-robin2-area', logdir='log', hosts=hosts, types=types)
for region in regions:
for processor in processors:
nameProcess = 'round-robin-process-' + processor + '-' + region
params = ['round-robin-\${processor}-\${region}-process.xml',
'processor', processor,
'region', region,
'processorBundleName', processorConfig[processor]['processorBundleName'],
'processorBundleVersion', processorConfig[processor]['processorBundleVersion'],
'processorName', processorConfig[processor]['processorName']
]
pm.execute('template-step.py', inputs, [nameProcess], parameters=params, logprefix=nameProcess)
if processorConfig[processor]['needsFormatting']:
nameFormat = 'round-robin-format-' + processor + '-' + region
params = ['round-robin-\${processor}-\${region}-format.xml',
'processor', processor,
'region', region
]
pm.execute('template-step.py', [nameProcess], [nameFormat], parameters=params, logprefix=nameFormat)
#======================================================
pm.wait_for_completion()
for year in years:
#for day in range(122): # 20080101 - 20080430
#for day in range(15,17): # 20080115, 20080116
#for day in range(1,58): # Jan, Feb
for day in days:
doy = str(day).zfill(3)
#inputs.append('/calvalus/projects/cawa/idepix/modis-nc/' + year + '/' + doy)
inputs.append('/calvalus/projects/cawa/idepix/modis/' + year + '/' +
doy)
hosts = [('localhost', 16)]
types = [('tcwv-step.sh', 4), ('tcwv-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='tcwv_modis', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
#for day in range(122): # 20080101 - 20080430
#for day in range(15,17): # 20080115, 20080116
#for day in range(1,58): # Jan, Feb
for day in days:
doy = str(day).zfill(3)
current_date = base_date + timedelta(day - 1)
datestring = current_date.strftime("%Y-%m-%d")
pm.execute('tcwv-step.sh', \
[ '/calvalus/projects/cawa/idepix/modis/' + year + '/' + doy ], \
[ '/calvalus/projects/cawa/tcwv/modis/' + year + '/' + doy ], \
parameters=['MODIS', datestring, datestring])
else:
#return 2 # test!!
return 31
######################## Idepix --> TCWV: ###########################
wvcciRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/WvcciTest'
idepixRootDir = wvcciRootDir + '/Idepix'
snapDir = '/group_workspaces/cems2/qa4ecv/vol4/software/snap'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l2-tcwv',
logdir='log',
hosts=[('localhost', 128)],
types=[('wvcci-l2-tcwv-step.sh', 128)])
for year in years:
for sensor in sensors:
for month in getMonth(year):
if os.path.exists(idepixRootDir + '/' + sensor + '/' + year + '/' +
month):
for day in range(1, getNumMonthDays(year, int(month)) + 1):
tcwvL2Dir = wvcciRootDir + '/Tcwv/' + sensor + '/' + year + '/' + month + '/' + str(
day).zfill(2)
if os.path.exists(idepixRootDir + '/' + sensor + '/' +
year + '/' + month + '/' +
str(day).zfill(2)):
def run(self):
allLocalPointData = []
for sensor in sensors:
allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor])
pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types)
for sensor in sensors:
localPointData = localPointDataRoot + "/" + pointData[sensor]
calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor]
handle = "ingest-point-data-" + sensor
pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle)
#======================================================
for vc in gains[sensor].keys():
params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml',
'station', 'global',
'sensor', sensor,
'vc', vc,
'calib', '"'+gains[sensor][vc]+'"',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis3/' + sensor + "-" + vc
]
handleMA = 'vc-analysis-ana-' + sensor + "-" + vc
pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA)
params = ['/calvalus/projects/vc-analysis3/' + sensor + "-" + vc,
'vc-analysis-' + sensor + "-" + vc
]
handleCP = 'vc-analysis-ana-' + sensor + "-" + vc + '-copy'
pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP)
params = ['ftp.brockmann-consult.de',
'oc-cci',
'vc-analysis-' + sensor + "-" + vc + '.tar.gz',
ftpDirs[sensor][vc]
]
handleFTP = 'vc-analysis-ana-' + sensor + "-" + vc + '-ftp'
pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP)
#======================================================
params = ['ma_vc-analysis-ana-idepix-' + sensor + '.xml',
'calvalusPointData', calvalusPointData,
'output', '/calvalus/projects/vc-analysis3/' + sensor + "-idepix"
]
handleMA = 'vc-analysis-ana-idepix-' + sensor
pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA)
params = ['/calvalus/projects/vc-analysis3/' + sensor + "-idepix",
'vc-analysis-' + sensor + "-idepix"
]
handleCP = 'vc-analysis-ana-idepix-' + sensor + '-copy'
pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP)
params = ['ftp.brockmann-consult.de',
'oc-cci',
'vc-analysis-' + sensor + '-idepix.tar.gz',
'VC/Gain_validation_2014-11-05'
]
handleFTP = 'vc-analysis-ana-' + sensor + '-ftp'
pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP)
#======================================================
pm.wait_for_completion()
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
#years = [ '2008' ]
years = [ '2005' ]
inputs = []
for year in years:
for month in getMonths(year):
inputs.append('/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month)
hosts = [('localhost',16)]
types = [('tcwv-step.sh',16), ('format-step.sh',2)]
pm = PMonitor(inputs, \
request='tcwv', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('tcwv-step.sh', \
[ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv-fullmission/' + year + '/' + month ], \
parameters=[str(minDate), str(maxDate)])
pm.execute('format-step.sh', \
[ '/calvalus/projects/cawa/tcwv-fullmission/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv-fullmission-nc/' + year + '/' + month ], \
parameters=[str(minDate), str(maxDate)])
def run(self):
pm = PMonitor(inputs, request='oc-complete', logdir='log', hosts=hosts, types=types)
#########################################################################################
merisBiasInputs = []
for year in years:
merisDailyTemplate = 'meris-daily-useIdepix-QAA-\${date}.xml'
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-QAA-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
merisDailyFormatInputs = []
merisDailyBSFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
polymerName = 'polymer-' + str(minDate)
params = ['meris-polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
merisDailyName = 'meris-daily-' + str(singleDay)
params = [merisDailyTemplate, \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName)
merisDailyFormatInputs.append(merisDailyName)
merisDailyBSName = 'meris-daily-bs-' + str(singleDay)
params = ['meris-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName)
merisDailyBSFormatInputs.append(merisDailyBSName)
if year >= '2003' and year <= '2007':
merisBiasInputs.append(merisDailyBSName)
merisFormatName = 'meris-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \
'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily' ]
#pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName)
merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily-bs' ]
#pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName)
merisBiasMapName = 'meris-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'meris', \
'sensorMarker', '10' ]
pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName)
merisBiasFormatName = 'meris-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'meris/bias-map-parts/part-*', \
'outputPath', 'meris/bias-map-netcdf-geo', \
'prefix', 'OC-meris-bias' ]
#pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName)
######################################################################
modisBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
modisFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
modisDailyBSName = 'modis-daily-bs-' + str(singleDay)
params = ['modis-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'doy', '%03d' % (singleDay.timetuple().tm_yday) ]
pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName)
modisFormatInputs.append(modisDailyBSName)
if year >= '2003' and year <= '2007':
modisBiasInputs.append(modisDailyBSName)
modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-modis-daily-bs' ]
#pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName)
modisBiasMapName = 'modis-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'modis', \
'sensorMarker', '11' ]
pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName)
modisBiasFormatName = 'modis-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'modis/bias-map-parts/part-*', \
'outputPath', 'modis/bias-map-netcdf-geo', \
'prefix', 'OC-modis-bias' ]
#pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName)
######################################################################
seawifsBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
seawifsFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
seawifsDailyBSName = 'seawifs-daily-bs-' + str(singleDay)
params = ['seawifs-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'doy', '%03d' % (singleDay.timetuple().tm_yday) ]
pm.execute('template-step.py', ['SEAWIFS_L3_daily'], [seawifsDailyBSName], parameters=params, logprefix=seawifsDailyBSName)
seawifsFormatInputs.append(seawifsDailyBSName)
if year >= '2003' and year <= '2007':
seawifsBiasInputs.append(seawifsDailyBSName)
seawifsFormatBSName = 'seawifs-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'seawifs/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'seawifs/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-seawifs-daily-bs' ]
#pm.execute('template-step.py', seawifsFormatInputs, [seawifsFormatBSName], parameters=params, logprefix=seawifsFormatBSName)
seawifsBiasMapName = 'seawifs-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'seawifs', \
'sensorMarker', '12' ]
pm.execute('template-step.py', seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName)
seawifsBiasFormatName = 'seawifs-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'seawifs/bias-map-parts/part-*', \
'outputPath', 'seawifs/bias-map-netcdf-geo', \
'prefix', 'OC-seawifs-bias' ]
#pm.execute('template-step.py', [seawifsBiasMapName], [seawifsBiasFormatName], parameters=params, logprefix=seawifsBiasFormatName)
######################################################################
mergedInput = [merisBiasMapName, modisBiasMapName, seawifsBiasMapName]
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
formatMergedInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 9)
for singleDay in dateRange(minDate, maxDate):
mergedName = 'merged-daily-' + str(singleDay)
params = ['sensor-merging-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', mergedInput, [mergedName], parameters=params, logprefix=mergedName)
formatMergedInputs.append(mergedName)
mergedFormatName = 'merged-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \
'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-merged-daily' ]
pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName)
#======================================================
pm.wait_for_completion()
import calendar
from pmonitor import PMonitor
start = '2015-06-25'
stop = '2016-11-30'
period = str((datetime.datetime.strptime(stop, '%Y-%m-%d').date() -
datetime.datetime.strptime(start, '%Y-%m-%d').date()).days + 1)
idepix_dir = '/calvalus/projects/urban-tep/s2-idepix/germany '
timescan_seq_dir = '/calvalus/projects/urban-tep/msi-germany-'
timescan_dir = '/calvalus/projects/urban-tep/msi-germany-20m'
inputs = [ idepix_dir ]
pm = PMonitor(inputs, request='germany',
types=[('msi-timescan',4),('msi-format',7)],
hosts=[('localhost',12)], logdir='log', script='pstep.py', simulation=False)
for lat in range(47,55):
for lon in range(6,15):
#for lat in range(47,49):
# for lon in range(6,8):
polygon = "POLYGON(({0} {2},{0} {3},{1} {3},{1} {2},{0} {2}))".format(lon,lon+1, lat,lat+1)
region = "h{0:03d}v{1:03d}".format(lon+180, 90-lat-1)
pm.execute('msi-timescan', [ idepix_dir ], [ timescan_seq_dir+region ],
[ 'polygon', "'{0}'".format(polygon),
'start', start, 'stop', stop, 'period', period, 'region', region ])
pm.execute('msi-format', [ timescan_seq_dir+region ], [ timescan_dir ],
[ 'region', region, 'polygon', "'{0}'".format(polygon) ])
pm.wait_for_completion()
#years = [ '2002' ]
inputs = []
for year in years:
for month in getMonths(year):
#inputs.append('/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month)
inputs.append('/calvalus/projects/cawa/idepix/meris-nc/' + year + '/' +
month)
#inputs.append('/calvalus/projects/cawa/idepix/meris/' + year + '/' + month)
hosts = [('localhost', 16)]
types = [('tcwv_v10-step.sh', 4), ('tcwv-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='tcwv_meris_v10', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
#(minDate, maxDate) = ('2008-02-19', '2008-02-20')
pm.execute('tcwv_v10-step.sh', \
[ '/calvalus/projects/cawa/idepix/meris-nc/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('tcwv-format-step.sh', \
[ '/calvalus/projects/cawa/tcwv/meris/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv/meris-nc/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
return allMonths
######################## L1b --> Idepix --> IdepixEraInterim --> TCWV: ###########################
wvcciRootDir = '/gws/nopw/j04/esacci_wv/odanne/WvcciRoot'
eraInterimRootDir = wvcciRootDir + '/auxiliary/era-interim-t2m-mslp-tcwv-u10-v10'
snapDir = '/gws/nopw/j04/esacci_wv/software/snap'
inputs = ['dummy']
# NEW PMonitor version, MB/TB Nov 2018:
m = PMonitor(inputs,
request='wvcci-l2-tcwv-modis-chain',
logdir='log',
hosts=[('localhost', 384)],
types=[('wvcci-l2-idepix-modis-step.sh', 128),
('wvcci-l2-tcwv-modis-step.sh', 256)],
polling="job_status_callback.sh")
for year in years:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
modisCloudMaskRootDir = wvcciRootDir + '/ModisCloudMask/MOD35_L2'
for month in getMonth(year):
if os.path.exists(l1bRootDir + '/' + year + '/' + month):
numMonthDays = monthrange(int(year), int(month))[1]
#for day in days:
# use notation as in 'lakes_products.py'
regions = ["Lake-Balaton"]
# regions = ['Lake-Bogoria', 'Lake-Elmenteita', 'Lake-Nakuru', 'Lake-Tuusulanjarvi', 'Lake-Ulemiste']
DIVERSITY_INST_DIR = os.environ["DIVERSITY_INST"]
# before starting, check if WKT files are available
# From now on we always use the full polygons (*.shape) rather than just rectangular boxes.
for region in regions:
boxWktFile = DIVERSITY_INST_DIR + "/wkt/" + region + ".shape"
if not os.access(boxWktFile, os.R_OK):
raise IOError("Unable to access " + boxWktFile)
inputs = ["dummy"]
hosts = [("localhost", 8)]
pm = PMonitor(inputs, request="geo_child_products", logdir="log", hosts=hosts, script="template.py")
BASE = "/calvalus/projects/diversity/prototype/"
for region in regions:
boxWktFile = "wkt/" + region + ".shape"
for year in years:
for month in getMonth(year):
geoChildDir = BASE + region + "/l1-child/" + year + "/" + month
l1_geo_name = "l1_geo-" + year + "-" + month + "-" + region
params = [
"year",
year,
"month",
month,
"region",
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
#years = [ '2008' ]
years = [ '2005' ]
inputs = []
for year in years:
for month in getMonths(year):
inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
hosts = [('localhost',16)]
types = [('idepix-step.sh',16), ('format-step.sh',2)]
pm = PMonitor(inputs, \
request='idepix', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('idepix-step.sh', \
[ '/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
parameters=[str(minDate), str(maxDate)])
#pm.execute('format-step.sh', \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission-nc/' + year + '/' + month ], \
# parameters=[str(minDate), str(maxDate)])
def run(self):
pm = PMonitor(inputs, request='oc-modis', logdir='log', hosts=hosts, types=types)
modisBiasInputs = []
for year in years:
months = monthsAll
if year == '2002':
months = months2002
elif year == '2012':
months = months2012
for month in months:
modisFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
modisDailyBSName = 'modis-daily-bs-' + str(singleDay)
params = ['modis-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ,\
'day', '%02d' % (singleDay.day) ]
pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName)
modisFormatInputs.append(modisDailyBSName)
if year >= '2003' and year <= '2007':
modisBiasInputs.append(modisDailyBSName)
modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-modis-daily-bs' ]
# debug only daily binned band-shifted
pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName)
modisBiasMapName = 'modis-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'modis', \
'sensorMarker', '11' ]
pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName)
modisBiasFormatName = 'modis-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'modis/bias-map-parts/part-*', \
'outputPath', 'modis/bias-map-netcdf-geo', \
'prefix', 'OC-modis-bias' ]
# debug only bias map
pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName)
#======================================================
pm.wait_for_completion()
return months
#################################################################################################
years = ['2010']
sensor = 'meris'
gaRootDir = '/gws/nopw/j04/esacci_wv/odanne/GlobAlbedoTest'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l3-staging-nc2browse',
logdir='log',
hosts=[('localhost', 64)],
types=[('wvcci-l3-staging-nc2browse-step.sh', 64)])
for year in years:
tcwvMosaicDir = gaRootDir + '/Mosaic/TCWV/' + sensor + '/' + year
stagingNc2browseResultDir = gaRootDir + '/staging/QL/tcwv/' + sensor + '/' + year
for month in getMonths(year):
# l3_tcwv_meris_2008-01-01_2008-01-31.nc
stagingNc2browseFile = tcwvMosaicDir + '/l3_tcwv_' + sensor + '_' + year + '-' + month + '-01_' + year + '-' + month + '-' + str(
getNumDaysInMonth(year, month)) + '.nc'
m.execute('wvcci-l3-staging-nc2browse-step.sh', ['dummy'],
[stagingNc2browseResultDir],
parameters=[
year, month,
### Provides one step:
### - staging 'browse2mov' --> 1-year movies from png browse files for each band + BHR RGB from Albedo mosaic netcdf files
#############################################################################################################################
years=['2010']
sensors=['meris']
bands=['tcwv_mean_mean']
gaRootDir = '/gws/nopw/j04/esacci_wv/odanne/GlobAlbedoTest'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l3-staging-browse2mov',
logdir='log',
hosts=[('localhost',64)],
types=[('wvcci-l3-staging-browse2mov-step.sh',64)])
### matching for all years:
for year in years:
for sensor in sensors:
for band in bands:
stagingMoviesInputDir = gaRootDir + '/staging/QL/tcwv/' + sensor + '/' + year + '/' + band
stagingMoviesResultDir = gaRootDir + '/staging/Movies/tcwv/' + sensor + '/' + year
m.execute('wvcci-l3-staging-browse2mov-step.sh', ['dummy'], [stagingMoviesResultDir],
parameters=[year,band,stagingMoviesInputDir, stagingMoviesResultDir])
# wait for processing to complete
m.wait_for_completion()
years = range(2015,2017)
months = range(1,13)
start = '2015-06-25'
stop = '2016-11-30'
period = str((datetime.datetime.strptime(stop, '%Y-%m-%d').date() -
datetime.datetime.strptime(start, '%Y-%m-%d').date()).days + 1)
l1c_dir = '/calvalus/eodata/S2_L1C/granules-urban'
idepix_dir = '/calvalus/projects/urban-tep/msi-idepix'
timescan_seq_dir = '/calvalus/projects/urban-tep/msi-tsseq'
timescan_dir = '/calvalus/projects/urban-tep/msi-timescan'
inputs = [ l1c_dir ]
pm = PMonitor(inputs, request='timescan',
types=[('msi-idepix',2),('msi-timescan',7),('msi-format',7)],
hosts=[('localhost',12)], logdir='log', script='pstep.py', simulation=False)
for year in years:
for month in months:
if '{0:04d}-{1:02d}'.format(year, month) < start[:7] or '{0:04d}-{1:02d}'.format(year, month) > stop[:7]:
continue
beginning_of_month = str(datetime.date(year, month, 1))
end_of_month = str(datetime.date(year, month, calendar.monthrange(year, month)[1]))
pm.execute('msi-idepix', [ l1c_dir ], [ idepix_dir ],
parameters=[ 'year', '{0:04d}'.format(year), 'month', '{0:02d}'.format(month),
'start', beginning_of_month, 'stop', end_of_month ])
for region in regions:
polygon = regions[region]
years = ['2003'] #test
doys = ['65']
snowModes = ['snow', 'nosnow', 'merge']
#scalings = ['6', '30', '60']
resolutions = ['05', '025', '005']
gaRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/GlobAlbedoTest'
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1'
brdfMosaicDir = gaRootDir + '/Mosaic/brdf'
#### Upscaling/Mosaicing ####
m = PMonitor([gaRootDir],
request='ga-l3-mosaic',
logdir='log',
hosts=[('localhost',16)],
types=[('ga-l3-mosaic-step.sh',16)])
## BRDF mosaicking
for year in years:
for doy in doys:
for snowMode in snowModes:
for resolution in resolutions:
brdfMosaicDir = gaRootDir + '/Mosaic/brdf/' + snowMode + '/' + resolution
m.execute('ga-l3-mosaic-step.sh', [gaRootDir], [brdfMosaicDir], parameters=[year,doy,snowMode,resolution,gaRootDir,beamDir])
## BRDF mosaic --> Albedo mosaic
# TODO
gaRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/GlobAlbedoTest'
bbdrRootDir = gaRootDir + '/BBDR'
noSnowPriorDir = '/group_workspaces/cems2/qa4ecv/vol3/newPrior_allBands/1km/NoSnow'
snowPriorDir = '/group_workspaces/cems2/qa4ecv/vol3/newPrior_allBands/1km/Snow'
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1'
doys = []
#for i in range(44):
#for i in range(3): # test
for i in range(1): # test
doy = 8*i + 1
doys.append(str(doy).zfill(3))
m = PMonitor([bbdrRootDir],
request='ga-l3-tile',
logdir='log',
hosts=[('localhost',4)])
#tiles = glob.glob1(bbdrTileDir+'/'+sensor+'/'+year, 'h??v??')
#tiles = glob.glob1(bbdrTileDir, 'h??v??')
#tiles = ['h18v03', 'h18v04'] # test
tiles = ['h18v04'] # test
tiles.sort()
for tile in tiles:
for doy in doys:
## daily accumulation, full accumulation, inversion, merge, albedo
albedoDir = gaRootDir + '/Albedo/' + tile
m.execute('ga-l3-tile-step.sh', [bbdrRootDir], [albedoDir], parameters=[tile,year,doy,'false',gaRootDir,noSnowPriorDir,snowPriorDir,beamDir])
m.execute('ga-l3-tile-step.sh', [bbdrRootDir], [albedoDir], parameters=[tile,year,doy,'true',gaRootDir,snowPriorDir,snowPriorDir,beamDir])
for region in regions:
shapeWktFile = DIVERSITY_INST_DIR + '/wkt/' + region + '.shape'
if not os.access(shapeWktFile, os.R_OK):
raise IOError('Unable to access ' + shapeWktFile)
# output format must be one of 'GeoTIFF' 'NetCDF' or 'NetCDF4'
outputFormat = 'GeoTIFF'
extension = 'nc'
if outputFormat == 'GeoTIFF':
extension = 'tif'
inputs = ['childs']
hosts = [('localhost', 16)]
# , simulation=True
pm = PMonitor(inputs, request='lake_products', logdir='log', hosts=hosts, script='template.py')
BASE = '/calvalus/projects/diversity/prototype/'
for region in regions:
shapeWktFile = 'wkt/' + region + '.shape'
boxWktFile = shapeWktFile
shallowDir = BASE + region + '/l3-shallow/'
# =============== shallow =======================
# L3 aggregation of ratio490
isNorthRegion = True
if isNorthRegion:
shallowStart = date(2008, 05, 01)
shallowStop = date(2008, 10, 31)
requiredIdepix = ['l2_idepix-2008-' + region]
#return 1 # test!!
return 30
else:
#return 2 # test!!
return 31
######################## EraInterim from Idepix: ###########################
wvcciRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/WvcciTest'
snapDir = '/group_workspaces/cems2/qa4ecv/vol4/software/snap'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l2-idepix-erainterim-modis',
logdir='log',
hosts=[('localhost', 128)],
types=[('wvcci-l2-idepix-erainterim-modis-step.sh', 128)])
for year in years:
for sensor in sensors:
idepixRootDir = wvcciRootDir + '/Idepix/' + sensor
for month in getMonth(year):
if os.path.exists(idepixRootDir + '/' + year + '/' + month):
#for iday in range(1, getNumMonthDays(year, int(month))+1):
for iday in range(2, 3):
day = str(iday).zfill(2)
years = ['2005', '2006']
inputs = []
for year in years:
for month in getMonths(year):
#inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
inputs.append('/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year +
'/' + month)
hosts = [('localhost', 16)]
types = [('idepix_v10-step.sh', 16), ('era-interim-meris-step.sh', 4),
('idepix-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix_meris_v10_catalysts', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('era-interim-meris-step.sh', \
[ '/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/era-interim/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('idepix_v10-step.sh', \
[ '/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year + '/' + month ,
'/calvalus/projects/cawa/era-interim/meris/' + year + '/' + month], \
[ '/calvalus/projects/cawa/idepix/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
def run(self):
pm = PMonitor(inputs, request='idepix-comparison', logdir='log', hosts=hosts, types=types)
for (month, firstDay, lastDay) in TIME_RANGES:
minDate = datetime.date(year, month, firstDay)
maxDate = datetime.date(year, month, lastDay)
polymerTag = 'polymer-' + str(minDate)
params = ['polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerTag], parameters=params, logprefix=polymerTag)
idepixOldTag = 'idepix-old-' + str(minDate)
params = ['idepix-\${variant}-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'old', \
'processorBundleName', 'beam-idepix-cc', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixOldTag], parameters=params, logprefix=idepixOldTag)
idepixNewTag = 'idepix-new-' + str(minDate)
params = ['idepix-\${variant}-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'new', \
'processorBundleName', 'beam-idepix-oc', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixNewTag], parameters=params, logprefix=idepixNewTag)
#idepixNewQlTag = 'idepix-new-ql-' + str(minDate)
#params = ['idepix-ql-\${variant}-\${year}-\${month}.xml', \
# 'minDate', str(minDate), \
# 'maxDate', str(maxDate), \
# 'variant', 'new', \
# 'year', str(year), \
# 'month', str(month) ]
#pm.execute('template-step.py', [idepixNewTag, polymerTag], [idepixNewQlTag], parameters=params, logprefix=idepixNewQlTag)
mergedL2Tag = 'merged-l2-' + str(minDate)
params = ['merge-polymer-and-two-idepix-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'variant', 'new', \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', [idepixNewTag, idepixOldTag, polymerTag], [mergedL2Tag], parameters=params, logprefix=mergedL2Tag)
mergedL2FormatTag = 'merge-l2-format-' + str(minDate)
params = ['l2-format-\${minDate}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', str(year), \
'month', str(month) ]
pm.execute('template-step.py', [mergedL2Tag], [mergedL2FormatTag], parameters=params, logprefix=mergedL2FormatTag)
for singleDay in dateRange(minDate, maxDate):
merisDailyNewName = 'meris-daily-new-' + str(singleDay)
merisDailyParams = ['daily-new-\${date}.xml', \
'date', str(singleDay), \
'year', '%4d' % (singleDay.year), \
'month', str(singleDay.month) ]
pm.execute('template-step.py', [polymerTag], [merisDailyNewName], parameters=merisDailyParams, logprefix=merisDailyNewName)
merisDailyName = 'meris-daily-old-' + str(singleDay)
merisDailyParams = ['daily-old-\${date}.xml', \
'date', str(singleDay), \
'year', '%4d' % (singleDay.year), \
'month', str(singleDay.month) ]
pm.execute('template-step.py', [polymerTag], [merisDailyName], parameters=merisDailyParams, logprefix=merisDailyName)
# quicklooks L2 polymer + old idepix ?)
# DONE # quicklooks L2 polymer + new idepix
# DONE # merge L2 polymer + old + new idepix --> Format to DIMAP
# daily L3 polymer + new idepix + quicklooks
# daily L3 polymer + old idepix + quicklooks)
#======================================================
pm.wait_for_completion()
# use notation as in 'lakes_products.py'
regions = ['Lake-Balaton']
#regions = ['Lake-Bogoria', 'Lake-Elmenteita', 'Lake-Nakuru', 'Lake-Tuusulanjarvi', 'Lake-Ulemiste']
DIVERSITY_INST_DIR = os.environ['DIVERSITY_INST']
# before starting, check if WKT files are available
# From now on we always use the full polygons (*.shape) rather than just rectangular boxes.
for region in regions:
boxWktFile = DIVERSITY_INST_DIR + '/wkt/' + region + '.shape'
if not os.access(boxWktFile, os.R_OK):
raise IOError('Unable to access ' + boxWktFile)
inputs = ['dummy']
hosts = [('localhost', 8)]
pm = PMonitor(inputs, request='geo_child_products', logdir='log', hosts=hosts, script='template.py')
BASE = '/calvalus/projects/diversity/prototype/'
for region in regions:
boxWktFile = 'wkt/' + region + '.shape'
for year in years:
for month in getMonth(year):
geoChildDir = BASE + region + '/l1-child/' + year + "/" + month
l1_geo_name = 'l1_geo-' + year + "-" + month + '-' + region
params = [
'year', year,
'month', month,
'region', region,
'wkt', 'include:'+boxWktFile,
'input', '/calvalus/eodata/MER_FSG_1P/v2013/' + year + '/' + month,
def run(self):
pm = PMonitor(inputs, request="oc-seawifs", logdir="log", hosts=hosts, types=types)
seawifsBiasInputs = []
for year in years:
months = monthsAll
if year == "2002":
months = months2002
elif year == "2012":
months = months2012
for month in months:
seawifsFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
seawifsDailyBSName = "seawifs-daily-bs-" + str(singleDay)
params = [
"seawifs-daily-bs-\${date}.xml",
"date",
str(singleDay),
"year",
year,
"month",
month,
"day",
"%02d" % (singleDay.day),
]
pm.execute(
"template-step.py",
["SEAWIFS_L3_daily"],
[seawifsDailyBSName],
parameters=params,
logprefix=seawifsDailyBSName,
)
seawifsFormatInputs.append(seawifsDailyBSName)
if year >= "2003" and year <= "2007":
seawifsBiasInputs.append(seawifsDailyBSName)
seawifsFormatBSName = "seawifs-daily-bs-format-" + month + "-" + year
params = [
"l3format-\${prefix}-\${date}.xml",
"date",
month + "-" + year,
"inputPath",
"seawifs/daily-bs/" + year + "/" + month + "/????-??-??/part-*",
"outputPath",
"seawifs/daily-bs/" + year + "/" + month + "/netcdf-geo",
"prefix",
"OC-seawifs-daily-bs",
]
# debug only daily binned band-shifted
pm.execute(
"template-step.py",
seawifsFormatInputs,
[seawifsFormatBSName],
parameters=params,
logprefix=seawifsFormatBSName,
)
seawifsBiasMapName = "seawifs-bias-map"
params = ["bias-map-\${sensor}.xml", "sensor", "seawifs", "sensorMarker", "12"]
pm.execute(
"template-step.py", seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName
)
seawifsBiasFormatName = "seawifs-bias-map-format"
params = [
"l3format-\${prefix}-\${date}.xml",
"date",
"5years",
"inputPath",
"seawifs/bias-map-parts/part-*",
"outputPath",
"seawifs/bias-map-netcdf-geo",
"prefix",
"OC-seawifs-bias",
]
# debug only bias map
pm.execute(
"template-step.py",
[seawifsBiasMapName],
[seawifsBiasFormatName],
parameters=params,
logprefix=seawifsBiasFormatName,
)
# ======================================================
pm.wait_for_completion()
#### main script: ####
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
years = ['2008']
inputs = []
for year in years:
for month in getMonths(year):
inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
hosts = [('localhost', 16)]
types = [('idepix_v10_CTP-step.sh', 16), ('idepix-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix_meris_v10_CTP', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('idepix_v10_CTP-step.sh', \
[ '/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/idepix_ctp/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('idepix-format-step.sh', \
[ '/calvalus/projects/cawa/idepix_ctp/meris/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/idepix_ctp/meris-nc/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
#return 1 # test!!
return 30
else:
#return 2 # test!!
return 31
######################## L1b --> Idepix: ###########################
wvcciRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/WvcciTest'
snapDir = '/group_workspaces/cems2/qa4ecv/vol4/software/snap'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l2-idepix',
logdir='log',
hosts=[('localhost', 128)],
types=[('wvcci-l2-idepix-step.sh', 128)])
for year in years:
for sensor in sensors:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
for month in getMonth(year):
if os.path.exists(l1bRootDir + '/' + year + '/' + month):
for day in range(1, getNumMonthDays(year, int(month)) + 1):
idepixL2Dir = wvcciRootDir + '/Idepix/' + sensor + '/' + year + '/' + month + '/' + str(
day).zfill(2)
print 'idepixL2Dir: ', idepixL2Dir
#for day in range(1,32): # Jan, TODO: many broken products, repeat download of MYD021KM for Jan
#for day in range(2,3):
#for day in range(1,58): # Jan, Feb
#for day in range(120,122):
for day in days:
doy = str(day).zfill(3)
inputs.append('/calvalus/projects/cawa/MODIS_MYD021KM/' + year + '/' +
doy)
hosts = [('localhost', 16)]
types = [('idepix-step.sh', 16), ('era-interim-modis-step.sh', 4),
('idepix-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix_modis', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
#for day in range(1,122): # 20080101 - 20080430
#for day in range(1,32): # Jan, TODO: many broken products, repeat download of MYD021KM for Jan
#for day in range(2,3):
#for day in range(1,58): # Jan, Feb
#for day in range(58,122): # Mar, Apr
#for day in range(120,122):
for day in days:
doy = str(day).zfill(3)
current_date = base_date + timedelta(day - 1)
datestring = current_date.strftime("%Y-%m-%d")
pm.execute('era-interim-modis-step.sh', \
def run(self):
pm = PMonitor(inputs, request='oc-meris', logdir='log', hosts=hosts, types=types)
merisBiasInputs = []
for year in years:
merisDailyTemplate = 'meris-daily-useIdepix-\${date}.xml'
months = monthsAll
if year == '2002':
months = months2002
# in 2002 the current version of idepix is not correct because of sensor re-programming
merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml'
elif year == '2012':
months = months2012
for month in months:
merisDailyFormatInputs = []
merisDailyBSFormatInputs = []
(minDate, maxDate) = getMinMaxDate(year, month)
idepixName = 'idepix-' + str(minDate)
params = ['meris-idepix-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName)
polymerName = 'polymer-' + str(minDate)
params = ['meris-polymer-\${year}-\${month}.xml', \
'minDate', str(minDate), \
'maxDate', str(maxDate), \
'year', year, \
'month', month ]
pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName)
# for now because we have not more test-data
minDate = datetime.date(int(year), int(month), 1)
maxDate = datetime.date(int(year), int(month), 1)
for singleDay in dateRange(minDate, maxDate):
merisDailyName = 'meris-daily-' + str(singleDay)
params = [merisDailyTemplate, \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [idepixName, polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName)
merisDailyFormatInputs.append(merisDailyName)
merisDailyBSName = 'meris-daily-bs-' + str(singleDay)
params = ['meris-daily-bs-\${date}.xml', \
'date', str(singleDay), \
'year', year, \
'month', month ]
pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName)
merisDailyBSFormatInputs.append(merisDailyBSName)
if year >= '2003' and year <= '2007':
merisBiasInputs.append(merisDailyBSName)
merisFormatName = 'meris-daily-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \
'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily' ]
# (debug only / user product ?) daily binned
pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName)
merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year
params = ['l3format-\${prefix}-\${date}.xml', \
'date', month + '-' + year, \
'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \
'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \
'prefix', 'OC-meris-daily-bs' ]
# debug only daily binned band-shifted
pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName)
merisBiasMapName = 'meris-bias-map'
params = ['bias-map-\${sensor}.xml', \
'sensor', 'meris', \
'sensorMarker', '10' ]
pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName)
merisBiasFormatName = 'meris-bias-map-format'
params = ['l3format-\${prefix}-\${date}.xml', \
'date', '5years', \
'inputPath', 'meris/bias-map-parts/part-*', \
'outputPath', 'meris/bias-map-netcdf-geo', \
'prefix', 'OC-meris-bias' ]
# debug only bias map
pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName)
#======================================================
pm.wait_for_completion()
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
#years = [ '2008' ]
years = ['2005']
inputs = []
for year in years:
for month in getMonths(year):
inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
hosts = [('localhost', 16)]
types = [('idepix-step.sh', 16), ('format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('idepix-step.sh', \
[ '/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
parameters=[str(minDate), str(maxDate)])
#pm.execute('format-step.sh', \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission-nc/' + year + '/' + month ], \
# parameters=[str(minDate), str(maxDate)])
return is_daily
else:
return false
######################## L1b --> Idepix --> IdepixEraInterim --> TCWV: ###########################
wvcciRootDir = '/gws/nopw/j04/esacci_wv/odanne/WvcciRoot'
eraInterimRootDir = wvcciRootDir + '/auxiliary/era-interim-t2m-mslp-tcwv-u10-v10'
inputs = ['dummy']
# NEW PMonitor version, MB/TB Nov 2018:
m = PMonitor(inputs,
request='wvcci-l2-tcwv-modis-chain_4',
logdir='log',
hosts=[('localhost', 256)],
types=[('wvcci-l2-tcwv-modis-step_4.sh', 256)],
polling="job_status_callback.sh")
for year in years:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
modisCloudMaskRootDir = wvcciRootDir + '/ModisCloudMask/MOD35_L2'
print('year: ' + year)
for month in getMonth(year):
print('month: ' + month)
if os.path.exists(l1bRootDir + '/' + year + '/' + month):
numMonthDays = monthrange(int(year), int(month))[1]
if not os.access(shapeWktFile, os.R_OK):
raise IOError('Unable to access ' + shapeWktFile)
# output format must be one of 'GeoTIFF' 'NetCDF' or 'NetCDF4'
outputFormat = 'GeoTIFF'
extension = 'nc'
if outputFormat == 'GeoTIFF':
extension = 'tif'
inputs = ['childs']
hosts = [('localhost', 16)]
# , simulation=True
pm = PMonitor(inputs,
request='lake_products',
logdir='log',
hosts=hosts,
script='template.py')
BASE = '/calvalus/projects/diversity/prototype/'
for region in regions:
shapeWktFile = 'wkt/' + region + '.shape'
boxWktFile = shapeWktFile
shallowDir = BASE + region + '/l3-shallow/'
# =============== shallow =======================
# L3 aggregation of ratio490
isNorthRegion = True
if isNorthRegion:
shallowStart = date(2008, 05, 01)
