def __init__(self, preconditions, usecase, hosts, calls, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
calls,
log_dir=log_dir,
simulation=simulation)
def __init__(self, preconditions, usecase, hosts, types, log_dir,
simulation):
"""
:type preconditions: list
:type usecase: str
:type hosts: list
:type calls: list
:type log_dir: str
:type simulation: bool
"""
self.pm = PMonitor(preconditions,
usecase,
hosts,
types,
logdir=log_dir,
simulation=simulation,
polling="job_status_callback.sh")
return is_daily
else:
return false
######################## L1b --> Idepix --> IdepixEraInterim --> TCWV: ###########################
wvcciRootDir = '/gws/nopw/j04/esacci_wv/odanne/WvcciRoot'
eraInterimRootDir = wvcciRootDir + '/auxiliary/era-interim-t2m-mslp-tcwv-u10-v10'
inputs = ['dummy']
# NEW PMonitor version, MB/TB Nov 2018:
m = PMonitor(inputs,
request='wvcci-l2-tcwv-modis-chain_4',
logdir='log',
hosts=[('localhost', 256)],
types=[('wvcci-l2-tcwv-modis-step_4.sh', 256)],
polling="job_status_callback.sh")
for year in years:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
modisCloudMaskRootDir = wvcciRootDir + '/ModisCloudMask/MOD35_L2'
print('year: ' + year)
for month in getMonth(year):
print('month: ' + month)
if os.path.exists(l1bRootDir + '/' + year + '/' + month):
numMonthDays = monthrange(int(year), int(month))[1]
years = ['2005', '2006']
inputs = []
for year in years:
for month in getMonths(year):
#inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
inputs.append('/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year +
'/' + month)
hosts = [('localhost', 16)]
types = [('idepix_v10-step.sh', 16), ('era-interim-meris-step.sh', 4),
('idepix-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix_meris_v10_catalysts', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('era-interim-meris-step.sh', \
[ '/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/era-interim/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('idepix_v10-step.sh', \
[ '/calvalus/projects/cawa/MERIS_RR_catalysts_N1/' + year + '/' + month ,
'/calvalus/projects/cawa/era-interim/meris/' + year + '/' + month], \
[ '/calvalus/projects/cawa/idepix/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
#return 1 # test!!
return 30
else:
#return 2 # test!!
return 31
######################## L1b --> Idepix: ###########################
wvcciRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/WvcciTest'
snapDir = '/group_workspaces/cems2/qa4ecv/vol4/software/snap'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l2-idepix',
logdir='log',
hosts=[('localhost', 128)],
types=[('wvcci-l2-idepix-step.sh', 128)])
for year in years:
for sensor in sensors:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
for month in getMonth(year):
if os.path.exists(l1bRootDir + '/' + year + '/' + month):
for day in range(1, getNumMonthDays(year, int(month)) + 1):
idepixL2Dir = wvcciRootDir + '/Idepix/' + sensor + '/' + year + '/' + month + '/' + str(
day).zfill(2)
print 'idepixL2Dir: ', idepixL2Dir
years = ['2003'] #test
doys = ['65']
snowModes = ['snow', 'nosnow', 'merge']
#scalings = ['6', '30', '60']
resolutions = ['05', '025', '005']
gaRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/GlobAlbedoTest'
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1'
brdfMosaicDir = gaRootDir + '/Mosaic/brdf'
#### Upscaling/Mosaicing ####
m = PMonitor([gaRootDir],
request='ga-l3-mosaic',
logdir='log',
hosts=[('localhost',16)],
types=[('ga-l3-mosaic-step.sh',16)])
## BRDF mosaicking
for year in years:
for doy in doys:
for snowMode in snowModes:
for resolution in resolutions:
brdfMosaicDir = gaRootDir + '/Mosaic/brdf/' + snowMode + '/' + resolution
m.execute('ga-l3-mosaic-step.sh', [gaRootDir], [brdfMosaicDir], parameters=[year,doy,snowMode,resolution,gaRootDir,beamDir])
## BRDF mosaic --> Albedo mosaic
# TODO
for year in years:
#for day in range(122): # 20080101 - 20080430
#for day in range(15,17): # 20080115, 20080116
#for day in range(1,58): # Jan, Feb
for day in days:
doy = str(day).zfill(3)
#inputs.append('/calvalus/projects/cawa/idepix/modis-nc/' + year + '/' + doy)
inputs.append('/calvalus/projects/cawa/idepix/modis/' + year + '/' +
doy)
hosts = [('localhost', 16)]
types = [('tcwv-step.sh', 4), ('tcwv-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='tcwv_modis', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
#for day in range(122): # 20080101 - 20080430
#for day in range(15,17): # 20080115, 20080116
#for day in range(1,58): # Jan, Feb
for day in days:
doy = str(day).zfill(3)
current_date = base_date + timedelta(day - 1)
datestring = current_date.strftime("%Y-%m-%d")
pm.execute('tcwv-step.sh', \
[ '/calvalus/projects/cawa/idepix/modis/' + year + '/' + doy ], \
[ '/calvalus/projects/cawa/tcwv/modis/' + year + '/' + doy ], \
parameters=['MODIS', datestring, datestring])
# use notation as in 'lakes_products.py'
regions = ['Lake-Balaton']
#regions = ['Lake-Bogoria', 'Lake-Elmenteita', 'Lake-Nakuru', 'Lake-Tuusulanjarvi', 'Lake-Ulemiste']
DIVERSITY_INST_DIR = os.environ['DIVERSITY_INST']
# before starting, check if WKT files are available
# From now on we always use the full polygons (*.shape) rather than just rectangular boxes.
for region in regions:
boxWktFile = DIVERSITY_INST_DIR + '/wkt/' + region + '.shape'
if not os.access(boxWktFile, os.R_OK):
raise IOError('Unable to access ' + boxWktFile)
inputs = ['dummy']
hosts = [('localhost', 8)]
pm = PMonitor(inputs, request='geo_child_products', logdir='log', hosts=hosts, script='template.py')
BASE = '/calvalus/projects/diversity/prototype/'
for region in regions:
boxWktFile = 'wkt/' + region + '.shape'
for year in years:
for month in getMonth(year):
geoChildDir = BASE + region + '/l1-child/' + year + "/" + month
l1_geo_name = 'l1_geo-' + year + "-" + month + '-' + region
params = [
'year', year,
'month', month,
'region', region,
'wkt', 'include:'+boxWktFile,
'input', '/calvalus/eodata/MER_FSG_1P/v2013/' + year + '/' + month,
return months
#################################################################################################
years = ['2010']
sensor = 'meris'
gaRootDir = '/gws/nopw/j04/esacci_wv/odanne/GlobAlbedoTest'
inputs = ['dummy']
m = PMonitor(inputs,
request='wvcci-l3-staging-nc2browse',
logdir='log',
hosts=[('localhost', 64)],
types=[('wvcci-l3-staging-nc2browse-step.sh', 64)])
for year in years:
tcwvMosaicDir = gaRootDir + '/Mosaic/TCWV/' + sensor + '/' + year
stagingNc2browseResultDir = gaRootDir + '/staging/QL/tcwv/' + sensor + '/' + year
for month in getMonths(year):
# l3_tcwv_meris_2008-01-01_2008-01-31.nc
stagingNc2browseFile = tcwvMosaicDir + '/l3_tcwv_' + sensor + '_' + year + '-' + month + '-01_' + year + '-' + month + '-' + str(
getNumDaysInMonth(year, month)) + '.nc'
m.execute('wvcci-l3-staging-nc2browse-step.sh', ['dummy'],
[stagingNc2browseResultDir],
parameters=[
year, month,
#years = [ '2002' ]
inputs = []
for year in years:
for month in getMonths(year):
#inputs.append('/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month)
inputs.append('/calvalus/projects/cawa/idepix/meris-nc/' + year + '/' +
month)
#inputs.append('/calvalus/projects/cawa/idepix/meris/' + year + '/' + month)
hosts = [('localhost', 16)]
types = [('tcwv_v10-step.sh', 4), ('tcwv-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='tcwv_meris_v10', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
#(minDate, maxDate) = ('2008-02-19', '2008-02-20')
pm.execute('tcwv_v10-step.sh', \
[ '/calvalus/projects/cawa/idepix/meris-nc/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('tcwv-format-step.sh', \
[ '/calvalus/projects/cawa/tcwv/meris/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/tcwv/meris-nc/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
years = ['2005']
inputs = []
for year in years:
for month in getMonths(year):
#inputs.append('/calvalus/projects/cawa/idepix_ctp/meris/' + year + '/' + month)
inputs.append('/calvalus/projects/cawa/idepix_ctp/meris-nc/' + year +
'/' + month)
hosts = [('localhost', 16)]
types = [('ctp-step.sh', 4), ('ctp-format-step.sh', 2)]
pm = PMonitor(inputs, \
request='ctp_meris', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
#(minDate, maxDate) = ('2005-07-01', '2005-07-02')
pm.execute('ctp-step.sh', \
[ '/calvalus/projects/cawa/idepix_ctp/meris-nc/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/ctp/meris/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
pm.execute('ctp-format-step.sh', \
[ '/calvalus/projects/cawa/ctp/meris/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/ctp/meris-nc/' + year + '/' + month ], \
parameters=['MERIS', str(minDate), str(maxDate)])
return allMonths
######################## L1b --> Idepix --> IdepixEraInterim --> TCWV: ###########################
wvcciRootDir = '/gws/nopw/j04/esacci_wv/odanne/WvcciRoot'
eraInterimRootDir = wvcciRootDir + '/auxiliary/era-interim-t2m-mslp-tcwv-u10-v10'
snapDir = '/gws/nopw/j04/esacci_wv/software/snap'
inputs = ['dummy']
# NEW PMonitor version, MB/TB Nov 2018:
m = PMonitor(inputs,
request='wvcci-l2-tcwv-modis-chain_tmp',
logdir='log',
hosts=[('localhost', 384)],
types=[('wvcci-l2-idepix-modis-step.sh', 128),
('wvcci-l2-tcwv-modis-step.sh', 256)],
polling="job_status_callback.sh")
print('bla 1')
for year in years:
l1bRootDir = wvcciRootDir + '/L1b/' + sensor
modisLandMaskRootDir = wvcciRootDir + '/ModisLandMask/MOD03'
modisCloudMaskRootDir = wvcciRootDir + '/ModisCloudMask/MOD35_L2'
print('bla 2')
for month in getMonth(year):
gaRootDir = '/group_workspaces/cems2/qa4ecv/vol4/olafd/GlobAlbedoTest'
bbdrRootDir = gaRootDir + '/BBDR'
noSnowPriorDir = '/group_workspaces/cems2/qa4ecv/vol3/newPrior_allBands/1km/NoSnow'
snowPriorDir = '/group_workspaces/cems2/qa4ecv/vol3/newPrior_allBands/1km/Snow'
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1'
doys = []
#for i in range(44):
#for i in range(3): # test
for i in range(1): # test
doy = 8*i + 1
doys.append(str(doy).zfill(3))
m = PMonitor([bbdrRootDir],
request='ga-l3-tile',
logdir='log',
hosts=[('localhost',4)])
#tiles = glob.glob1(bbdrTileDir+'/'+sensor+'/'+year, 'h??v??')
#tiles = glob.glob1(bbdrTileDir, 'h??v??')
#tiles = ['h18v03', 'h18v04'] # test
tiles = ['h18v04'] # test
tiles.sort()
for tile in tiles:
for doy in doys:
## daily accumulation, full accumulation, inversion, merge, albedo
albedoDir = gaRootDir + '/Albedo/' + tile
m.execute('ga-l3-tile-step.sh', [bbdrRootDir], [albedoDir], parameters=[tile,year,doy,'false',gaRootDir,noSnowPriorDir,snowPriorDir,beamDir])
m.execute('ga-l3-tile-step.sh', [bbdrRootDir], [albedoDir], parameters=[tile,year,doy,'true',gaRootDir,snowPriorDir,snowPriorDir,beamDir])
#years = [ '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011','2012' ]
#years = [ '2008' ]
years = ['2005']
inputs = []
for year in years:
for month in getMonths(year):
inputs.append('/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month)
hosts = [('localhost', 16)]
types = [('idepix-step.sh', 16), ('format-step.sh', 2)]
pm = PMonitor(inputs, \
request='idepix', \
logdir='log', \
hosts=hosts, \
types=types)
for year in years:
for month in getMonths(year):
(minDate, maxDate) = getMinMaxDate(year, month)
pm.execute('idepix-step.sh', \
[ '/calvalus/eodata/MER_RR__1P/r03/' + year + '/' + month ], \
[ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
parameters=[str(minDate), str(maxDate)])
#pm.execute('format-step.sh', \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission/' + year + '/' + month ], \
# [ '/calvalus/projects/cawa/idepix-RR-fullmission-nc/' + year + '/' + month ], \
# parameters=[str(minDate), str(maxDate)])
