class Monitor: def __init__(self, preconditions, usecase, hosts, types, log_dir, simulation): """ :type preconditions: list :type usecase: str :type hosts: list :type calls: list :type log_dir: str :type simulation: bool """ self.pm = PMonitor(preconditions, usecase, hosts, types, logdir=log_dir, simulation=simulation, polling="job_status_callback.sh") def execute(self, job): """ :type job: Job """ self.pm.execute(job.get_call(), job.get_preconditions(), job.get_postconditions(), job.get_parameters(), logprefix=job.get_name()) def wait_for_completion(self): self.pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='oc-idepix-meris', logdir='log', hosts=hosts, types=types) for year in years: months = monthsAll if year == '2002': months = months2002 # in 2002 the current version of idepix is not correct because of sensor re-programming merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml' elif year == '2012': months = months2012 for month in months: (minDate, maxDate) = getMinMaxDate(year, month) idepixName = 'meris-idepix-' + year + '-' + month params = ['meris-idepix-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName) #====================================================== pm.wait_for_completion()
class Monitor: def __init__(self, preconditions, usecase, hosts, calls, log_dir, simulation): """ :type preconditions: list :type usecase: str :type hosts: list :type calls: list :type log_dir: str :type simulation: bool """ self.pm = PMonitor(preconditions, usecase, hosts, calls, log_dir=log_dir, simulation=simulation) def execute(self, job): """ :type job: Job """ self.pm.execute(job.get_call(), job.get_preconditions(), job.get_postconditions(), job.get_parameters(), log_prefix=job.get_name()) def wait_for_completion(self): self.pm.wait_for_completion() def wait_for_completion_and_terminate(self): self.pm.wait_for_completion_and_terminate()
def run(self): pm = PMonitor(inputs, request='round-robin2-matchups', logdir='log', hosts=hosts, types=types) for sensor in sensors: sensorPointData = pointDataLocal[sensor] calvalusPointData = calvalusPointDataRoot + '/' + sensor + '-' + basename(sensorPointData) pm.execute('ingest-point-data.sh', [sensorPointData], [calvalusPointData], logprefix="ingest-point-data-" + sensor) eodata = sensorEodata[sensor] for processor in processors: if sensor in processorConfig[processor]['processorName']: params = ['round-robin-\${processor}-\${sensor}-matchup.xml', 'sensor', sensor, 'processor', processor, 'eodataRoot', eodata['root'], 'eodataSuffix', eodata['suffix'], 'processorBundleName', processorConfig[processor]['processorBundleName'], 'processorBundleVersion', processorConfig[processor]['processorBundleVersion'], 'processorName', processorConfig[processor]['processorName'][sensor], 'pointData', calvalusPointData, 'namenode', namenode ] matchupName = 'round-robin-matchup-' + processor + '-' + sensor pm.execute('template-step.py', [calvalusPointData], [matchupName], parameters=params, logprefix=matchupName) #====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='oc-modis', logdir='log', hosts=hosts, types=types) modisBiasInputs = [] for year in years: months = monthsAll if year == '2002': months = months2002 elif year == '2012': months = months2012 for month in months: modisFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 1) for singleDay in dateRange(minDate, maxDate): modisDailyBSName = 'modis-daily-bs-' + str(singleDay) params = ['modis-daily-bs-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ,\ 'day', '%02d' % (singleDay.day) ] pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName) modisFormatInputs.append(modisDailyBSName) if year >= '2003' and year <= '2007': modisBiasInputs.append(modisDailyBSName) modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \ 'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-modis-daily-bs' ] # debug only daily binned band-shifted pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName) modisBiasMapName = 'modis-bias-map' params = ['bias-map-\${sensor}.xml', \ 'sensor', 'modis', \ 'sensorMarker', '11' ] pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName) modisBiasFormatName = 'modis-bias-map-format' params = ['l3format-\${prefix}-\${date}.xml', \ 'date', '5years', \ 'inputPath', 'modis/bias-map-parts/part-*', \ 'outputPath', 'modis/bias-map-netcdf-geo', \ 'prefix', 'OC-modis-bias' ] # debug only bias map pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName) #====================================================== pm.wait_for_completion()
def run(self): allLocalPointData = [] for sensor in sensors: for station in sensorStations[sensor]: allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor][station]) print allLocalPointData pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types) for sensor in sensors: for station in sensorStations[sensor]: localPointData = localPointDataRoot + "/" + pointData[sensor][station] calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor][station] handle = "ingest-point-data-" + sensor + "-" + station pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle) #====================================================== for vc in gainsWithVC[sensor].keys(): params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml', 'station', station, 'sensor', sensor, 'vc', vc, 'calib', '"'+gainsWithVC[sensor][vc]+'"', 'calvalusPointData', calvalusPointData, 'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc ] handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle) #====================================================== vc = 'withoutvc' params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml', 'station', station, 'sensor', sensor, 'vc', vc, 'calib', '"'+gainsWithoutVC[sensor]+'"', 'calvalusPointData', calvalusPointData, 'output', '/calvalus/projects/vc-analysis/' + sensor + "-" + station + "-" + vc ] handle = 'vc-analysis-ana-' + sensor + "-" + station + "-" + vc pm.execute('template-step.py', [calvalusPointData], [handle], parameters=params, logprefix=handle) #====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='oc-merging', logdir='log', hosts=hosts, types=types) for year in years: months = monthsAll if year == '2002': months = months2002 elif year == '2012': months = months2012 for month in months: formatMergedInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 1) for singleDay in dateRange(minDate, maxDate): mergedName = 'merged-daily-' + str(singleDay) params = ['sensor-merging-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', ['input'], [mergedName], parameters=params, logprefix=mergedName) formatMergedInputs.append(mergedName) mergedFormatName = 'merged-daily-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \ 'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-merged-daily' ] pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName) #====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='round-robin2-area', logdir='log', hosts=hosts, types=types) for region in regions: for processor in processors: nameProcess = 'round-robin-process-' + processor + '-' + region params = ['round-robin-\${processor}-\${region}-process.xml', 'processor', processor, 'region', region, 'processorBundleName', processorConfig[processor]['processorBundleName'], 'processorBundleVersion', processorConfig[processor]['processorBundleVersion'], 'processorName', processorConfig[processor]['processorName'] ] pm.execute('template-step.py', inputs, [nameProcess], parameters=params, logprefix=nameProcess) if processorConfig[processor]['needsFormatting']: nameFormat = 'round-robin-format-' + processor + '-' + region params = ['round-robin-\${processor}-\${region}-format.xml', 'processor', processor, 'region', region ] pm.execute('template-step.py', [nameProcess], [nameFormat], parameters=params, logprefix=nameFormat) #====================================================== pm.wait_for_completion()
# cloud product e.g. MOD35_L2.A2015196.1855.061.2017321064215.hdf modisCloudMaskFiles = fnmatch.filter( os.listdir(modisCloudMaskRootDir + '/' + year + '/' + month + '/' + str(day).zfill(2)), fileFilter) modisCloudMaskPath = modisCloudMaskRootDir + '/' + year + '/' + month + '/' + str( day).zfill( 2 ) + '/' + modisCloudMaskFiles[0] if os.path.exists(modisCloudMaskPath): #print( 'Daily product: ' + str(is_daily_product(modisCloudMaskPath)) + ' --> ' + modisCloudMaskPath) # =============== Merge MODIS L1b with ERA-INTERIM, then TCWV from Idepix-ERA-INTERIM merge product ======================= l1bEraFile = l1bFileBase + '_l1b-era-interim.nc' m.execute( 'wvcci-l2-tcwv-modis-step_4.sh', ['dummy'], [l1bEraFile], parameters=[ l1bPath, l1bFiles[index], modisCloudMaskPath, year, month, day, hhmm, wvcciRootDir ]) m.wait_for_completion()
def run(self): pm = PMonitor(inputs, request="oc-seawifs", logdir="log", hosts=hosts, types=types) seawifsBiasInputs = [] for year in years: months = monthsAll if year == "2002": months = months2002 elif year == "2012": months = months2012 for month in months: seawifsFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 1) for singleDay in dateRange(minDate, maxDate): seawifsDailyBSName = "seawifs-daily-bs-" + str(singleDay) params = [ "seawifs-daily-bs-\${date}.xml", "date", str(singleDay), "year", year, "month", month, "day", "%02d" % (singleDay.day), ] pm.execute( "template-step.py", ["SEAWIFS_L3_daily"], [seawifsDailyBSName], parameters=params, logprefix=seawifsDailyBSName, ) seawifsFormatInputs.append(seawifsDailyBSName) if year >= "2003" and year <= "2007": seawifsBiasInputs.append(seawifsDailyBSName) seawifsFormatBSName = "seawifs-daily-bs-format-" + month + "-" + year params = [ "l3format-\${prefix}-\${date}.xml", "date", month + "-" + year, "inputPath", "seawifs/daily-bs/" + year + "/" + month + "/????-??-??/part-*", "outputPath", "seawifs/daily-bs/" + year + "/" + month + "/netcdf-geo", "prefix", "OC-seawifs-daily-bs", ] # debug only daily binned band-shifted pm.execute( "template-step.py", seawifsFormatInputs, [seawifsFormatBSName], parameters=params, logprefix=seawifsFormatBSName, ) seawifsBiasMapName = "seawifs-bias-map" params = ["bias-map-\${sensor}.xml", "sensor", "seawifs", "sensorMarker", "12"] pm.execute( "template-step.py", seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName ) seawifsBiasFormatName = "seawifs-bias-map-format" params = [ "l3format-\${prefix}-\${date}.xml", "date", "5years", "inputPath", "seawifs/bias-map-parts/part-*", "outputPath", "seawifs/bias-map-netcdf-geo", "prefix", "OC-seawifs-bias", ] # debug only bias map pm.execute( "template-step.py", [seawifsBiasMapName], [seawifsBiasFormatName], parameters=params, logprefix=seawifsBiasFormatName, ) # ====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='oc-complete', logdir='log', hosts=hosts, types=types) ######################################################################################### merisBiasInputs = [] for year in years: merisDailyTemplate = 'meris-daily-useIdepix-QAA-\${date}.xml' months = monthsAll if year == '2002': months = months2002 # in 2002 the current version of idepix is not correct because of sensor re-programming merisDailyTemplate = 'meris-daily-useIdepix-QAA-2002-\${date}.xml' elif year == '2012': months = months2012 for month in months: merisDailyFormatInputs = [] merisDailyBSFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) polymerName = 'polymer-' + str(minDate) params = ['meris-polymer-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 9) for singleDay in dateRange(minDate, maxDate): merisDailyName = 'meris-daily-' + str(singleDay) params = [merisDailyTemplate, \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', [polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName) merisDailyFormatInputs.append(merisDailyName) merisDailyBSName = 'meris-daily-bs-' + str(singleDay) params = ['meris-daily-bs-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName) merisDailyBSFormatInputs.append(merisDailyBSName) if year >= '2003' and year <= '2007': merisBiasInputs.append(merisDailyBSName) merisFormatName = 'meris-daily-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \ 'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-meris-daily' ] #pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName) merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \ 'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-meris-daily-bs' ] #pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName) merisBiasMapName = 'meris-bias-map' params = ['bias-map-\${sensor}.xml', \ 'sensor', 'meris', \ 'sensorMarker', '10' ] pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName) merisBiasFormatName = 'meris-bias-map-format' params = ['l3format-\${prefix}-\${date}.xml', \ 'date', '5years', \ 'inputPath', 'meris/bias-map-parts/part-*', \ 'outputPath', 'meris/bias-map-netcdf-geo', \ 'prefix', 'OC-meris-bias' ] #pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName) ###################################################################### modisBiasInputs = [] for year in years: months = monthsAll if year == '2002': months = months2002 elif year == '2012': months = months2012 for month in months: modisFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 9) for singleDay in dateRange(minDate, maxDate): modisDailyBSName = 'modis-daily-bs-' + str(singleDay) params = ['modis-daily-bs-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ,\ 'doy', '%03d' % (singleDay.timetuple().tm_yday) ] pm.execute('template-step.py', ['MODIS_L3_daily'], [modisDailyBSName], parameters=params, logprefix=modisDailyBSName) modisFormatInputs.append(modisDailyBSName) if year >= '2003' and year <= '2007': modisBiasInputs.append(modisDailyBSName) modisFormatBSName = 'modis-daily-bs-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'modis/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \ 'outputPath', 'modis/daily-bs/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-modis-daily-bs' ] #pm.execute('template-step.py', modisFormatInputs, [modisFormatBSName], parameters=params, logprefix=modisFormatBSName) modisBiasMapName = 'modis-bias-map' params = ['bias-map-\${sensor}.xml', \ 'sensor', 'modis', \ 'sensorMarker', '11' ] pm.execute('template-step.py', modisBiasInputs, [modisBiasMapName], parameters=params, logprefix=modisBiasMapName) modisBiasFormatName = 'modis-bias-map-format' params = ['l3format-\${prefix}-\${date}.xml', \ 'date', '5years', \ 'inputPath', 'modis/bias-map-parts/part-*', \ 'outputPath', 'modis/bias-map-netcdf-geo', \ 'prefix', 'OC-modis-bias' ] #pm.execute('template-step.py', [modisBiasMapName], [modisBiasFormatName], parameters=params, logprefix=modisBiasFormatName) ###################################################################### seawifsBiasInputs = [] for year in years: months = monthsAll if year == '2002': months = months2002 elif year == '2012': months = months2012 for month in months: seawifsFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 9) for singleDay in dateRange(minDate, maxDate): seawifsDailyBSName = 'seawifs-daily-bs-' + str(singleDay) params = ['seawifs-daily-bs-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ,\ 'doy', '%03d' % (singleDay.timetuple().tm_yday) ] pm.execute('template-step.py', ['SEAWIFS_L3_daily'], [seawifsDailyBSName], parameters=params, logprefix=seawifsDailyBSName) seawifsFormatInputs.append(seawifsDailyBSName) if year >= '2003' and year <= '2007': seawifsBiasInputs.append(seawifsDailyBSName) seawifsFormatBSName = 'seawifs-daily-bs-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'seawifs/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \ 'outputPath', 'seawifs/daily-bs/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-seawifs-daily-bs' ] #pm.execute('template-step.py', seawifsFormatInputs, [seawifsFormatBSName], parameters=params, logprefix=seawifsFormatBSName) seawifsBiasMapName = 'seawifs-bias-map' params = ['bias-map-\${sensor}.xml', \ 'sensor', 'seawifs', \ 'sensorMarker', '12' ] pm.execute('template-step.py', seawifsBiasInputs, [seawifsBiasMapName], parameters=params, logprefix=seawifsBiasMapName) seawifsBiasFormatName = 'seawifs-bias-map-format' params = ['l3format-\${prefix}-\${date}.xml', \ 'date', '5years', \ 'inputPath', 'seawifs/bias-map-parts/part-*', \ 'outputPath', 'seawifs/bias-map-netcdf-geo', \ 'prefix', 'OC-seawifs-bias' ] #pm.execute('template-step.py', [seawifsBiasMapName], [seawifsBiasFormatName], parameters=params, logprefix=seawifsBiasFormatName) ###################################################################### mergedInput = [merisBiasMapName, modisBiasMapName, seawifsBiasMapName] for year in years: months = monthsAll if year == '2002': months = months2002 elif year == '2012': months = months2012 for month in months: formatMergedInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 9) for singleDay in dateRange(minDate, maxDate): mergedName = 'merged-daily-' + str(singleDay) params = ['sensor-merging-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', mergedInput, [mergedName], parameters=params, logprefix=mergedName) formatMergedInputs.append(mergedName) mergedFormatName = 'merged-daily-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'merged-daily/' + year + '/' + month + '/????-??-??-parts/part-*', \ 'outputPath', 'merged-daily/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-merged-daily' ] pm.execute('template-step.py', formatMergedInputs, [mergedFormatName], parameters=params, logprefix=mergedFormatName) #====================================================== pm.wait_for_completion()
beamDir = '/group_workspaces/cems/globalalbedo/soft/beam-5.0.1' brdfMosaicDir = gaRootDir + '/Mosaic/brdf' #### Upscaling/Mosaicing #### m = PMonitor([gaRootDir], request='ga-l3-mosaic', logdir='log', hosts=[('localhost',16)], types=[('ga-l3-mosaic-step.sh',16)]) ## BRDF mosaicking for year in years: for doy in doys: for snowMode in snowModes: for resolution in resolutions: brdfMosaicDir = gaRootDir + '/Mosaic/brdf/' + snowMode + '/' + resolution m.execute('ga-l3-mosaic-step.sh', [gaRootDir], [brdfMosaicDir], parameters=[year,doy,snowMode,resolution,gaRootDir,beamDir]) ## BRDF mosaic --> Albedo mosaic # TODO # wait for processing to complete m.wait_for_completion() ##################################################################################################### #os.system('date')
def run(self): allLocalPointData = [] for sensor in sensors: allLocalPointData.append(localPointDataRoot + "/" + pointData[sensor]) pm = PMonitor(allLocalPointData, request='ma_vc-analysis-ana', logdir='log', hosts=hosts, types=types) for sensor in sensors: localPointData = localPointDataRoot + "/" + pointData[sensor] calvalusPointData = calvalusPointDataRoot + '/' + pointData[sensor] handle = "ingest-point-data-" + sensor pm.execute('ingest-point-data.sh', [localPointData], [calvalusPointData], logprefix=handle) #====================================================== for vc in gains[sensor].keys(): params = ['ma_vc-analysis-ana-' + sensor + '-\${station}-\${vc}.xml', 'station', 'global', 'sensor', sensor, 'vc', vc, 'calib', '"'+gains[sensor][vc]+'"', 'calvalusPointData', calvalusPointData, 'output', '/calvalus/projects/vc-analysis3/' + sensor + "-" + vc ] handleMA = 'vc-analysis-ana-' + sensor + "-" + vc pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA) params = ['/calvalus/projects/vc-analysis3/' + sensor + "-" + vc, 'vc-analysis-' + sensor + "-" + vc ] handleCP = 'vc-analysis-ana-' + sensor + "-" + vc + '-copy' pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP) params = ['ftp.brockmann-consult.de', 'oc-cci', 'vc-analysis-' + sensor + "-" + vc + '.tar.gz', ftpDirs[sensor][vc] ] handleFTP = 'vc-analysis-ana-' + sensor + "-" + vc + '-ftp' pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP) #====================================================== params = ['ma_vc-analysis-ana-idepix-' + sensor + '.xml', 'calvalusPointData', calvalusPointData, 'output', '/calvalus/projects/vc-analysis3/' + sensor + "-idepix" ] handleMA = 'vc-analysis-ana-idepix-' + sensor pm.execute('template-step.py', [calvalusPointData], [handleMA], parameters=params, logprefix=handleMA) params = ['/calvalus/projects/vc-analysis3/' + sensor + "-idepix", 'vc-analysis-' + sensor + "-idepix" ] handleCP = 'vc-analysis-ana-idepix-' + sensor + '-copy' pm.execute('copyMaResultsToLocal.sh', [handleMA], [handleCP], parameters=params, logprefix=handleCP) params = ['ftp.brockmann-consult.de', 'oc-cci', 'vc-analysis-' + sensor + '-idepix.tar.gz', 'VC/Gain_validation_2014-11-05' ] handleFTP = 'vc-analysis-ana-' + sensor + '-ftp' pm.execute('putOnFTP.sh', [handleCP], [handleFTP], parameters=params, logprefix=handleFTP) #====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='idepix-comparison', logdir='log', hosts=hosts, types=types) for (month, firstDay, lastDay) in TIME_RANGES: minDate = datetime.date(year, month, firstDay) maxDate = datetime.date(year, month, lastDay) polymerTag = 'polymer-' + str(minDate) params = ['polymer-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', str(year), \ 'month', str(month) ] pm.execute('template-step.py', ['MERIS_L1B'], [polymerTag], parameters=params, logprefix=polymerTag) idepixOldTag = 'idepix-old-' + str(minDate) params = ['idepix-\${variant}-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'variant', 'old', \ 'processorBundleName', 'beam-idepix-cc', \ 'year', str(year), \ 'month', str(month) ] pm.execute('template-step.py', ['MERIS_L1B'], [idepixOldTag], parameters=params, logprefix=idepixOldTag) idepixNewTag = 'idepix-new-' + str(minDate) params = ['idepix-\${variant}-\${minDate}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'variant', 'new', \ 'processorBundleName', 'beam-idepix-oc', \ 'year', str(year), \ 'month', str(month) ] pm.execute('template-step.py', ['MERIS_L1B'], [idepixNewTag], parameters=params, logprefix=idepixNewTag) #idepixNewQlTag = 'idepix-new-ql-' + str(minDate) #params = ['idepix-ql-\${variant}-\${year}-\${month}.xml', \ # 'minDate', str(minDate), \ # 'maxDate', str(maxDate), \ # 'variant', 'new', \ # 'year', str(year), \ # 'month', str(month) ] #pm.execute('template-step.py', [idepixNewTag, polymerTag], [idepixNewQlTag], parameters=params, logprefix=idepixNewQlTag) mergedL2Tag = 'merged-l2-' + str(minDate) params = ['merge-polymer-and-two-idepix-\${minDate}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'variant', 'new', \ 'year', str(year), \ 'month', str(month) ] pm.execute('template-step.py', [idepixNewTag, idepixOldTag, polymerTag], [mergedL2Tag], parameters=params, logprefix=mergedL2Tag) mergedL2FormatTag = 'merge-l2-format-' + str(minDate) params = ['l2-format-\${minDate}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', str(year), \ 'month', str(month) ] pm.execute('template-step.py', [mergedL2Tag], [mergedL2FormatTag], parameters=params, logprefix=mergedL2FormatTag) for singleDay in dateRange(minDate, maxDate): merisDailyNewName = 'meris-daily-new-' + str(singleDay) merisDailyParams = ['daily-new-\${date}.xml', \ 'date', str(singleDay), \ 'year', '%4d' % (singleDay.year), \ 'month', str(singleDay.month) ] pm.execute('template-step.py', [polymerTag], [merisDailyNewName], parameters=merisDailyParams, logprefix=merisDailyNewName) merisDailyName = 'meris-daily-old-' + str(singleDay) merisDailyParams = ['daily-old-\${date}.xml', \ 'date', str(singleDay), \ 'year', '%4d' % (singleDay.year), \ 'month', str(singleDay.month) ] pm.execute('template-step.py', [polymerTag], [merisDailyName], parameters=merisDailyParams, logprefix=merisDailyName) # quicklooks L2 polymer + old idepix ?) # DONE # quicklooks L2 polymer + new idepix # DONE # merge L2 polymer + old + new idepix --> Format to DIMAP # daily L3 polymer + new idepix + quicklooks # daily L3 polymer + old idepix + quicklooks) #====================================================== pm.wait_for_completion()
def run(self): pm = PMonitor(inputs, request='oc-meris', logdir='log', hosts=hosts, types=types) merisBiasInputs = [] for year in years: merisDailyTemplate = 'meris-daily-useIdepix-\${date}.xml' months = monthsAll if year == '2002': months = months2002 # in 2002 the current version of idepix is not correct because of sensor re-programming merisDailyTemplate = 'meris-daily-useIdepix-2002-\${date}.xml' elif year == '2012': months = months2012 for month in months: merisDailyFormatInputs = [] merisDailyBSFormatInputs = [] (minDate, maxDate) = getMinMaxDate(year, month) idepixName = 'idepix-' + str(minDate) params = ['meris-idepix-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', ['MERIS_L1B'], [idepixName], parameters=params, logprefix=idepixName) polymerName = 'polymer-' + str(minDate) params = ['meris-polymer-\${year}-\${month}.xml', \ 'minDate', str(minDate), \ 'maxDate', str(maxDate), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', ['MERIS_L1B'], [polymerName], parameters=params, logprefix=polymerName) # for now because we have not more test-data minDate = datetime.date(int(year), int(month), 1) maxDate = datetime.date(int(year), int(month), 1) for singleDay in dateRange(minDate, maxDate): merisDailyName = 'meris-daily-' + str(singleDay) params = [merisDailyTemplate, \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', [idepixName, polymerName], [merisDailyName], parameters=params, logprefix=merisDailyName) merisDailyFormatInputs.append(merisDailyName) merisDailyBSName = 'meris-daily-bs-' + str(singleDay) params = ['meris-daily-bs-\${date}.xml', \ 'date', str(singleDay), \ 'year', year, \ 'month', month ] pm.execute('template-step.py', [merisDailyName], [merisDailyBSName], parameters=params, logprefix=merisDailyBSName) merisDailyBSFormatInputs.append(merisDailyBSName) if year >= '2003' and year <= '2007': merisBiasInputs.append(merisDailyBSName) merisFormatName = 'meris-daily-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'meris/daily/' + year + '/' + month + '/????-??-??-L3-1/part-*', \ 'outputPath', 'meris/daily/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-meris-daily' ] # (debug only / user product ?) daily binned pm.execute('template-step.py', merisDailyFormatInputs, [merisFormatName], parameters=params, logprefix=merisFormatName) merisFormatBSName = 'meris-daily-bs-format-' + month + '-' + year params = ['l3format-\${prefix}-\${date}.xml', \ 'date', month + '-' + year, \ 'inputPath', 'meris/daily-bs/' + year + '/' + month + '/????-??-??/part-*', \ 'outputPath', 'meris/daily-bs/' + year + '/' + month + '/netcdf-geo', \ 'prefix', 'OC-meris-daily-bs' ] # debug only daily binned band-shifted pm.execute('template-step.py', merisDailyBSFormatInputs, [merisFormatBSName], parameters=params, logprefix=merisFormatBSName) merisBiasMapName = 'meris-bias-map' params = ['bias-map-\${sensor}.xml', \ 'sensor', 'meris', \ 'sensorMarker', '10' ] pm.execute('template-step.py', merisBiasInputs, [merisBiasMapName], parameters=params, logprefix=merisBiasMapName) merisBiasFormatName = 'meris-bias-map-format' params = ['l3format-\${prefix}-\${date}.xml', \ 'date', '5years', \ 'inputPath', 'meris/bias-map-parts/part-*', \ 'outputPath', 'meris/bias-map-netcdf-geo', \ 'prefix', 'OC-meris-bias' ] # debug only bias map pm.execute('template-step.py', [merisBiasMapName], [merisBiasFormatName], parameters=params, logprefix=merisBiasFormatName) #====================================================== pm.wait_for_completion()