def postprocess(self):
self.logger.info('post-processing with resolution %d m',
self.config.resolution)
res = True
if self.tables.exportBandList() == False:
res = False
# validate the meta data:
xp = L2A_XmlParser(self.config, 'T2A')
xp.export()
xp.validate()
if self.config.postprocess() == False:
res = False
if multiprocessing.active_children() == False:
#Create the manifest.safe (L2A)
mn = L2A_Manifest(self.config)
mn.generate(self.config._L2A_UP_DIR, self.config.L2A_MANIFEST_SAFE)
xp = L2A_XmlParser(self.config, 'Manifest')
xp.export()
xp.validate()
tMeasure = time() - self._localTimestamp
self.config.writeTimeEstimation(tMeasure)
return res
def preprocess(self):
self.logger.info('pre-processing with resolution %d m',
self.config.resolution)
# this is to check the config for the L2A_AtmCorr in ahead.
# This has historical reasons due to ATCOR porting.
# Should be moved to the L2A_Config for better design:
dummy = L2A_AtmCorr(self.config, self.logger)
dummy.checkConfiguration()
# validate the meta data:
xp = L2A_XmlParser(self.config, 'T1C')
xp.export()
xp.validate()
if (self.tables.checkBandCount() == False):
self.logger.fatal('insufficient nr. of bands in tile: ' +
self.config.L2A_TILE_ID)
return False
if (self.config.ozoneContent == '0'):
try:
ozoneMeasured = self.tables.getAuxData(self.tables.OZO)
self.config.ozoneMean = ozoneMeasured.mean()
self.logger.info('ozone mean value is: ' +
str(self.config.ozoneMean))
except:
if self.config.midLatitude == 'SUMMER':
self.logger.info(
'no ozone data present, standard mid summer will be used'
)
self.config.ozoneContent = 'h'
elif self.config.midLatitude == 'WINTER':
self.logger.info(
'no ozone data present, standard mid winter will be used'
)
self.config.ozoneContent = 'w'
else:
self.logger.info(
'no ozone data present and no mid latitude configured, standard mid summer will be used'
)
self.config.ozoneContent = 'h'
if (self.tables.importBandList() == False):
self.logger.fatal('import of band list failed')
return False
if ((self.config.aerosolType != 'AUTO')
and (self.config.midLatitude != 'AUTO')):
self.config.createAtmDataFilename()
return True
def postprocess(self):
self.logger.info('post-processing with resolution %d m',
self.config.resolution)
res = True
if not self.tables.exportBandList():
res = False
if self.config.resolution == 20 and self.config.downsample20to60 == True:
self.tables.downsampleBandList_20to60_andExport()
self.config = self.tables.config
if not self.config.postprocess():
res = False
#Create the manifest.safe (L2A)
mn = L2A_Manifest(self.config)
mn.generate(self.config._L2A_UP_DIR, self.config.L2A_MANIFEST_SAFE)
xp = L2A_XmlParser(self.config, 'Manifest')
xp.export()
xp.validate()
return res
def preprocess(self):
self.logger.info('pre-processing with resolution %d m',
self.config.resolution)
# this is to check the config for the L2A_AtmCorr in ahead.
# This has historical reasons due to ATCOR porting.
# Should be moved to the L2A_Config for better design:
dummy = L2A_AtmCorr(self.config, self.logger)
dummy.checkConfiguration()
self.config._stat = {'read': {}, 'write': {}}
# validate the meta data:
xp = L2A_XmlParser(self.config, 'T1C')
xp.validate()
if (self.tables.checkBandCount() == False):
self.logger.fatal('insufficient nr. of bands in tile: ' +
self.config.L2A_TILE_ID)
return False
if self.config.midLatitude == 'AUTO':
self.config.setMidLatitude()
if self.config.ozoneSetpoint == 0:
try:
ozoneMeasured = self.tables.getAuxData(self.tables.OZO)
ozoneSetpoint = ozoneMeasured[ozoneMeasured > 0].mean()
except:
self.logger.warning(
'no ozone values found in input data, default (331) will be used'
)
ozoneSetpoint = 331
self.config.setOzoneContentFromMetadata(ozoneSetpoint)
elif self.config.aerosolType != 'AUTO':
self.config.createAtmDataFilename()
if (self.tables.importBandList() == False):
self.logger.fatal('import of band list failed')
return False
return True
def preprocess(self):
self.logger.info('Pre-processing with resolution %d m', self.config.resolution)
# this is to check the config for the L2A_AtmCorr in ahead.
# This has historical reasons due to ATCOR porting.
# Should be moved to the L2A_Config for better design:
dummy = L2A_AtmCorr(self.config, self.logger)
dummy.checkConfiguration()
# validate the meta data:
xp = L2A_XmlParser(self.config, 'T1C')
xp.export()
xp.validate()
if(self.tables.checkBandCount() == False):
self.logger.fatal('insufficient nr. of bands in tile: ' + self.config.L2A_TILE_ID)
return False
if(self.tables.importBandList() == False):
self.logger.fatal('import of band list failed')
return False
return True
#!/usr/bin/env python
#!/usr/bin/env python
def main(args=None):
import argparse
config = L2A_Config(None)
descr = config.processorName +'. Version: '+ config.processorVersion + ', created: '+ config.processorDate + \
', supporting Level-1C product version: ' + config.productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument(
'directory',
help='Directory where the Level-1C input files are located')
parser.add_argument(
'--resolution',
type=int,
choices=[10, 20, 60],
help=
'Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed'
)
parser.add_argument(
'--sc_only',
action='store_true',
help='Performs only the scene classification at 60 or 20m resolution')
parser.add_argument(
'--cr_only',
action='store_true',
help='Performs only the creation of the L2A product tree, no processing'
)
# parser.add_argument('--profile', action='store_true', help='Profiles the processor\'s performance')
parser.add_argument(
'--refresh',
action='store_true',
help='Performs a refresh of the persistent configuration before start')
parser.add_argument('--GIP_L2A', help='Select the user GIPP')
parser.add_argument('--GIP_L2A_SC',
help='Select the scene classification GIPP')
parser.add_argument('--GIP_L2A_AC',
help='Select the atmospheric correction GIPP')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/' or directory[-1] == '\\':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if (os.path.isabs(directory)) == False:
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
directory = os.path.normpath(directory)
if os.path.exists(directory) == False:
stderrWrite('directory "%s" does not exist\n.' % directory)
return FAILURE
# check if directory argument contains a tile. If yes, split the tile from path,
# put the tile in the config object created below as selected tile,
# create the new path for the user directory.
selectedTile = None
if 'GRANULE' in directory:
dirname, selectedTile = os.path.split(directory)
directory = os.path.dirname(dirname)
test = os.path.basename(directory)
S2A_L1C_mask = 'S2A_????_???_???L1C*'
if (fnmatch.fnmatch(test, S2A_L1C_mask) == False):
stderrWrite(
'L1C user product directory must match the following mask: %s\n' %
S2A_L1C_mask)
stderrWrite('but is: %s\n' % test)
return FAILURE
config = L2A_Config(None, directory)
HelloWorld = config.processorName + ', ' + config.processorVersion + ', created: ' + config.processorDate
stdoutWrite('\n%s started ...\n' % HelloWorld)
# if(args.profile == True):
# import cProfile, pstats, StringIO
# pr = cProfile.Profile()
# pr.enable()
if args.resolution == None:
resolution = 0
else:
resolution = args.resolution
# create and initialize the base log system:
dirname, basename = os.path.split(directory)
L2A_UP_ID = basename[:4] + 'USER' + basename[8:]
L2A_UP_ID = L2A_UP_ID.replace('1C_', '2A_')
logName = L2A_UP_ID + '_report.xml'
logDir = config.logDir
logLevel = config.logLevel
fnLog = os.path.join(logDir, logName)
if not os.path.exists(logDir):
os.mkdir(logDir)
try:
f = open(config.processingStatusFn, 'w')
f.write('0.0\n')
f.close()
except:
stderrWrite('cannot create process status file: %s\n' %
config.processingStatusFn)
return FAILURE
try:
f = open(fnLog, 'w')
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<Sen2Cor_Level-2A_Report_File>\n')
f.close()
except:
stderrWrite('cannot update the report file: %s\n' % fnLog)
return FAILURE
# Just a normal logger
logger = logging.getLogger('sen2cor')
handler = logging.FileHandler(fnLog)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.level = getLevel(logLevel)
logger.info('logging for the main process initialized')
config.logger = logger
CFG = 'cfg'
if args.GIP_L2A != None:
config._configFn = os.path.join(config.home, CFG, args.GIP_L2A)
if args.GIP_L2A_SC != None:
config.configSC = os.path.join(config.home, CFG, args.GIP_L2A_SC)
if args.GIP_L2A_AC != None:
config.configAC = os.path.join(config.home, CFG, args.GIP_L2A_AC)
config.workDir = directory
config.resolution = resolution
config.scOnly = args.sc_only
config.crOnly = args.cr_only
config.refresh = args.refresh
config.selectedTile = selectedTile
result = config.readPreferences()
if result == False:
return FAILURE
config.tStart = time()
config.setTimeEstimation(resolution)
L2A_TILES = updateTiles(config)
if L2A_TILES == False:
return FAILURE
result = SUCCESS
if config.crOnly == False:
scheduler = L2A_Schedule(config, L2A_TILES)
result = scheduler.sync()
config.logger = logger
# validate the meta data on user product level:
try:
xp = L2A_XmlParser(config, 'UP2A')
xp.validate()
except:
logger.error('parsing error for user product')
result = FAILURE
# if(args.profile == True):
# pr.disable()
# s = StringIO.StringIO()
# sortby = 'cumulative'
# ps = pstats.Stats(pr, stream=s).sort_stats(sortby).print_stats(.25, 'L2A_')
# ps.print_stats()
# profile = s.getvalue()
# s.close()
# with open(os.path.join(getScriptDir(), 'log', 'profile'), 'w') as textFile:
# textFile.write(profile)
# textFile.close()
else:
config.logger = logger
#Create the manifest.safe (L2A)
mn = L2A_Manifest(config)
mn.generate(config.L2A_UP_DIR, config.L2A_MANIFEST_SAFE)
try:
xp = L2A_XmlParser(config, 'Manifest')
xp.validate()
except:
logger.error('parsing error for manifest')
result = FAILURE
if postprocess(config) == False:
result = FAILURE
if result == FAILURE:
stdoutWrite(
'Progress[%]: 100.00 : Application terminated with at least one error.\n'
)
else:
stdoutWrite(
'Progress[%]: 100.00 : Application terminated successfully.\n')
return result
def generate(self):
DATASTRIP = 'DATASTRIP'
self.logger.stream('Progress[%]: 0.00 : Generating datastrip metadata')
self.L1C_DS_MTD_XML = self.config.L1C_DS_MTD_XML
#input
try:
xp = L2A_XmlParser(self.config, 'DS1C')
if not xp.validate():
self.logger.fatal('Incorrect datastrip L1C xml format')
return FAILURE
tr = xp.getTree('General_Info', 'Datastrip_Time_Info')
sensingStart = tr.DATASTRIP_SENSING_START.text.split('Z')[0]
sensingStart = datetime.strptime(sensingStart,'%Y-%m-%dT%H:%M:%S.%f')
self._sensingStart = strftime('%Y%m%dT%H%M%S', sensingStart.timetuple())
except:
self.logger.fatal('no sensing start in datastrip')
self.time = datetime.utcnow()
generationTimeStr = strftime('%Y%m%dT%H%M%S', self.time.timetuple())
if self.operationMode == 'GENERATE_DATASTRIP':
self.L2A_DS_DIR = self.config.output_dir
self.L2A_DS_ID = '_'.join(['S'+self.config.spacecraftName.split('-')[-1],'OPER_MSI_L2A_DS',self.processing_centre, \
generationTimeStr,'S'+self._sensingStart,self._pbStr])
else: #TOOLBOX
if self.processing_centre == None:
ai = xp.getTree('General_Info','Processing_Info')
self.processing_centre = ai.PROCESSING_CENTER.text
self.L2A_DS_DIR = os.path.join(self.config.L2A_UP_DIR, DATASTRIP)
self.L2A_DS_ID = 'DS_' + self.processing_centre + '_' + generationTimeStr + '_S' + self._sensingStart
self.L2A_DS_MTD_XML = os.path.join(self.L2A_DS_DIR, self.L2A_DS_ID, 'MTD_DS.xml')
#output
newdir = os.path.join(self.L2A_DS_DIR, self.L2A_DS_ID)
if self.operationMode == 'GENERATE_DATASTRIP':
olddir = os.path.join(self.L1C_DS_DIR, self.L1C_DS_ID)
if not os.path.isdir(newdir):
os.makedirs(newdir)
os.makedirs(os.path.join(newdir,'QI_DATA'))
copyfile(self.L1C_DS_MTD_XML,os.path.join(newdir,os.path.basename(self.L1C_DS_MTD_XML)))
else: #TOOLBOX
olddir = os.path.join(self.L2A_DS_DIR,self.L1C_DS_ID)
if os.path.exists(olddir):
os.rename(olddir,newdir)
self.logger.info('datastrip directory is: ' + newdir)
qiDir = os.path.join(newdir, 'QI_DATA')
filelist = sorted(os.listdir(qiDir))
mask = '*.xml'
for fnIn in filelist:
if fnmatch.fnmatch(fnIn, mask):
os.remove(os.path.join(qiDir, fnIn))
if not os.path.isdir(newdir):
self.logger.error('cannot find L2A datastrip directory')
return False
#find L2A datastrip metadada, rename and change it:
L2A_DS_SUBDIR = newdir
L1C_DS_MTD_XML = os.path.basename(self.L1C_DS_MTD_XML)
L2A_DS_MTD_XML = 'MTD_DS.xml'
oldfile = os.path.join(L2A_DS_SUBDIR, L1C_DS_MTD_XML)
newfile = os.path.join(L2A_DS_SUBDIR, L2A_DS_MTD_XML)
self.config.L2A_DS_MTD_XML = newfile
self.L2A_DS_MTD_XML = newfile
os.rename(oldfile, newfile)
found = self.add_new_features()
if not found:
self.logger.fatal('no subdirectory in datastrip')
else:
self.logger.stream('L1C datastrip found, L2A datastrip successfully generated')
return found
def add_new_features(self):
xp = L2A_XmlParser(self.config, 'DS2A')
if (xp.convert() == False):
self.logger.fatal('error in converting datastrip metadata to level 2A')
ti = xp.getTree('Image_Data_Info', 'Tiles_Information')
for tile in ti.Tile_List.iterchildren():
tileIdStr = tile.items()[0][1].replace('L1C','L2A')
tileIdLst = [i for i in tileIdStr.split("_") if i != ""]
if self.processing_centre:
tileIdLst[5] = self.processing_centre
else:
tileIdLst[5] = self.get_processing_centre_from_L1C_metadata()
tileIdLst[6] = strftime('%Y%m%dT%H%M%S', self.time.timetuple())
tileIdLst[-1] = self._pbStr
tileIdStr = "_".join(tileIdLst)
newElement = etree.Element('Tile', tileId = tileIdStr)
ti.Tile_List.replace(tile,newElement)
# writing features in L2A datastrip metadata
pi2a = xp.getTree('General_Info', 'Processing_Info')
pi2a.UTC_DATE_TIME = strftime('%Y-%m-%dT%H:%M:%S.', self.time.timetuple()) + str(self.time.microsecond)[:3]+'Z'
if self.processing_centre:
pi2a.PROCESSING_CENTER = self.processing_centre
auxdata = xp.getTree('Auxiliary_Data_Info', 'GIPP_List')
if self.config.configSC:
dummy, configSC = os.path.split(self.config.configSC)
gippFn = etree.Element('GIPP_FILENAME', type='GIP_L2ACSC',
version='%04d' % long(self.config.processorVersion.replace('.', '')))
gippFn.text = configSC.split('.')[0]
auxdata.append(gippFn)
if self.config.configAC:
dummy, configAC = os.path.split(self.config.configAC)
gippFn = etree.Element('GIPP_FILENAME', type='GIP_L2ACAC',
version='%04d' % long(self.config.processorVersion.replace('.', '')))
gippFn.text = configAC.split('.')[0]
auxdata.append(gippFn)
if self.config.processingBaseline:
baseline = '{:05.2f}'.format(float(self.config.processingBaseline))
gippFn = etree.Element('GIPP_FILENAME', type='GIP_PROBA2', version=self._pbStr[1:].replace('.', ''))
elif self.config.configPB:
pb = L2A_XmlParser(self.config, 'PB_GIPP')
baseline = pb.getTree('DATA', 'Baseline_Version')
gippFn = etree.Element('GIPP_FILENAME', type='GIP_PROBA2', version=baseline.text.replace('.', ''))
dummy, configPB = os.path.split(self.config.configPB)
gippFn.text = configPB.split('.')[0]
auxdata.append(gippFn)
pi2a.PROCESSING_BASELINE = baseline
# SIIMPC-1255 , GNR : update of Sen2cor processing baseline in the datatake ID
di2a = xp.getTree('General_Info', 'Datatake_Info')
datatakeIdentifier = di2a.attrib['datatakeIdentifier'].split('_N')
datatakeIdentifier[-1] = '{:05.2f}'.format(float(baseline))
di2a.attrib['datatakeIdentifier'] = '_N'.join(datatakeIdentifier)
# SIIMPC-1152 RBS Patch: Fix GNR
pic = xp.getTree('Image_Data_Info', 'Radiometric_Info')
pic.QUANTIFICATION_VALUE.tag = 'QUANTIFICATION_VALUES_LIST'
qvl = objectify.Element('QUANTIFICATION_VALUES_LIST')
qvl.BOA_QUANTIFICATION_VALUE = str(int(self.config._dnScale))
qvl.BOA_QUANTIFICATION_VALUE.attrib['unit'] = 'none'
qvl.AOT_QUANTIFICATION_VALUE = str(self.config.L2A_AOT_QUANTIFICATION_VALUE)
qvl.AOT_QUANTIFICATION_VALUE.attrib['unit'] = 'none'
qvl.WVP_QUANTIFICATION_VALUE = str(self.config.L2A_WVP_QUANTIFICATION_VALUE)
qvl.WVP_QUANTIFICATION_VALUE.attrib['unit'] = 'cm'
pic.QUANTIFICATION_VALUES_LIST = qvl
lfn = etree.Element('LUT_FILENAME')
lfn.text = 'None'
auxinfo = xp.getRoot('Auxiliary_Data_Info')
if (xp.getTree('Auxiliary_Data_Info', 'GRI_List')) == False:
gfn = xp.getTree('Auxiliary_Data_Info', 'GRI_FILENAME')
del gfn[:]
gl = objectify.Element('GRI_List')
gl.append(gfn)
auxinfo.append(gl)
try:
ll = xp.getTree('Auxiliary_Data_Info', 'LUT_List')
ll.append(lfn)
except:
ll = objectify.Element('LUT_List')
ll.append(lfn)
auxinfo.append(ll)
try:
esacciWaterBodies = self.config.esacciWaterBodiesReference
esacciWaterBodies = os.path.join(self.config.auxDir, esacciWaterBodies)
esacciLandCover = self.config.esacciLandCoverReference
esacciLandCover = os.path.join(self.config.auxDir, esacciLandCover)
esacciSnowConditionDir = self.config.esacciSnowConditionDirReference
esacciSnowConditionDir = os.path.join(self.config.auxDir, esacciSnowConditionDir)
item = xp.getTree('Auxiliary_Data_Info', 'SNOW_CLIMATOLOGY_MAP')
item._setText(self.config.snowMapReference)
item = xp.getTree('Auxiliary_Data_Info', 'ESACCI_WaterBodies_Map')
if ((os.path.isfile(esacciWaterBodies)) == True):
item._setText(self.config.esacciWaterBodiesReference)
else:
item._setText('None')
item = xp.getTree('Auxiliary_Data_Info', 'ESACCI_LandCover_Map')
if ((os.path.isfile(esacciLandCover)) == True):
item._setText(self.config.esacciLandCoverReference)
else:
item._setText('None')
item = xp.getTree('Auxiliary_Data_Info', 'ESACCI_SnowCondition_Map_Dir')
if (os.path.isdir(esacciSnowConditionDir)) == True:
item._setText(self.config.esacciSnowConditionDirReference)
else:
item._setText('None')
except:
item = etree.Element('SNOW_CLIMATOLOGY_MAP')
item.text = self.config.snowMapReference
auxinfo.append(item)
item = etree.Element('ESACCI_WaterBodies_Map')
if ((os.path.isfile(esacciWaterBodies)) == True):
item.text = self.config.esacciWaterBodiesReference
else:
item.text = 'None'
auxinfo.append(item)
item = etree.Element('ESACCI_LandCover_Map')
if ((os.path.isfile(esacciLandCover)) == True):
item.text = self.config.esacciLandCoverReference
else:
item.text = 'None'
auxinfo.append(item)
item = etree.Element('ESACCI_SnowCondition_Map_Dir')
if (os.path.isdir(esacciSnowConditionDir)) == True:
item.text = self.config.esacciSnowConditionDirReference
else:
item.text = 'None'
auxinfo.append(item)
# Addon UMW 180326: set production DEM with L2A info:
item = xp.getTree('Auxiliary_Data_Info', 'PRODUCTION_DEM_TYPE')
if self.config.demDirectory == 'NONE':
item._setText('None')
else:
item._setText(self.config.demReference)
self.time = datetime.utcnow()
ai2a = xp.getTree('General_Info', 'Archiving_Info')
ai2a.ARCHIVING_TIME = strftime('%Y-%m-%dT%H:%M:%S.', self.time.timetuple()) + str(self.time.microsecond)[:3]+'Z'
if self.archiving_centre:
ai2a.ARCHIVING_CENTRE = self.archiving_centre
xp.export()
xp.validate()
return True
def get_processing_centre_from_L1C_metadata(self):
xp = L2A_XmlParser(self.config, 'DS1C')
pi1c = xp.getTree('General_Info', 'Processing_Info')
self.processing_centre = pi1c.PROCESSING_CENTER.text
return self.processing_centre