def parNotFound(self, parameter):
''' Throw a fatal error message if a configuration parameter is not found and terminate the application.
'''
self.logger.fatal('Configuration parameter %s not found in %s', parameter, self._configFn)
stderrWrite('Configuration parameter <%s> not found in %s\n' % (parameter, self._configFn))
stderrWrite('Program is forced to terminate.')
self.__exit__()
def process(self, tables):
''' Perform the L3 processing.
:param tables: the table object for the current tile
:type tables: a reference to the table object
:return: true if processing succeeds, false else
:rtype: bool
'''
self._tables = tables
astr = 'L3_Process: processing with resolution ' + str(self.config.resolution) + ' m'
self.config.timestamp(astr)
self.config.timestamp('L3_Process: start of Pre Processing')
if(self.preprocess() == False):
return False
self.config.timestamp('L3_Process: start of Spatio Temporal Processing')
self.config.logger.info('Performing Spatio Temporal Processing with resolution %d m', self.config.resolution)
if(self._l3Synthesis.process(self._tables) == False):
return False
# append processed tile to list
processedTile = self.config.product.L2A_TILE_ID + '_' + str(self.config.resolution) + '\n'
processedFn = self.config.sourceDir + '/' + 'processed'
try:
f = open(processedFn, 'a')
f.write(processedTile)
f.flush()
f.close()
except:
stderrWrite('Could not update processed tile history.\n')
self.config.exitError()
return False
return True
def updateUserProduct(self, userProduct):
''' Update the current user product.
Check if at target product is present.
:param userProduct: the user product identifier.
:type userProduct: str
:return: true, if target product exists.
:rtype: boolean
'''
self.config.L2A_UP_ID = userProduct
self.config.L2A_UP_DIR = os.path.join(self.config.sourceDir,
userProduct)
if self.config.namingConvention == 'SAFE_STANDARD':
filelist = sorted(os.listdir(self.config.L2A_UP_DIR))
filemask = 'S2A_USER_MTD_*.xml'
for file in filelist:
if fnmatch.fnmatch(file, filemask):
self.config.L2A_UP_MTD_XML = os.path.join(
self.config.L2A_UP_DIR, file)
break
else:
self.config.L2A_UP_MTD_XML = os.path.join(self.config.L2A_UP_DIR,
'MTD_MSIL2A.xml')
if self.existL3_TargetProduct(userProduct) == False:
stderrWrite('directory "%s" L3 target product is missing\n.' %
self.config.targetDir)
self.config.exitError()
return True
def reinitL3_TargetProduct(self):
''' Reinit the L3 target product
:return: true if succesful
:rtype: bool
'''
L3_DS_ID = None
L3_TARGET_MASK = '*L03_*'
dirlist = sorted(os.listdir(self.config.targetDir))
for L3_TARGET_ID in dirlist:
if fnmatch.fnmatch(L3_TARGET_ID, L3_TARGET_MASK) == True:
self.config.L3_TARGET_ID = L3_TARGET_ID
self.config.L3_TARGET_DIR = os.path.join(
self.config.targetDir, L3_TARGET_ID)
break
# create user product:
filelist = sorted(os.listdir(self.config.L3_TARGET_DIR))
found = False
for filename in filelist:
if (fnmatch.fnmatch(filename, 'MTD_MSIL03.xml') == True):
found = True
break
if found == False:
stderrWrite('No metadata for user product')
self.config.exitError()
self.config.L3_TARGET_MTD_XML = os.path.join(self.config.L3_TARGET_DIR,
filename)
xp = L3_XmlParser(self.config, 'UP03')
l3qii = xp.getRoot('Quality_Indicators_Info')
node = objectify.Element('Classification_QI')
node.attrib['resolution'] = str(self.config.resolution)
if self.insert(l3qii, node):
xp.export()
L3_DS_MASK = 'DS_*'
DATASTRIP = 'DATASTRIP'
L3_DS_DIR = os.path.join(self.config.targetDir, L3_TARGET_ID,
DATASTRIP)
dirlist = sorted(os.listdir(L3_DS_DIR))
for L3_DS_ID in dirlist:
if fnmatch.fnmatch(L3_DS_ID, L3_DS_MASK) == True:
self.config.L3_DS_ID = L3_DS_ID
break
if L3_DS_ID is not None:
L3_DS_MTD_XML = 'MTD_DS.xml'
self.config.L3_DS_MTD_XML = os.path.join(L3_DS_DIR, L3_DS_ID,
L3_DS_MTD_XML)
return True
return False
def setUpMetadataId(self, L1C_UP_DIR):
S2A_mask = 'S2A_*_MTD_*1C_*.xml'
filelist = sorted(os.listdir(L1C_UP_DIR))
found = False
for filename in filelist:
if(fnmatch.fnmatch(filename, S2A_mask) == True):
found = True
break
if found == False:
stderrWrite('No metadata for user product.\n')
self.config.exitError()
self.config.product.L1C_UP_MTD_XML = os.path.join(L1C_UP_DIR, filename)
return
def updateUserProduct(self, userProduct):
''' Update the current user product.
Check if at target product is present.
:param userProduct: the user product identifier.
:type userProduct: str
:return: true, if target product exists.
:rtype: boolean
'''
self.product.L2A_UP_ID = userProduct
if self.product.existL3_TargetProduct() == False:
stderrWrite('directory "%s" L3 target product is missing\n.' % self.targetDir)
self.exitError()
return True
def updateUserProduct(self, userProduct):
''' Update the current user product.
Check if at target product is present.
:param userProduct: the user product identifier.
:type userProduct: str
:return: true, if target product exists.
:rtype: boolean
'''
self.config.L2A_UP_ID = userProduct
if self.existL3_TargetProduct() == False:
stderrWrite('directory "%s" L3 target product is missing\n.' %
self.config.targetDir)
self.config.exitError()
return True
def exitError(self, reason = None):
''' Throw an error message if a fatal error occurred and and terminate the application.
'''
stderrWrite('Fatal error occurred, see report file for details.')
if reason: stderrWrite('Reason: ' + reason)
self.__exit__()
def main(args=None):
''' Processes command line,
initializes the config and product modules and starts sequentially
the L3 processing
'''
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the Level-2A input files are located')
parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed')
parser.add_argument('--clean', action='store_true', help='Removes the L3 product in the target directory before processing. Be careful!')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if not os.path.isabs(directory):
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
directory = os.path.normpath(directory)
if not os.path.exists(directory):
stderrWrite('directory "%s" does not exist\n.' % directory)
return False
if not args.resolution:
resolutions = [60,20,10]
else:
resolutions = [args.resolution]
cleanDone = False # do only one clean up of target, if requested.
for resolution in resolutions:
config = L3_Config(resolution, directory)
result = False
processedFn = os.path.join(directory, 'processed')
if args.clean and not cleanDone:
stdoutWrite('Cleaning target directory ...\n')
config.cleanTarget = True
try:
os.remove(processedFn)
except:
stdoutWrite('No history file present ...\n')
cleanDone = True
config.init(processorVersion)
processor = doTheLoop(config)
if processor == -1:
stderrWrite('Application terminated with errors, see log file and traces.\n')
return 1
elif processor == None:
stdoutWrite('All tiles already processed.\n')
else:
processor.postProcessing()
stdoutWrite('Application terminated successfully.\n')
return 0
def doTheLoop(config):
''' Initializes a product and processor object. Cycles through all input products and granules
and calls the sequential processing of the individual tiles if the criteria for processing are fulfilled.
:param config: the config object
:type config: a reference to the config object
:return: the processor object, for doing the preprocessing, -1 if processing error occurred.
:rtype: processor object or -1
'''
HelloWorld = processorName + ', ' + processorVersion + ', created: ' + processorDate
stdoutWrite('\n%s started with %dm resolution ...\n' % (HelloWorld, config.resolution))
dirlist = os.listdir(config.sourceDir)
upList = sortObservationStartTime(dirlist)
tileFilter = config.tileFilter
# Process all unprocessed L2A products:
UP_mask = '*_MSIL2A_*'
product = L3_Product(config)
processor = L3_Process(config)
proc = None
for L2A_UP_ID in upList:
if not config.checkTimeRange(L2A_UP_ID):
continue
product.updateUserProduct(L2A_UP_ID)
if config.namingConvention == 'SAFE_STANDARD':
Tile_mask = '*L2A_*'
else:
Tile_mask = 'L2A_*'
GRANULE = os.path.join(config.sourceDir, L2A_UP_ID, 'GRANULE')
tilelist = sorted(os.listdir(GRANULE))
for tile in tilelist:
# process only L2A tiles:
if not fnmatch.fnmatch(tile, Tile_mask):
continue
# ignore already processed tiles:
if config.tileExists(tile):
continue
# apply tile filter:
if not config.tileIsSelected(tile, tileFilter):
continue
if not config.checkTileConsistency(GRANULE, tile):
continue
tStart = time()
product.config.L2A_TILE_ID = tile
tables = L3_Tables(product)
tables.init()
# no processing if first initialisation:
# check existence of Bands - B2 is always present:
if not tables.testBand('L2A', tables.B02):
# append processed tile to list
if not config.appendTile():
config.exitError()
continue
proc, result = processor.process(tables)
if result == -1:
stderrWrite('Application terminated with errors, see log file and traces.\n')
return result
elif result == 1:
return proc
elif result == 0:
tMeasure = time() - tStart
config.writeTimeEstimation(config.resolution, tMeasure)
return proc
def main(args=None):
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the reflectance input files are located')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if(os.path.isabs(directory)) == False:
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
elif os.path.exists(args.directory) == False:
stderrWrite('directory "%s" does not exist\n.' % args.directory)
return False
config = L3_Config('60', directory, 'R2R_GIPP')
config.init(processorVersion)
result = False
HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate
stdoutWrite('\n%s started ...\n' % HelloWorld)
processor = R2R_Converter(config)
L1C_mask = 'S2?_*L1C_*'
exclusionMask = 'S2?_*_RAD'
upList = sorted(os.listdir(directory))
l1cList = []
nrUserProducts = 0
for upId in upList:
if(fnmatch.fnmatch(upId, exclusionMask) == True):
continue
elif(fnmatch.fnmatch(upId, L1C_mask) == True):
nrUserProducts += 1
l1cList.append(upId)
for upId in l1cList:
stdoutWrite('%d user product(s) to be converted ...\n' % nrUserProducts)
nrUserProducts -=1
upFullPath = os.path.join(directory, upId)
targetDir = config.targetDir
if targetDir == 'DEFAULT':
targetDir = directory + '_RAD'
else:
targetDir = os.path.join(targetDir, upId)
copy_tree(upFullPath, targetDir)
processor.setUpMetadataId(targetDir)
result = processor.convert(targetDir)
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
else:
config.logger.info('User product ' + upId + ' converted')
stdoutWrite('%d user product(s) remain\n\n' % nrUserProducts)
stdoutWrite('Application terminated successfully.\n')
return result
def createL3_TargetProduct(self):
''' Create the L3 target product.
:return: true if succesful
:rtype: bool
'''
self.config.logger.info('Creating L3 target product ...')
L2A_UP_MASK = '*L2A_*'
#L2A_UP_DIR = os.path.join(self.config.sourceDir, self.config.L2A_UP_ID)
L2A_UP_DIR = self.config.L2A_UP_DIR
# detect the filename for the datastrip metadata:
L2A_DS_DIR = os.path.join(L2A_UP_DIR, 'DATASTRIP')
if os.path.exists(L2A_DS_DIR) == False:
stderrWrite('directory "%s" does not exist.\n' % L2A_DS_DIR)
self.config.exitError()
return False
if self.config.namingConvention == 'SAFE_STANDARD':
L2A_DS_MASK = 'S2?_USER_MSI_L2A_DS_*'
else:
L2A_DS_MASK = 'DS_*'
dirlist = sorted(os.listdir(L2A_DS_DIR))
found = False
for dirname in dirlist:
if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True):
found = True
break
L2A_DS_DIR = os.path.join(L2A_DS_DIR, dirname)
if self.config.namingConvention == 'SAFE_STANDARD':
L2A_DS_MTD_XML = (dirname[:-7] + '.xml').replace('_MSI_', '_MTD_')
else:
L2A_DS_MTD_XML = 'MTD_DS.xml'
L2A_DS_MTD_XML = os.path.join(L2A_DS_DIR, L2A_DS_MTD_XML)
if found == False:
stderrWrite('No metadata in datastrip\n.')
self.config.exitError()
dirname, basename = os.path.split(L2A_UP_DIR)
if (fnmatch.fnmatch(basename, L2A_UP_MASK) == False):
stderrWrite(basename +
': identifier "%s" is missing.' % L2A_UP_MASK)
self.config.exitError()
return False
GRANULE = os.path.join(L2A_UP_DIR, 'GRANULE')
if os.path.exists(GRANULE) == False:
stderrWrite('directory "' + GRANULE + '" does not exist.')
self.config.exitError()
return False
#
# the product (directory) structure:
#-------------------------------------------------------
L3_TARGET_ID = basename
L3_TARGET_ID = L3_TARGET_ID.replace('L2A_', 'L03_')
strList = L3_TARGET_ID.split('_')
if self.config.namingConvention == 'SAFE_STANDARD':
L3_TARGET_ID = strList[0] + '_' + strList[3] + '_' + strList[
5] + '_' + 'N0000_R000_T00XXX_' + self.config.minTime + '.SAFE'
else:
L3_TARGET_ID = L3_TARGET_ID.replace(strList[6],
self.config.minTime + '.SAFE')
if self.config.targetDir != 'DEFAULT':
targetDir = self.config.targetDir
if os.name == 'nt' and not '\\\\?\\' in targetDir:
targetDir = u'\\'.join([u'\\\\?', targetDir])
if (os.path.exists(targetDir) == False):
os.mkdir(targetDir)
else:
targetDir = dirname
self.config.targetDir = targetDir
L3_TARGET_DIR = os.path.join(targetDir, L3_TARGET_ID)
self.config.L3_TARGET_DIR = L3_TARGET_DIR
self.config.L3_TARGET_ID = L3_TARGET_ID
L2A_INSPIRE_XML = os.path.join(L2A_UP_DIR, 'INSPIRE.xml')
L2A_MANIFEST_SAFE = os.path.join(L2A_UP_DIR, 'manifest.safe')
L3_INSPIRE_XML = os.path.join(L3_TARGET_DIR, 'INSPIRE.xml')
L3_MANIFEST_SAFE = os.path.join(L3_TARGET_DIR, 'manifest.safe')
AUX_DATA = 'AUX_DATA'
DATASTRIP = 'DATASTRIP'
GRANULE = 'GRANULE'
HTML = 'HTML'
REP_INFO = 'rep_info'
self.config.L2A_DS_MTD_XML = L2A_DS_MTD_XML
xp = L3_XmlParser(self.config, 'DS2A')
xp.export()
xp.validate()
copy_tree(os.path.join(L2A_UP_DIR, AUX_DATA),
os.path.join(L3_TARGET_DIR, AUX_DATA))
copy_tree(os.path.join(L2A_UP_DIR, DATASTRIP),
os.path.join(L3_TARGET_DIR, DATASTRIP))
copy_tree(os.path.join(L2A_UP_DIR, HTML),
os.path.join(L3_TARGET_DIR, HTML))
copy_tree(os.path.join(L2A_UP_DIR, REP_INFO),
os.path.join(L3_TARGET_DIR, REP_INFO))
copy_file(L2A_INSPIRE_XML, L3_INSPIRE_XML)
copy_file(L2A_MANIFEST_SAFE, L3_MANIFEST_SAFE)
if not os.path.exists(os.path.join(L3_TARGET_DIR, GRANULE)):
os.mkdir(os.path.join(L3_TARGET_DIR, GRANULE))
self.config.L3_INSPIRE_XML = L2A_INSPIRE_XML
self.config.L3_MANIFEST_SAFE = L2A_MANIFEST_SAFE
self.config.L3_TARGET_DIR = L3_TARGET_DIR
#create user product:
if self.config.namingConvention == 'SAFE_STANDARD':
S2A_mask = 'S2?_USER_MTD_SAFL2A_*.xml'
else:
S2A_mask = 'MTD_MSIL2A.xml'
filelist = sorted(os.listdir(L2A_UP_DIR))
found = False
for filename in filelist:
if (fnmatch.fnmatch(filename, S2A_mask) == True):
found = True
break
if found == False:
stderrWrite('No metadata for user product')
self.config.exitError()
# prepare L3 User Product metadata file
fn_L2A = os.path.join(L2A_UP_DIR, filename)
fn_L3 = 'MTD_MSIL03.xml'
fn_L3 = os.path.join(L3_TARGET_DIR, fn_L3)
self.config.L2A_UP_MTD_XML = fn_L2A
xp = L3_XmlParser(self.config, 'UP2A')
if xp.convert() != 0:
xp = L3_XmlParser(self.config, 'UP2A')
xp.validate()
self.config.L3_TARGET_MTD_XML = fn_L3
# copy L2A schemes from config_dir into rep_info:
upScheme3 = self.config.upScheme3
basename = os.path.basename(upScheme3)
copy_file(os.path.join(self.config.configDir, upScheme3),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
tileScheme3 = self.config.tileScheme3
basename = os.path.basename(tileScheme3)
copy_file(os.path.join(self.config.configDir, tileScheme3),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
dsScheme3 = self.config.dsScheme3
basename = os.path.basename(dsScheme3)
copy_file(os.path.join(self.config.configDir, dsScheme3),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
# copy L3 User Product metadata file:
copy_file(fn_L2A, fn_L3)
# remove old L2A entries from L3_TARGET_MTD_XML:
xp = L3_XmlParser(self.config, 'UP03')
if (xp.convert() == False):
self.logger.fatal(
'error in converting user product metadata to level 3')
stderrWrite('Error in converting user product metadata to level 3')
self.config.exitError()
xp = L3_XmlParser(self.config, 'UP03')
pi = xp.getTree('General_Info', 'Product_Info')
# update L2A entries from L2A_UP_MTD_XML:
tmin = self.config.minTime
tmax = self.config.maxTime
pi.PRODUCT_START_TIME = tmin[:4] + '-' + tmin[4:6] + '-' + tmin[
6:11] + ':' + tmin[11:13] + ':' + tmin[13:15] + '.000Z'
pi.PRODUCT_STOP_TIME = tmax[:4] + '-' + tmax[4:6] + '-' + tmax[
6:11] + '-' + tmax[11:13] + ':' + tmax[13:15] + '.000Z'
# update L2A and L3 entries from L2A_UP_MTD_XML:
try:
del pi.PRODUCT_URI_1C
except:
pass
pi.PRODUCT_URI = self.config.L3_TARGET_ID
pi.PROCESSING_LEVEL = 'Level-3p'
pi.PRODUCT_TYPE = 'S2MSI3p'
pi.PROCESSING_ALGORITHM = self.config.algorithm
pi.RADIOMETRIC_PREFERENCE = self.config.radiometricPreference
dt = datetime.utcnow()
pi.GENERATION_TIME = strftime('%Y-%m-%dT%H:%M:%SZ', dt.timetuple())
if self.config.namingConvention == 'SAFE_STANDARD':
try:
qo = pi.Query_Options
del qo[:]
qo = objectify.Element('Query_Options')
qo.attrib['completeSingleTile'] = "false"
qo.append(objectify.Element('PRODUCT_FORMAT'))
qo.PRODUCT_FORMAT = 'SAFE_COMPACT'
pi.append(qo)
except:
pass
try:
gl = pi.Product_Organisation.Granule_List
del gl[:]
except:
pass
try:
pic = xp.getTree('General_Info', 'Product_Image_Characteristics')
del pic.QUANTIFICATION_VALUES_LIST.L1C_TOA_QUANTIFICATION_VALUE
except:
pass
try:
aux = xp.getRoot('L2A_Auxiliary_Data_Info')
del aux[:]
except:
pass
try:
aux = xp.getRoot('Auxiliary_Data_Info')
aux.clear()
node = objectify.Element('GIPP_LIST')
aux.append(node)
except:
pass
try:
qii = xp.getRoot('L2A_Quality_Indicators_Info')
del qii[:]
except:
pass
try:
qii = xp.getRoot('Quality_Indicators_Info')
qii.clear()
node = objectify.Element('Classification_QI')
node.attrib['resolution'] = str(self.config.resolution)
qii.append(node)
except:
pass
xp.export()
#create datastrip ID:
L3_DS_DIR = os.path.join(L3_TARGET_DIR, DATASTRIP)
dirlist = sorted(os.listdir(L3_DS_DIR))
found = False
for dirname in dirlist:
if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True):
found = True
break
if found == False:
stderrWrite('No subdirectory in datastrip')
self.config.exitError()
if self.config.namingConvention == 'SAFE_STANDARD':
L3_DS_ID = dirname[17:-7]
L3_DS_OLD = os.path.join(L3_DS_DIR, dirname)
L3_DS_DIR = os.path.join(L3_DS_DIR, L3_DS_ID)
os.rename(L3_DS_OLD, L3_DS_DIR)
else:
L3_DS_ID = dirname
L3_DS_DIR = os.path.join(L3_DS_DIR, L3_DS_ID)
self.config.L3_DS_ID = L3_DS_ID
filelist = sorted(os.listdir(L3_DS_DIR))
found = False
L2A_DS_MTD_MSK = 'S2?_USER_MTD_L2A_DS_*.xml'
for filename in filelist:
if (fnmatch.fnmatch(filename, L2A_DS_MTD_MSK) == True):
L3_DS_OLD = os.path.join(L3_DS_DIR, filename)
L3_DS_MTD = os.path.join(L3_DS_DIR, 'MTD_DS.xml')
os.rename(L3_DS_OLD, L3_DS_MTD)
filename = 'MTD_DS.xml'
found = True
elif (fnmatch.fnmatch(filename, 'MTD_DS.xml') == True):
found = True
break
if found == False:
stderrWrite('No metadata in datastrip')
self.config.exitError()
self.config.L3_DS_MTD_XML = os.path.join(L3_DS_DIR, filename)
xp = L3_XmlParser(self.config, 'DS03')
if (xp.convert() == False):
stderrWrite('Error in converting datastrip metadata to level 3.')
self.logger.fatal(
'error in converting datastrip metadata to level 3.')
self.config.exitError()
xp = L3_XmlParser(self.config, 'DS03')
ti = xp.getTree('Image_Data_Info', 'Tiles_Information')
del ti.Tile_List.Tile[:]
ri = xp.getTree('Image_Data_Info', 'Radiometric_Info')
qvl = objectify.Element('QUANTIFICATION_VALUES_LIST')
qvl.BOA_QUANTIFICATION_VALUE = str(int(self.config.dnScale))
qvl.BOA_QUANTIFICATION_VALUE.attrib['unit'] = 'none'
qvl.AOT_QUANTIFICATION_VALUE = str(
self.config.L2A_AOT_QUANTIFICATION_VALUE)
qvl.AOT_QUANTIFICATION_VALUE.attrib['unit'] = 'none'
qvl.WVP_QUANTIFICATION_VALUE = str(
self.config.L2A_WVP_QUANTIFICATION_VALUE)
qvl.WVP_QUANTIFICATION_VALUE.attrib['unit'] = 'cm'
ri.QUANTIFICATION_VALUES_LIST = qvl
auxinfo = xp.getRoot('Auxiliary_Data_Info')
if (xp.getTree('Auxiliary_Data_Info', 'GRI_List')) == False:
gfn = xp.getTree('Auxiliary_Data_Info', 'GRI_FILENAME')
del gfn[:]
gl = etree.Element('GRI_List')
gl.append(gfn)
auxinfo.append(gl)
if (xp.getTree('Auxiliary_Data_Info', 'LUT_List')) == False:
ll = etree.Element('LUT_List')
auxinfo.append(ll)
if (xp.getTree('Auxiliary_Data_Info',
'SNOW_CLIMATOLOGY_MAP')) == False:
ll = etree.Element('SNOW_CLIMATOLOGY_MAP')
ll.text = 'None'
auxinfo.append(ll)
if (xp.getTree('Auxiliary_Data_Info',
'ESACCI_WaterBodies_Map')) == False:
ll = etree.Element('ESACCI_WaterBodies_Map')
ll.text = 'None'
auxinfo.append(ll)
if (xp.getTree('Auxiliary_Data_Info',
'ESACCI_LandCover_Map')) == False:
ll = etree.Element('ESACCI_LandCover_Map')
ll.text = 'None'
auxinfo.append(ll)
if (xp.getTree('Auxiliary_Data_Info',
'ESACCI_SnowCondition_Map_Dir')) == False:
ll = etree.Element('ESACCI_SnowCondition_Map_Dir')
ll.text = 'None'
auxinfo.append(ll)
xp.export()
self.createTable()
return True
def main(args=None):
''' Processes command line,
initializes the config and product modules and starts sequentially
the L2A and L3 processing.
'''
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the Level-2A input files are located')
parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, must be 10, 20 or 60 [m]')
parser.add_argument('--clean', action='store_true', help='Removes all processed files in target directory. Be careful!')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if(os.path.isabs(directory)) == False:
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
elif os.path.exists(args.directory) == False:
stderrWrite('directory "%s" does not exist\n.' % args.directory)
return False
if args.resolution == None:
resolution = 60
else:
resolution = args.resolution
config = L3_Config(resolution, directory)
config.init(processorVersion)
processedTiles = ''
result = False
processedFn = directory + '/' + 'processed'
if args.clean:
stdoutWrite('Cleaning target directory ...\n')
try:
shutil.rmtree(config.targetDir)
except:
pass
try:
os.remove(processedFn)
except:
stdoutWrite('No history file present ...\n')
HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate
stdoutWrite('\n%s started ...\n' % HelloWorld)
upList = sorted(os.listdir(directory))
# Check if unprocessed L1C products exist. If yes, process first:
L1C_mask = 'S2?_*L1C_*'
product = L3_Product(config)
processor = L2A_Process(config)
for L1C_UP_ID in upList:
if(fnmatch.fnmatch(L1C_UP_ID, L1C_mask) == False):
continue
if config.checkTimeRange(L1C_UP_ID) == False:
continue
tilelist = product.createL2A_UserProduct(L1C_UP_ID)
for tile in tilelist:
# process only L1C tiles:
if fnmatch.fnmatch(tile, L1C_mask) == False:
continue
# ignore already processed tiles:
if product.tileExists(tile) == True:
continue
# finally, process the remaining tiles:
stdoutWrite('\nL1C tile %s found, will be classified first ...\n' % tile)
tStart = time()
tables = L2A_Tables(config, tile)
if processor.process(tables) == False:
config.exitError()
return False
tile += '_' + str(config.resolution)
if product.appendTile(tile) == False:
config.exitError()
return False
# Now process all unprocessed L2A products:
L2A_mask = 'S2?_*L2A_*'
processor = L3_Process(config)
for L2A_UP_ID in upList:
if(fnmatch.fnmatch(L2A_UP_ID, L2A_mask) == False):
continue
if config.checkTimeRange(L2A_UP_ID) == False:
continue
config.updateUserProduct(L2A_UP_ID)
GRANULE = directory + '/' + L2A_UP_ID + '/GRANULE'
tilelist = sorted(os.listdir(GRANULE))
for tile in tilelist:
# process only L2A tiles:
if fnmatch.fnmatch(tile, L2A_mask) == False:
continue
# ignore already processed tiles:
if product.tileExists(tile):
continue
tStart = time()
nrTilesProcessed = product.getNrTilesProcessed(tile)
config.updateTile(tile, nrTilesProcessed)
tables = L3_Tables(config)
tables.init()
# no processing if first initialisation:
# check existence of Bands - B2 is always present:
if tables.testBand('L2A', 2) == False:
# append processed tile to list
tile += '_' + str(config.resolution)
if product.appendTile(tile) == False:
config.exitError()
continue
result = processor.process(tables)
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
tMeasure = time() - tStart
config.writeTimeEstimation(resolution, tMeasure)
if result == True:
result = processor.postprocess()
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
stdoutWrite('\nApplication terminated successfully.\n')
return result
def createL3_TargetProduct(self):
''' Create the L3 target product.
:return: true if succesful
:rtype: bool
'''
self.config.logger.info('Creating L3 target product ...')
L2A_UP_MASK = '*2A_*'
L2A_UP_DIR = os.path.join(self.config.sourceDir, self.config.L2A_UP_ID)
# detect the filename for the datastrip metadata:
L2A_DS_DIR = os.path.join(L2A_UP_DIR, 'DATASTRIP')
if os.path.exists(L2A_DS_DIR) == False:
stderrWrite('directory "%s" does not exist.\n' % L2A_DS_DIR)
self.config.exitError()
return False
L2A_DS_MASK = '*_L2A_*'
dirlist = sorted(os.listdir(L2A_DS_DIR))
found = False
for dirname in dirlist:
if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True):
found = True
break
L2A_DS_DIR = os.path.join(L2A_DS_DIR, dirname)
L2A_DS_MTD_XML = (dirname[:-7] + '.xml').replace('_MSI_', '_MTD_')
L2A_DS_MTD_XML = os.path.join(L2A_DS_DIR, L2A_DS_MTD_XML)
if found == False:
stderrWrite('No metadata in datastrip\n.')
self.config.exitError()
dirname, basename = os.path.split(L2A_UP_DIR)
if (fnmatch.fnmatch(basename, L2A_UP_MASK) == False):
stderrWrite(basename + ': identifier "*2A_*" is missing.')
self.config.exitError()
return False
GRANULE = os.path.join(L2A_UP_DIR, 'GRANULE')
if os.path.exists(GRANULE) == False:
stderrWrite('directory "' + GRANULE + '" does not exist.')
self.config.exitError()
return False
#
# the product (directory) structure:
#-------------------------------------------------------
L3_TARGET_ID = basename
L3_TARGET_ID = L3_TARGET_ID.replace('L2A_', 'L03_')
L3_TARGET_ID = L3_TARGET_ID.replace(L3_TARGET_ID[47:62],
self.config.minTime)
L3_TARGET_ID = L3_TARGET_ID.replace(L3_TARGET_ID[63:78],
self.config.maxTime)
if self.config.targetDir != 'DEFAULT':
targetDir = self.config.targetDir
if os.name == 'nt' and not '\\\\?\\' in targetDir:
targetDir = u'\\'.join([u'\\\\?', targetDir])
if (os.path.exists(targetDir) == False):
os.mkdir(targetDir)
else:
targetDir = dirname
self.config.targetDir = targetDir
L3_TARGET_DIR = os.path.join(targetDir, L3_TARGET_ID)
self.config.L3_TARGET_DIR = L3_TARGET_DIR
self.config.L3_TARGET_ID = L3_TARGET_ID
L2A_INSPIRE_XML = os.path.join(L2A_UP_DIR, 'INSPIRE.xml')
L2A_MANIFEST_SAFE = os.path.join(L2A_UP_DIR, 'manifest.safe')
L3_INSPIRE_XML = os.path.join(L3_TARGET_DIR, 'INSPIRE.xml')
L3_MANIFEST_SAFE = os.path.join(L3_TARGET_DIR, 'manifest.safe')
AUX_DATA = 'AUX_DATA'
DATASTRIP = 'DATASTRIP'
GRANULE = 'GRANULE'
HTML = 'HTML'
REP_INFO = 'rep_info'
self.config.L2A_DS_MTD_XML = L2A_DS_MTD_XML
xp = L3_XmlParser(self.config, 'DS2A')
xp.export()
xp.validate()
copy_tree(os.path.join(L2A_UP_DIR, AUX_DATA),
os.path.join(L3_TARGET_DIR, AUX_DATA))
copy_tree(os.path.join(L2A_UP_DIR, DATASTRIP),
os.path.join(L3_TARGET_DIR, DATASTRIP))
copy_tree(os.path.join(L2A_UP_DIR, HTML),
os.path.join(L3_TARGET_DIR, HTML))
copy_tree(os.path.join(L2A_UP_DIR, REP_INFO),
os.path.join(L3_TARGET_DIR, REP_INFO))
copy_file(L2A_INSPIRE_XML, L3_INSPIRE_XML)
copy_file(L2A_MANIFEST_SAFE, L3_MANIFEST_SAFE)
if not os.path.exists(os.path.join(L3_TARGET_DIR, GRANULE)):
os.mkdir(os.path.join(L3_TARGET_DIR, GRANULE))
self.config.L3_INSPIRE_XML = L2A_INSPIRE_XML
self.config.L3_MANIFEST_SAFE = L2A_MANIFEST_SAFE
self.config.L3_TARGET_DIR = L3_TARGET_DIR
#create user product:
S2A_mask = 'S2A_*'
filelist = sorted(os.listdir(L2A_UP_DIR))
found = False
for filename in filelist:
if (fnmatch.fnmatch(filename, S2A_mask) == True):
found = True
break
if found == False:
stderrWrite('No metadata for user product')
self.config.exitError()
# prepare L3 User Product metadata file
fn_L2A = os.path.join(L2A_UP_DIR, filename)
fn_L3 = filename[:4] + 'USER' + filename[8:]
fn_L3 = fn_L3.replace('L2A_', 'L03_')
fn_L3 = os.path.join(L3_TARGET_DIR, fn_L3)
self.config.L2A_UP_MTD_XML = fn_L2A
xp = L3_XmlParser(self.config, 'UP2A')
xp.export()
xp.validate()
self.config.L3_TARGET_MTD_XML = fn_L3
# copy L2A schemes from config_dir into rep_info:
xp = L3_XmlParser(self.config, 'GIPP')
cs = xp.getRoot('Common_Section')
upScheme2a = cs.UP_Scheme_2A.text
basename = os.path.basename(upScheme2a)
copy_file(os.path.join(self.config.configDir, upScheme2a),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
tileScheme2a = cs.Tile_Scheme_2A.text
basename = os.path.basename(tileScheme2a)
copy_file(os.path.join(self.config.configDir, tileScheme2a),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
dsScheme2a = cs.DS_Scheme_2A.text
basename = os.path.basename(dsScheme2a)
copy_file(os.path.join(self.config.configDir, dsScheme2a),
os.path.join(L3_TARGET_DIR, REP_INFO, basename))
# copy L3 User Product metadata file:
copy_file(fn_L2A, fn_L3)
# remove old L2A entries from L3_TARGET_MTD_XML:
xp = L3_XmlParser(self.config, 'UP03')
if (xp.convert() == False):
self.logger.fatal(
'error in converting user product metadata to level 3')
stderrWrite('Error in converting user product metadata to level 3')
self.config.exitError()
xp = L3_XmlParser(self.config, 'UP03')
pi = xp.getTree('General_Info', 'L3_Product_Info')
# update L2A entries from L2A_UP_MTD_XML:
# 2015-07-30T10:39:14.021Z
tmin = self.config.minTime
tmax = self.config.maxTime
pi.PRODUCT_START_TIME = tmin[:4] + '-' + tmin[4:6] + '-' + tmin[
6:11] + ':' + tmin[11:13] + ':' + tmin[13:15] + '.000Z'
pi.PRODUCT_STOP_TIME = tmax[:4] + '-' + tmax[4:6] + '-' + tmax[
6:11] + '-' + tmax[11:13] + ':' + tmax[13:15] + '.000Z'
pi.PROCESSING_LEVEL = 'Level-3p'
pi.PRODUCT_TYPE = 'S2MSI3p'
pi.PROCESSING_ALGORITHM = self.config.algorithm
pi.RADIOMETRIC_PREFERENCE = self.config.radiometricPreference
dt = datetime.utcnow()
pi.GENERATION_TIME = strftime('%Y-%m-%dT%H:%M:%SZ', dt.timetuple())
qo = pi.Query_Options
del qo[:]
del pi.L3_Product_Organisation.Granule_List[:]
aux = xp.getRoot('Auxiliary_Data_Info')
del aux[:]
l3auxData = xp.getTree('L3_Auxiliary_Data_Info', 'Aux_Data')
l3auxData.clear()
qii = xp.getRoot('Quality_Indicators_Info')
del qii[:]
l3icqi = xp.getTree('L3_Quality_Indicators_Info', 'Image_Content_QI')
del l3icqi[:]
l3qii = xp.getRoot('L3_Quality_Indicators_Info')
tree = objectify.Element('L3_Classification_QI')
tree.attrib['resolution'] = str(self.config.resolution)
l3qii.append(tree)
xp.export()
#create datastrip ID:
L3_DS_DIR = os.path.join(L3_TARGET_DIR, DATASTRIP)
dirlist = sorted(os.listdir(L3_DS_DIR))
found = False
for dirname in dirlist:
if (fnmatch.fnmatch(dirname, S2A_mask) == True):
found = True
break
if found == False:
stderrWrite('No subdirectory in datastrip')
self.config.exitError()
L2A_DS_ID = dirname
L3_DS_ID = L2A_DS_ID[:4] + 'USER' + L2A_DS_ID[8:]
L3_DS_ID = L3_DS_ID.replace('L2A_', 'L03_')
self.config.L3_DS_ID = L3_DS_ID
olddir = os.path.join(L3_DS_DIR, L2A_DS_ID)
newdir = os.path.join(L3_DS_DIR, L3_DS_ID)
os.rename(olddir, newdir)
#find datastrip metadada, rename and change it:
L3_DS_DIR = newdir
filelist = sorted(os.listdir(L3_DS_DIR))
found = False
for filename in filelist:
if (fnmatch.fnmatch(filename, S2A_mask) == True):
found = True
break
if found == False:
stderrWrite('No metadata in datastrip')
self.config.exitError()
LXX_DS_MTD_XML = filename
L3_DS_MTD_XML = LXX_DS_MTD_XML[:4] + 'USER' + LXX_DS_MTD_XML[8:]
L3_DS_MTD_XML = L3_DS_MTD_XML.replace('L2A_', 'L03_')
oldfile = os.path.join(L3_DS_DIR, LXX_DS_MTD_XML)
newfile = os.path.join(L3_DS_DIR, L3_DS_MTD_XML)
self.config.L3_DS_MTD_XML = newfile
os.rename(oldfile, newfile)
xp = L3_XmlParser(self.config, 'DS03')
if (xp.convert() == False):
stderrWrite('Error in converting datastrip metadata to level 3.')
self.logger.fatal(
'error in converting datastrip metadata to level 3.')
self.config.exitError()
xp = L3_XmlParser(self.config, 'DS03')
ti = xp.getTree('Image_Data_Info', 'Tiles_Information')
del ti.Tile_List.Tile[:]
xp.export()
self.createTable()
return True
