def timestamp(self, procedure):
''' Marks a time stamp for the duration of the previous procedure.
:param procedure: the previous procedure.
:type procedure: str
'''
tNow = datetime.now()
tDelta = tNow - self._timestamp
self._timestamp = tNow
self.logger.info('Procedure: ' + procedure + ', elapsed time[s]: %0.3f' % tDelta.total_seconds())
if(self.logger.getEffectiveLevel() != logging.NOTSET):
stdoutWrite('Procedure %s, elapsed time[s]: %0.3f\n' % (procedure, tDelta.total_seconds()))
#else:
increment = tDelta.total_seconds() / self._tEstimation
self._tTotal += increment
tTotalPercentage = float32(self._tTotal * 100.0)
stdoutWrite('Progress[%%]: %03.2f : ' % tTotalPercentage)
#stdout.flush()
f = open(self._processingStatusFn, 'w')
f.write(str(tTotalPercentage) + '\n')
f.close()
return
def validate(self):
""" Validator for the metadata.
:return: true, if metadata are valid.
:rtype: boolean
"""
fn = os.path.basename(self._xmlFn)
self._config.logger.info('validating metadata file %s against scheme' %
fn)
try:
schema = etree.XMLSchema(
file=os.path.join(self._config.configDir, self._scheme))
parser = etree.XMLParser(schema=schema)
objectify.parse(self._xmlFn, parser)
self._config.logger.info('metadata file is valid')
ret = True
except etree.XMLSyntaxError, err:
stdoutWrite(
'Metadata file is invalid, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
#!/usr/bin/env python
def main(args=None):
''' Processes command line,
initializes the config and product modules and starts sequentially
the L3 processing
'''
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the Level-2A input files are located')
parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed')
parser.add_argument('--clean', action='store_true', help='Removes the L3 product in the target directory before processing. Be careful!')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if not os.path.isabs(directory):
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
directory = os.path.normpath(directory)
if not os.path.exists(directory):
stderrWrite('directory "%s" does not exist\n.' % directory)
return False
if not args.resolution:
resolutions = [60,20,10]
else:
resolutions = [args.resolution]
cleanDone = False # do only one clean up of target, if requested.
for resolution in resolutions:
config = L3_Config(resolution, directory)
result = False
processedFn = os.path.join(directory, 'processed')
if args.clean and not cleanDone:
stdoutWrite('Cleaning target directory ...\n')
config.cleanTarget = True
try:
os.remove(processedFn)
except:
stdoutWrite('No history file present ...\n')
cleanDone = True
config.init(processorVersion)
processor = doTheLoop(config)
if processor == -1:
stderrWrite('Application terminated with errors, see log file and traces.\n')
return 1
elif processor == None:
stdoutWrite('All tiles already processed.\n')
else:
processor.postProcessing()
stdoutWrite('Application terminated successfully.\n')
return 0
def doTheLoop(config):
''' Initializes a product and processor object. Cycles through all input products and granules
and calls the sequential processing of the individual tiles if the criteria for processing are fulfilled.
:param config: the config object
:type config: a reference to the config object
:return: the processor object, for doing the preprocessing, -1 if processing error occurred.
:rtype: processor object or -1
'''
HelloWorld = processorName + ', ' + processorVersion + ', created: ' + processorDate
stdoutWrite('\n%s started with %dm resolution ...\n' % (HelloWorld, config.resolution))
dirlist = os.listdir(config.sourceDir)
upList = sortObservationStartTime(dirlist)
tileFilter = config.tileFilter
# Process all unprocessed L2A products:
UP_mask = '*_MSIL2A_*'
product = L3_Product(config)
processor = L3_Process(config)
proc = None
for L2A_UP_ID in upList:
if not config.checkTimeRange(L2A_UP_ID):
continue
product.updateUserProduct(L2A_UP_ID)
if config.namingConvention == 'SAFE_STANDARD':
Tile_mask = '*L2A_*'
else:
Tile_mask = 'L2A_*'
GRANULE = os.path.join(config.sourceDir, L2A_UP_ID, 'GRANULE')
tilelist = sorted(os.listdir(GRANULE))
for tile in tilelist:
# process only L2A tiles:
if not fnmatch.fnmatch(tile, Tile_mask):
continue
# ignore already processed tiles:
if config.tileExists(tile):
continue
# apply tile filter:
if not config.tileIsSelected(tile, tileFilter):
continue
if not config.checkTileConsistency(GRANULE, tile):
continue
tStart = time()
product.config.L2A_TILE_ID = tile
tables = L3_Tables(product)
tables.init()
# no processing if first initialisation:
# check existence of Bands - B2 is always present:
if not tables.testBand('L2A', tables.B02):
# append processed tile to list
if not config.appendTile():
config.exitError()
continue
proc, result = processor.process(tables)
if result == -1:
stderrWrite('Application terminated with errors, see log file and traces.\n')
return result
elif result == 1:
return proc
elif result == 0:
tMeasure = time() - tStart
config.writeTimeEstimation(config.resolution, tMeasure)
return proc
def convert(self, targetDir):
#basename = os.path.basename(targetDir)
L1C_mask = 'S2?_*L1C_*'
L2A_mask = 'S2?_*L2A_*'
XML_mask = 'S2?_*.xml'
kwargs = {"tilesize": (2048, 2048), "prog": "RPCL"}
tiledir = targetDir + '/GRANULE'
os.chdir(tiledir)
tiles = sorted(os.listdir(tiledir))
nrTiles = len(tiles)
for tilename in tiles:
# needed for self.config.d2:
if(('S2' in tilename) == False):
nrTiles -=1
continue
stdoutWrite('%d tile(s) to be converted ...\n' % nrTiles)
self.config.logger.info('Converting reflectance to radiance')
stdoutWrite('Converting reflectance to radiance ...\n')
# get metadata filename:
globlist = tiledir + '/' + tilename + '/' + XML_mask
for fn in glob.glob(globlist):
filename = fn
if(fnmatch.fnmatch(tilename, L1C_mask) == True):
globlist = tiledir + '/' + tilename + '/IMG_DATA/S2*_B??.jp2'
elif(fnmatch.fnmatch(tilename, L2A_mask) == True):
globlist = tiledir + '/' + tilename + '/IMG_DATA/R??m/S2*_B??_*.jp2'
nrBands = len(glob.glob(globlist))
stdoutWrite('%d bands to be converted ...\n' % nrBands)
for band in glob.glob(globlist):
nrBands -= 1
basename = os.path.basename(band)
stdoutWrite('Import image %s ...\n' % basename)
indataset = glymur.Jp2k(band)
indataArr = indataset[:]
self.config.logger.info('Image: %s', basename)
self.config.logger.debug('Reflectance values:')
self.config.logger.debug('Min: %s', str(indataArr.min()))
self.config.logger.debug('Mean: %s', str(indataArr.mean()))
self.config.logger.debug('Max: %s', str(indataArr.max()))
self.setResolution(basename)
self.setBandIndex(basename)
self.getTileMetadata(filename)
outdataArr = self.refl2rad(indataArr)
stdoutWrite('converted.\n')
self.config.logger.debug('Radiance values - upscaled with 100:')
self.config.logger.debug('Min: %s', str(outdataArr.min()))
self.config.logger.debug('Mean: %s', str(outdataArr.mean()))
self.config.logger.debug('Max: %s', str(outdataArr.max()))
stdoutWrite('Export image ...\n')
glymur.Jp2k(band, outdataArr, **kwargs)
self.config.logger.info('Band ' + basename + ' converted')
stdoutWrite('%d bands remain.\n' % nrBands)
return
def main(args=None):
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the reflectance input files are located')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if(os.path.isabs(directory)) == False:
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
elif os.path.exists(args.directory) == False:
stderrWrite('directory "%s" does not exist\n.' % args.directory)
return False
config = L3_Config('60', directory, 'R2R_GIPP')
config.init(processorVersion)
result = False
HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate
stdoutWrite('\n%s started ...\n' % HelloWorld)
processor = R2R_Converter(config)
L1C_mask = 'S2?_*L1C_*'
exclusionMask = 'S2?_*_RAD'
upList = sorted(os.listdir(directory))
l1cList = []
nrUserProducts = 0
for upId in upList:
if(fnmatch.fnmatch(upId, exclusionMask) == True):
continue
elif(fnmatch.fnmatch(upId, L1C_mask) == True):
nrUserProducts += 1
l1cList.append(upId)
for upId in l1cList:
stdoutWrite('%d user product(s) to be converted ...\n' % nrUserProducts)
nrUserProducts -=1
upFullPath = os.path.join(directory, upId)
targetDir = config.targetDir
if targetDir == 'DEFAULT':
targetDir = directory + '_RAD'
else:
targetDir = os.path.join(targetDir, upId)
copy_tree(upFullPath, targetDir)
processor.setUpMetadataId(targetDir)
result = processor.convert(targetDir)
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
else:
config.logger.info('User product ' + upId + ' converted')
stdoutWrite('%d user product(s) remain\n\n' % nrUserProducts)
stdoutWrite('Application terminated successfully.\n')
return result
def main(args=None):
''' Processes command line,
initializes the config and product modules and starts sequentially
the L2A and L3 processing.
'''
import argparse
descr = processorName +', '+ processorVersion +', created: '+ processorDate + \
', supporting Level-1C product version: ' + productVersion + '.'
parser = argparse.ArgumentParser(description=descr)
parser.add_argument('directory', help='Directory where the Level-2A input files are located')
parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, must be 10, 20 or 60 [m]')
parser.add_argument('--clean', action='store_true', help='Removes all processed files in target directory. Be careful!')
args = parser.parse_args()
# SIITBX-49: directory should not end with '/':
directory = args.directory
if directory[-1] == '/':
directory = directory[:-1]
# check if directory argument starts with a relative path. If not, expand:
if(os.path.isabs(directory)) == False:
cwd = os.getcwd()
directory = os.path.join(cwd, directory)
elif os.path.exists(args.directory) == False:
stderrWrite('directory "%s" does not exist\n.' % args.directory)
return False
if args.resolution == None:
resolution = 60
else:
resolution = args.resolution
config = L3_Config(resolution, directory)
config.init(processorVersion)
processedTiles = ''
result = False
processedFn = directory + '/' + 'processed'
if args.clean:
stdoutWrite('Cleaning target directory ...\n')
try:
shutil.rmtree(config.targetDir)
except:
pass
try:
os.remove(processedFn)
except:
stdoutWrite('No history file present ...\n')
HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate
stdoutWrite('\n%s started ...\n' % HelloWorld)
upList = sorted(os.listdir(directory))
# Check if unprocessed L1C products exist. If yes, process first:
L1C_mask = 'S2?_*L1C_*'
product = L3_Product(config)
processor = L2A_Process(config)
for L1C_UP_ID in upList:
if(fnmatch.fnmatch(L1C_UP_ID, L1C_mask) == False):
continue
if config.checkTimeRange(L1C_UP_ID) == False:
continue
tilelist = product.createL2A_UserProduct(L1C_UP_ID)
for tile in tilelist:
# process only L1C tiles:
if fnmatch.fnmatch(tile, L1C_mask) == False:
continue
# ignore already processed tiles:
if product.tileExists(tile) == True:
continue
# finally, process the remaining tiles:
stdoutWrite('\nL1C tile %s found, will be classified first ...\n' % tile)
tStart = time()
tables = L2A_Tables(config, tile)
if processor.process(tables) == False:
config.exitError()
return False
tile += '_' + str(config.resolution)
if product.appendTile(tile) == False:
config.exitError()
return False
# Now process all unprocessed L2A products:
L2A_mask = 'S2?_*L2A_*'
processor = L3_Process(config)
for L2A_UP_ID in upList:
if(fnmatch.fnmatch(L2A_UP_ID, L2A_mask) == False):
continue
if config.checkTimeRange(L2A_UP_ID) == False:
continue
config.updateUserProduct(L2A_UP_ID)
GRANULE = directory + '/' + L2A_UP_ID + '/GRANULE'
tilelist = sorted(os.listdir(GRANULE))
for tile in tilelist:
# process only L2A tiles:
if fnmatch.fnmatch(tile, L2A_mask) == False:
continue
# ignore already processed tiles:
if product.tileExists(tile):
continue
tStart = time()
nrTilesProcessed = product.getNrTilesProcessed(tile)
config.updateTile(tile, nrTilesProcessed)
tables = L3_Tables(config)
tables.init()
# no processing if first initialisation:
# check existence of Bands - B2 is always present:
if tables.testBand('L2A', 2) == False:
# append processed tile to list
tile += '_' + str(config.resolution)
if product.appendTile(tile) == False:
config.exitError()
continue
result = processor.process(tables)
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
tMeasure = time() - tStart
config.writeTimeEstimation(resolution, tMeasure)
if result == True:
result = processor.postprocess()
if(result == False):
stderrWrite('Application terminated with errors, see log file and traces.\n')
return False
stdoutWrite('\nApplication terminated successfully.\n')
return result
class L3_XmlParser():
''' A parser for the assignment of xml based metadata to python configuration objects and vice versa.
Performs also a validation of the metadata against the corresponding scheme.
:param productStr: the product string for the given metadata (via __init__).
:type productStr: a string.
'''
def __init__(self, config, productStr):
self._config = config
self._productStr = productStr
self._xmlFn = None
self._xmlName = None
self._root = None
self._tree = None
self._scheme = None
if (productStr == 'GIPP'):
self._xmlFn = config.configFn
self._scheme = config.gippScheme3
elif (productStr == 'UP2A'):
self._xmlFn = config.L2A_UP_MTD_XML
self._scheme = config.upScheme2a
elif (productStr == 'UP03'):
self._xmlFn = config.L3_TARGET_MTD_XML
self._scheme = config.upScheme3
elif (productStr == 'DS2A'):
self._xmlFn = config.L2A_DS_MTD_XML
self._scheme = config.dsScheme2a
elif (productStr == 'DS03'):
self._xmlFn = config.L3_DS_MTD_XML
self._scheme = config.dsScheme3
elif (productStr == 'T2A'):
self._xmlFn = config.L2A_TILE_MTD_XML
self._scheme = config.tileScheme2a
elif (productStr == 'T03'):
self._xmlFn = config.L3_TILE_MTD_XML
self._scheme = config.tileScheme3
elif (productStr == 'Manifest'):
self._xmlFn = config.L3_MANIFEST_SAFE
self._scheme = config.manifestScheme
else:
config.logger.fatal(
'wrong product identifier for xml structure: ' + productStr)
config.exitError()
self.setRoot()
return
def getRoot(self, key=None):
''' Gets the root of an xml tree, addressed by the corresponding key.
:param key: the search key
:type key: a string
:return: the tree
:rtype: an element tree
'''
try:
if key == None:
return self._root
else:
root = self._root[key]
return root
except:
return False
def setRoot(self):
''' Sets the root of an xml tree.
:return: true if succesful
:rtype: bool
'''
if self._root is not None:
return True
try:
doc = objectify.parse(self._xmlFn)
self._root = doc.getroot()
return True
except:
return False
def getTree(self, key, subkey):
''' Gets the subtree of an xml tree, addressed by the corresponding key and subkey.
:param key: the search key
:type key: a string
:param subkey: the search subkey
:type subkey: a string
:return: the tree
:rtype: an element tree
'''
try:
tree = self._root[key]
return tree['{}' + subkey]
except:
return False
def setTree(self, key, subkey):
''' Sets the subtree of an xml tree, addressed by the corresponding key and subkey.
:param key: the search key
:type key: a string
:param subkey: the search subkey
:type subkey: a string
:return: true if succesful
:rtype: bool
'''
try:
root = self._root[key]
except:
return False
try:
self._tree = root['{}' + subkey]
return True
except:
self._tree = root
if (self.append(subkey, '') == True):
try:
self._tree = root['{}' + subkey]
self.export()
return True
except:
return False
return False
def validate(self):
""" Validator for the metadata.
:return: true, if metadata are valid.
:rtype: boolean
"""
fn = os.path.basename(self._xmlFn)
self._config.logger.info('validating metadata file %s against scheme' %
fn)
err = 'unknown'
try:
schema = etree.XMLSchema(
file=os.path.join(self._config.configDir, self._scheme))
parser = etree.XMLParser(schema=schema)
objectify.parse(self._xmlFn, parser)
self._config.logger.info('metadata file is valid')
ret = True
except etree.XMLSyntaxError, err:
stdoutWrite(
'Syntax error in metadata, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except etree.XMLSchemaError, err:
stdoutWrite('Error in xml schema, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
objectify.parse(self._xmlFn, parser)
self._config.logger.info('metadata file is valid')
ret = True
except etree.XMLSyntaxError, err:
stdoutWrite(
'Syntax error in metadata, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except etree.XMLSchemaError, err:
stdoutWrite('Error in xml schema, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except etree.XMLSchemaParseError, err:
stdoutWrite(
'Error in parsing xml schema, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except etree.XMLSchemaValidateError, err:
stdoutWrite(
'Error in validating scheme, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except:
stdoutWrite('Unspecific Error in metadata.\n')
self._config.logger.error('unspecific error in metadata')
ret = False
if ret == False:
fn)
try:
schema = etree.XMLSchema(
file=os.path.join(self._config.configDir, self._scheme))
parser = etree.XMLParser(schema=schema)
objectify.parse(self._xmlFn, parser)
self._config.logger.info('metadata file is valid')
ret = True
except etree.XMLSyntaxError, err:
stdoutWrite(
'Metadata file is invalid, see report file for details.\n')
self._config.logger.error('Schema file: %s' % self._scheme)
self._config.logger.error('Details: %s' % str(err))
ret = False
except:
stdoutWrite('Unspecific Error in metadata.\n')
self._config.logger.error('unspecific error in metadata')
ret = False
return ret
def append(self, key, value):
try:
e = etree.Element(key)
e.text = value
self._tree.append(e)
return True
except:
return False
def export(self):