def parNotFound(self, parameter): ''' Throw a fatal error message if a configuration parameter is not found and terminate the application. ''' self.logger.fatal('Configuration parameter %s not found in %s', parameter, self._configFn) stderrWrite('Configuration parameter <%s> not found in %s\n' % (parameter, self._configFn)) stderrWrite('Program is forced to terminate.') self.__exit__()
def process(self, tables): ''' Perform the L3 processing. :param tables: the table object for the current tile :type tables: a reference to the table object :return: true if processing succeeds, false else :rtype: bool ''' self._tables = tables astr = 'L3_Process: processing with resolution ' + str(self.config.resolution) + ' m' self.config.timestamp(astr) self.config.timestamp('L3_Process: start of Pre Processing') if(self.preprocess() == False): return False self.config.timestamp('L3_Process: start of Spatio Temporal Processing') self.config.logger.info('Performing Spatio Temporal Processing with resolution %d m', self.config.resolution) if(self._l3Synthesis.process(self._tables) == False): return False # append processed tile to list processedTile = self.config.product.L2A_TILE_ID + '_' + str(self.config.resolution) + '\n' processedFn = self.config.sourceDir + '/' + 'processed' try: f = open(processedFn, 'a') f.write(processedTile) f.flush() f.close() except: stderrWrite('Could not update processed tile history.\n') self.config.exitError() return False return True
def updateUserProduct(self, userProduct): ''' Update the current user product. Check if at target product is present. :param userProduct: the user product identifier. :type userProduct: str :return: true, if target product exists. :rtype: boolean ''' self.config.L2A_UP_ID = userProduct self.config.L2A_UP_DIR = os.path.join(self.config.sourceDir, userProduct) if self.config.namingConvention == 'SAFE_STANDARD': filelist = sorted(os.listdir(self.config.L2A_UP_DIR)) filemask = 'S2A_USER_MTD_*.xml' for file in filelist: if fnmatch.fnmatch(file, filemask): self.config.L2A_UP_MTD_XML = os.path.join( self.config.L2A_UP_DIR, file) break else: self.config.L2A_UP_MTD_XML = os.path.join(self.config.L2A_UP_DIR, 'MTD_MSIL2A.xml') if self.existL3_TargetProduct(userProduct) == False: stderrWrite('directory "%s" L3 target product is missing\n.' % self.config.targetDir) self.config.exitError() return True
def reinitL3_TargetProduct(self): ''' Reinit the L3 target product :return: true if succesful :rtype: bool ''' L3_DS_ID = None L3_TARGET_MASK = '*L03_*' dirlist = sorted(os.listdir(self.config.targetDir)) for L3_TARGET_ID in dirlist: if fnmatch.fnmatch(L3_TARGET_ID, L3_TARGET_MASK) == True: self.config.L3_TARGET_ID = L3_TARGET_ID self.config.L3_TARGET_DIR = os.path.join( self.config.targetDir, L3_TARGET_ID) break # create user product: filelist = sorted(os.listdir(self.config.L3_TARGET_DIR)) found = False for filename in filelist: if (fnmatch.fnmatch(filename, 'MTD_MSIL03.xml') == True): found = True break if found == False: stderrWrite('No metadata for user product') self.config.exitError() self.config.L3_TARGET_MTD_XML = os.path.join(self.config.L3_TARGET_DIR, filename) xp = L3_XmlParser(self.config, 'UP03') l3qii = xp.getRoot('Quality_Indicators_Info') node = objectify.Element('Classification_QI') node.attrib['resolution'] = str(self.config.resolution) if self.insert(l3qii, node): xp.export() L3_DS_MASK = 'DS_*' DATASTRIP = 'DATASTRIP' L3_DS_DIR = os.path.join(self.config.targetDir, L3_TARGET_ID, DATASTRIP) dirlist = sorted(os.listdir(L3_DS_DIR)) for L3_DS_ID in dirlist: if fnmatch.fnmatch(L3_DS_ID, L3_DS_MASK) == True: self.config.L3_DS_ID = L3_DS_ID break if L3_DS_ID is not None: L3_DS_MTD_XML = 'MTD_DS.xml' self.config.L3_DS_MTD_XML = os.path.join(L3_DS_DIR, L3_DS_ID, L3_DS_MTD_XML) return True return False
def setUpMetadataId(self, L1C_UP_DIR): S2A_mask = 'S2A_*_MTD_*1C_*.xml' filelist = sorted(os.listdir(L1C_UP_DIR)) found = False for filename in filelist: if(fnmatch.fnmatch(filename, S2A_mask) == True): found = True break if found == False: stderrWrite('No metadata for user product.\n') self.config.exitError() self.config.product.L1C_UP_MTD_XML = os.path.join(L1C_UP_DIR, filename) return
def updateUserProduct(self, userProduct): ''' Update the current user product. Check if at target product is present. :param userProduct: the user product identifier. :type userProduct: str :return: true, if target product exists. :rtype: boolean ''' self.product.L2A_UP_ID = userProduct if self.product.existL3_TargetProduct() == False: stderrWrite('directory "%s" L3 target product is missing\n.' % self.targetDir) self.exitError() return True
def updateUserProduct(self, userProduct): ''' Update the current user product. Check if at target product is present. :param userProduct: the user product identifier. :type userProduct: str :return: true, if target product exists. :rtype: boolean ''' self.config.L2A_UP_ID = userProduct if self.existL3_TargetProduct() == False: stderrWrite('directory "%s" L3 target product is missing\n.' % self.config.targetDir) self.config.exitError() return True
def exitError(self, reason = None): ''' Throw an error message if a fatal error occurred and and terminate the application. ''' stderrWrite('Fatal error occurred, see report file for details.') if reason: stderrWrite('Reason: ' + reason) self.__exit__()
def main(args=None): ''' Processes command line, initializes the config and product modules and starts sequentially the L3 processing ''' import argparse descr = processorName +', '+ processorVersion +', created: '+ processorDate + \ ', supporting Level-1C product version: ' + productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument('directory', help='Directory where the Level-2A input files are located') parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed') parser.add_argument('--clean', action='store_true', help='Removes the L3 product in the target directory before processing. Be careful!') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if not os.path.isabs(directory): cwd = os.getcwd() directory = os.path.join(cwd, directory) directory = os.path.normpath(directory) if not os.path.exists(directory): stderrWrite('directory "%s" does not exist\n.' % directory) return False if not args.resolution: resolutions = [60,20,10] else: resolutions = [args.resolution] cleanDone = False # do only one clean up of target, if requested. for resolution in resolutions: config = L3_Config(resolution, directory) result = False processedFn = os.path.join(directory, 'processed') if args.clean and not cleanDone: stdoutWrite('Cleaning target directory ...\n') config.cleanTarget = True try: os.remove(processedFn) except: stdoutWrite('No history file present ...\n') cleanDone = True config.init(processorVersion) processor = doTheLoop(config) if processor == -1: stderrWrite('Application terminated with errors, see log file and traces.\n') return 1 elif processor == None: stdoutWrite('All tiles already processed.\n') else: processor.postProcessing() stdoutWrite('Application terminated successfully.\n') return 0
def doTheLoop(config): ''' Initializes a product and processor object. Cycles through all input products and granules and calls the sequential processing of the individual tiles if the criteria for processing are fulfilled. :param config: the config object :type config: a reference to the config object :return: the processor object, for doing the preprocessing, -1 if processing error occurred. :rtype: processor object or -1 ''' HelloWorld = processorName + ', ' + processorVersion + ', created: ' + processorDate stdoutWrite('\n%s started with %dm resolution ...\n' % (HelloWorld, config.resolution)) dirlist = os.listdir(config.sourceDir) upList = sortObservationStartTime(dirlist) tileFilter = config.tileFilter # Process all unprocessed L2A products: UP_mask = '*_MSIL2A_*' product = L3_Product(config) processor = L3_Process(config) proc = None for L2A_UP_ID in upList: if not config.checkTimeRange(L2A_UP_ID): continue product.updateUserProduct(L2A_UP_ID) if config.namingConvention == 'SAFE_STANDARD': Tile_mask = '*L2A_*' else: Tile_mask = 'L2A_*' GRANULE = os.path.join(config.sourceDir, L2A_UP_ID, 'GRANULE') tilelist = sorted(os.listdir(GRANULE)) for tile in tilelist: # process only L2A tiles: if not fnmatch.fnmatch(tile, Tile_mask): continue # ignore already processed tiles: if config.tileExists(tile): continue # apply tile filter: if not config.tileIsSelected(tile, tileFilter): continue if not config.checkTileConsistency(GRANULE, tile): continue tStart = time() product.config.L2A_TILE_ID = tile tables = L3_Tables(product) tables.init() # no processing if first initialisation: # check existence of Bands - B2 is always present: if not tables.testBand('L2A', tables.B02): # append processed tile to list if not config.appendTile(): config.exitError() continue proc, result = processor.process(tables) if result == -1: stderrWrite('Application terminated with errors, see log file and traces.\n') return result elif result == 1: return proc elif result == 0: tMeasure = time() - tStart config.writeTimeEstimation(config.resolution, tMeasure) return proc
def main(args=None): import argparse descr = processorName +', '+ processorVersion +', created: '+ processorDate + \ ', supporting Level-1C product version: ' + productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument('directory', help='Directory where the reflectance input files are located') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if(os.path.isabs(directory)) == False: cwd = os.getcwd() directory = os.path.join(cwd, directory) elif os.path.exists(args.directory) == False: stderrWrite('directory "%s" does not exist\n.' % args.directory) return False config = L3_Config('60', directory, 'R2R_GIPP') config.init(processorVersion) result = False HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate stdoutWrite('\n%s started ...\n' % HelloWorld) processor = R2R_Converter(config) L1C_mask = 'S2?_*L1C_*' exclusionMask = 'S2?_*_RAD' upList = sorted(os.listdir(directory)) l1cList = [] nrUserProducts = 0 for upId in upList: if(fnmatch.fnmatch(upId, exclusionMask) == True): continue elif(fnmatch.fnmatch(upId, L1C_mask) == True): nrUserProducts += 1 l1cList.append(upId) for upId in l1cList: stdoutWrite('%d user product(s) to be converted ...\n' % nrUserProducts) nrUserProducts -=1 upFullPath = os.path.join(directory, upId) targetDir = config.targetDir if targetDir == 'DEFAULT': targetDir = directory + '_RAD' else: targetDir = os.path.join(targetDir, upId) copy_tree(upFullPath, targetDir) processor.setUpMetadataId(targetDir) result = processor.convert(targetDir) if(result == False): stderrWrite('Application terminated with errors, see log file and traces.\n') return False else: config.logger.info('User product ' + upId + ' converted') stdoutWrite('%d user product(s) remain\n\n' % nrUserProducts) stdoutWrite('Application terminated successfully.\n') return result
def createL3_TargetProduct(self): ''' Create the L3 target product. :return: true if succesful :rtype: bool ''' self.config.logger.info('Creating L3 target product ...') L2A_UP_MASK = '*L2A_*' #L2A_UP_DIR = os.path.join(self.config.sourceDir, self.config.L2A_UP_ID) L2A_UP_DIR = self.config.L2A_UP_DIR # detect the filename for the datastrip metadata: L2A_DS_DIR = os.path.join(L2A_UP_DIR, 'DATASTRIP') if os.path.exists(L2A_DS_DIR) == False: stderrWrite('directory "%s" does not exist.\n' % L2A_DS_DIR) self.config.exitError() return False if self.config.namingConvention == 'SAFE_STANDARD': L2A_DS_MASK = 'S2?_USER_MSI_L2A_DS_*' else: L2A_DS_MASK = 'DS_*' dirlist = sorted(os.listdir(L2A_DS_DIR)) found = False for dirname in dirlist: if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True): found = True break L2A_DS_DIR = os.path.join(L2A_DS_DIR, dirname) if self.config.namingConvention == 'SAFE_STANDARD': L2A_DS_MTD_XML = (dirname[:-7] + '.xml').replace('_MSI_', '_MTD_') else: L2A_DS_MTD_XML = 'MTD_DS.xml' L2A_DS_MTD_XML = os.path.join(L2A_DS_DIR, L2A_DS_MTD_XML) if found == False: stderrWrite('No metadata in datastrip\n.') self.config.exitError() dirname, basename = os.path.split(L2A_UP_DIR) if (fnmatch.fnmatch(basename, L2A_UP_MASK) == False): stderrWrite(basename + ': identifier "%s" is missing.' % L2A_UP_MASK) self.config.exitError() return False GRANULE = os.path.join(L2A_UP_DIR, 'GRANULE') if os.path.exists(GRANULE) == False: stderrWrite('directory "' + GRANULE + '" does not exist.') self.config.exitError() return False # # the product (directory) structure: #------------------------------------------------------- L3_TARGET_ID = basename L3_TARGET_ID = L3_TARGET_ID.replace('L2A_', 'L03_') strList = L3_TARGET_ID.split('_') if self.config.namingConvention == 'SAFE_STANDARD': L3_TARGET_ID = strList[0] + '_' + strList[3] + '_' + strList[ 5] + '_' + 'N0000_R000_T00XXX_' + self.config.minTime + '.SAFE' else: L3_TARGET_ID = L3_TARGET_ID.replace(strList[6], self.config.minTime + '.SAFE') if self.config.targetDir != 'DEFAULT': targetDir = self.config.targetDir if os.name == 'nt' and not '\\\\?\\' in targetDir: targetDir = u'\\'.join([u'\\\\?', targetDir]) if (os.path.exists(targetDir) == False): os.mkdir(targetDir) else: targetDir = dirname self.config.targetDir = targetDir L3_TARGET_DIR = os.path.join(targetDir, L3_TARGET_ID) self.config.L3_TARGET_DIR = L3_TARGET_DIR self.config.L3_TARGET_ID = L3_TARGET_ID L2A_INSPIRE_XML = os.path.join(L2A_UP_DIR, 'INSPIRE.xml') L2A_MANIFEST_SAFE = os.path.join(L2A_UP_DIR, 'manifest.safe') L3_INSPIRE_XML = os.path.join(L3_TARGET_DIR, 'INSPIRE.xml') L3_MANIFEST_SAFE = os.path.join(L3_TARGET_DIR, 'manifest.safe') AUX_DATA = 'AUX_DATA' DATASTRIP = 'DATASTRIP' GRANULE = 'GRANULE' HTML = 'HTML' REP_INFO = 'rep_info' self.config.L2A_DS_MTD_XML = L2A_DS_MTD_XML xp = L3_XmlParser(self.config, 'DS2A') xp.export() xp.validate() copy_tree(os.path.join(L2A_UP_DIR, AUX_DATA), os.path.join(L3_TARGET_DIR, AUX_DATA)) copy_tree(os.path.join(L2A_UP_DIR, DATASTRIP), os.path.join(L3_TARGET_DIR, DATASTRIP)) copy_tree(os.path.join(L2A_UP_DIR, HTML), os.path.join(L3_TARGET_DIR, HTML)) copy_tree(os.path.join(L2A_UP_DIR, REP_INFO), os.path.join(L3_TARGET_DIR, REP_INFO)) copy_file(L2A_INSPIRE_XML, L3_INSPIRE_XML) copy_file(L2A_MANIFEST_SAFE, L3_MANIFEST_SAFE) if not os.path.exists(os.path.join(L3_TARGET_DIR, GRANULE)): os.mkdir(os.path.join(L3_TARGET_DIR, GRANULE)) self.config.L3_INSPIRE_XML = L2A_INSPIRE_XML self.config.L3_MANIFEST_SAFE = L2A_MANIFEST_SAFE self.config.L3_TARGET_DIR = L3_TARGET_DIR #create user product: if self.config.namingConvention == 'SAFE_STANDARD': S2A_mask = 'S2?_USER_MTD_SAFL2A_*.xml' else: S2A_mask = 'MTD_MSIL2A.xml' filelist = sorted(os.listdir(L2A_UP_DIR)) found = False for filename in filelist: if (fnmatch.fnmatch(filename, S2A_mask) == True): found = True break if found == False: stderrWrite('No metadata for user product') self.config.exitError() # prepare L3 User Product metadata file fn_L2A = os.path.join(L2A_UP_DIR, filename) fn_L3 = 'MTD_MSIL03.xml' fn_L3 = os.path.join(L3_TARGET_DIR, fn_L3) self.config.L2A_UP_MTD_XML = fn_L2A xp = L3_XmlParser(self.config, 'UP2A') if xp.convert() != 0: xp = L3_XmlParser(self.config, 'UP2A') xp.validate() self.config.L3_TARGET_MTD_XML = fn_L3 # copy L2A schemes from config_dir into rep_info: upScheme3 = self.config.upScheme3 basename = os.path.basename(upScheme3) copy_file(os.path.join(self.config.configDir, upScheme3), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) tileScheme3 = self.config.tileScheme3 basename = os.path.basename(tileScheme3) copy_file(os.path.join(self.config.configDir, tileScheme3), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) dsScheme3 = self.config.dsScheme3 basename = os.path.basename(dsScheme3) copy_file(os.path.join(self.config.configDir, dsScheme3), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) # copy L3 User Product metadata file: copy_file(fn_L2A, fn_L3) # remove old L2A entries from L3_TARGET_MTD_XML: xp = L3_XmlParser(self.config, 'UP03') if (xp.convert() == False): self.logger.fatal( 'error in converting user product metadata to level 3') stderrWrite('Error in converting user product metadata to level 3') self.config.exitError() xp = L3_XmlParser(self.config, 'UP03') pi = xp.getTree('General_Info', 'Product_Info') # update L2A entries from L2A_UP_MTD_XML: tmin = self.config.minTime tmax = self.config.maxTime pi.PRODUCT_START_TIME = tmin[:4] + '-' + tmin[4:6] + '-' + tmin[ 6:11] + ':' + tmin[11:13] + ':' + tmin[13:15] + '.000Z' pi.PRODUCT_STOP_TIME = tmax[:4] + '-' + tmax[4:6] + '-' + tmax[ 6:11] + '-' + tmax[11:13] + ':' + tmax[13:15] + '.000Z' # update L2A and L3 entries from L2A_UP_MTD_XML: try: del pi.PRODUCT_URI_1C except: pass pi.PRODUCT_URI = self.config.L3_TARGET_ID pi.PROCESSING_LEVEL = 'Level-3p' pi.PRODUCT_TYPE = 'S2MSI3p' pi.PROCESSING_ALGORITHM = self.config.algorithm pi.RADIOMETRIC_PREFERENCE = self.config.radiometricPreference dt = datetime.utcnow() pi.GENERATION_TIME = strftime('%Y-%m-%dT%H:%M:%SZ', dt.timetuple()) if self.config.namingConvention == 'SAFE_STANDARD': try: qo = pi.Query_Options del qo[:] qo = objectify.Element('Query_Options') qo.attrib['completeSingleTile'] = "false" qo.append(objectify.Element('PRODUCT_FORMAT')) qo.PRODUCT_FORMAT = 'SAFE_COMPACT' pi.append(qo) except: pass try: gl = pi.Product_Organisation.Granule_List del gl[:] except: pass try: pic = xp.getTree('General_Info', 'Product_Image_Characteristics') del pic.QUANTIFICATION_VALUES_LIST.L1C_TOA_QUANTIFICATION_VALUE except: pass try: aux = xp.getRoot('L2A_Auxiliary_Data_Info') del aux[:] except: pass try: aux = xp.getRoot('Auxiliary_Data_Info') aux.clear() node = objectify.Element('GIPP_LIST') aux.append(node) except: pass try: qii = xp.getRoot('L2A_Quality_Indicators_Info') del qii[:] except: pass try: qii = xp.getRoot('Quality_Indicators_Info') qii.clear() node = objectify.Element('Classification_QI') node.attrib['resolution'] = str(self.config.resolution) qii.append(node) except: pass xp.export() #create datastrip ID: L3_DS_DIR = os.path.join(L3_TARGET_DIR, DATASTRIP) dirlist = sorted(os.listdir(L3_DS_DIR)) found = False for dirname in dirlist: if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True): found = True break if found == False: stderrWrite('No subdirectory in datastrip') self.config.exitError() if self.config.namingConvention == 'SAFE_STANDARD': L3_DS_ID = dirname[17:-7] L3_DS_OLD = os.path.join(L3_DS_DIR, dirname) L3_DS_DIR = os.path.join(L3_DS_DIR, L3_DS_ID) os.rename(L3_DS_OLD, L3_DS_DIR) else: L3_DS_ID = dirname L3_DS_DIR = os.path.join(L3_DS_DIR, L3_DS_ID) self.config.L3_DS_ID = L3_DS_ID filelist = sorted(os.listdir(L3_DS_DIR)) found = False L2A_DS_MTD_MSK = 'S2?_USER_MTD_L2A_DS_*.xml' for filename in filelist: if (fnmatch.fnmatch(filename, L2A_DS_MTD_MSK) == True): L3_DS_OLD = os.path.join(L3_DS_DIR, filename) L3_DS_MTD = os.path.join(L3_DS_DIR, 'MTD_DS.xml') os.rename(L3_DS_OLD, L3_DS_MTD) filename = 'MTD_DS.xml' found = True elif (fnmatch.fnmatch(filename, 'MTD_DS.xml') == True): found = True break if found == False: stderrWrite('No metadata in datastrip') self.config.exitError() self.config.L3_DS_MTD_XML = os.path.join(L3_DS_DIR, filename) xp = L3_XmlParser(self.config, 'DS03') if (xp.convert() == False): stderrWrite('Error in converting datastrip metadata to level 3.') self.logger.fatal( 'error in converting datastrip metadata to level 3.') self.config.exitError() xp = L3_XmlParser(self.config, 'DS03') ti = xp.getTree('Image_Data_Info', 'Tiles_Information') del ti.Tile_List.Tile[:] ri = xp.getTree('Image_Data_Info', 'Radiometric_Info') qvl = objectify.Element('QUANTIFICATION_VALUES_LIST') qvl.BOA_QUANTIFICATION_VALUE = str(int(self.config.dnScale)) qvl.BOA_QUANTIFICATION_VALUE.attrib['unit'] = 'none' qvl.AOT_QUANTIFICATION_VALUE = str( self.config.L2A_AOT_QUANTIFICATION_VALUE) qvl.AOT_QUANTIFICATION_VALUE.attrib['unit'] = 'none' qvl.WVP_QUANTIFICATION_VALUE = str( self.config.L2A_WVP_QUANTIFICATION_VALUE) qvl.WVP_QUANTIFICATION_VALUE.attrib['unit'] = 'cm' ri.QUANTIFICATION_VALUES_LIST = qvl auxinfo = xp.getRoot('Auxiliary_Data_Info') if (xp.getTree('Auxiliary_Data_Info', 'GRI_List')) == False: gfn = xp.getTree('Auxiliary_Data_Info', 'GRI_FILENAME') del gfn[:] gl = etree.Element('GRI_List') gl.append(gfn) auxinfo.append(gl) if (xp.getTree('Auxiliary_Data_Info', 'LUT_List')) == False: ll = etree.Element('LUT_List') auxinfo.append(ll) if (xp.getTree('Auxiliary_Data_Info', 'SNOW_CLIMATOLOGY_MAP')) == False: ll = etree.Element('SNOW_CLIMATOLOGY_MAP') ll.text = 'None' auxinfo.append(ll) if (xp.getTree('Auxiliary_Data_Info', 'ESACCI_WaterBodies_Map')) == False: ll = etree.Element('ESACCI_WaterBodies_Map') ll.text = 'None' auxinfo.append(ll) if (xp.getTree('Auxiliary_Data_Info', 'ESACCI_LandCover_Map')) == False: ll = etree.Element('ESACCI_LandCover_Map') ll.text = 'None' auxinfo.append(ll) if (xp.getTree('Auxiliary_Data_Info', 'ESACCI_SnowCondition_Map_Dir')) == False: ll = etree.Element('ESACCI_SnowCondition_Map_Dir') ll.text = 'None' auxinfo.append(ll) xp.export() self.createTable() return True
def main(args=None): ''' Processes command line, initializes the config and product modules and starts sequentially the L2A and L3 processing. ''' import argparse descr = processorName +', '+ processorVersion +', created: '+ processorDate + \ ', supporting Level-1C product version: ' + productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument('directory', help='Directory where the Level-2A input files are located') parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, must be 10, 20 or 60 [m]') parser.add_argument('--clean', action='store_true', help='Removes all processed files in target directory. Be careful!') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if(os.path.isabs(directory)) == False: cwd = os.getcwd() directory = os.path.join(cwd, directory) elif os.path.exists(args.directory) == False: stderrWrite('directory "%s" does not exist\n.' % args.directory) return False if args.resolution == None: resolution = 60 else: resolution = args.resolution config = L3_Config(resolution, directory) config.init(processorVersion) processedTiles = '' result = False processedFn = directory + '/' + 'processed' if args.clean: stdoutWrite('Cleaning target directory ...\n') try: shutil.rmtree(config.targetDir) except: pass try: os.remove(processedFn) except: stdoutWrite('No history file present ...\n') HelloWorld = processorName +', '+ processorVersion +', created: '+ processorDate stdoutWrite('\n%s started ...\n' % HelloWorld) upList = sorted(os.listdir(directory)) # Check if unprocessed L1C products exist. If yes, process first: L1C_mask = 'S2?_*L1C_*' product = L3_Product(config) processor = L2A_Process(config) for L1C_UP_ID in upList: if(fnmatch.fnmatch(L1C_UP_ID, L1C_mask) == False): continue if config.checkTimeRange(L1C_UP_ID) == False: continue tilelist = product.createL2A_UserProduct(L1C_UP_ID) for tile in tilelist: # process only L1C tiles: if fnmatch.fnmatch(tile, L1C_mask) == False: continue # ignore already processed tiles: if product.tileExists(tile) == True: continue # finally, process the remaining tiles: stdoutWrite('\nL1C tile %s found, will be classified first ...\n' % tile) tStart = time() tables = L2A_Tables(config, tile) if processor.process(tables) == False: config.exitError() return False tile += '_' + str(config.resolution) if product.appendTile(tile) == False: config.exitError() return False # Now process all unprocessed L2A products: L2A_mask = 'S2?_*L2A_*' processor = L3_Process(config) for L2A_UP_ID in upList: if(fnmatch.fnmatch(L2A_UP_ID, L2A_mask) == False): continue if config.checkTimeRange(L2A_UP_ID) == False: continue config.updateUserProduct(L2A_UP_ID) GRANULE = directory + '/' + L2A_UP_ID + '/GRANULE' tilelist = sorted(os.listdir(GRANULE)) for tile in tilelist: # process only L2A tiles: if fnmatch.fnmatch(tile, L2A_mask) == False: continue # ignore already processed tiles: if product.tileExists(tile): continue tStart = time() nrTilesProcessed = product.getNrTilesProcessed(tile) config.updateTile(tile, nrTilesProcessed) tables = L3_Tables(config) tables.init() # no processing if first initialisation: # check existence of Bands - B2 is always present: if tables.testBand('L2A', 2) == False: # append processed tile to list tile += '_' + str(config.resolution) if product.appendTile(tile) == False: config.exitError() continue result = processor.process(tables) if(result == False): stderrWrite('Application terminated with errors, see log file and traces.\n') return False tMeasure = time() - tStart config.writeTimeEstimation(resolution, tMeasure) if result == True: result = processor.postprocess() if(result == False): stderrWrite('Application terminated with errors, see log file and traces.\n') return False stdoutWrite('\nApplication terminated successfully.\n') return result
def createL3_TargetProduct(self): ''' Create the L3 target product. :return: true if succesful :rtype: bool ''' self.config.logger.info('Creating L3 target product ...') L2A_UP_MASK = '*2A_*' L2A_UP_DIR = os.path.join(self.config.sourceDir, self.config.L2A_UP_ID) # detect the filename for the datastrip metadata: L2A_DS_DIR = os.path.join(L2A_UP_DIR, 'DATASTRIP') if os.path.exists(L2A_DS_DIR) == False: stderrWrite('directory "%s" does not exist.\n' % L2A_DS_DIR) self.config.exitError() return False L2A_DS_MASK = '*_L2A_*' dirlist = sorted(os.listdir(L2A_DS_DIR)) found = False for dirname in dirlist: if (fnmatch.fnmatch(dirname, L2A_DS_MASK) == True): found = True break L2A_DS_DIR = os.path.join(L2A_DS_DIR, dirname) L2A_DS_MTD_XML = (dirname[:-7] + '.xml').replace('_MSI_', '_MTD_') L2A_DS_MTD_XML = os.path.join(L2A_DS_DIR, L2A_DS_MTD_XML) if found == False: stderrWrite('No metadata in datastrip\n.') self.config.exitError() dirname, basename = os.path.split(L2A_UP_DIR) if (fnmatch.fnmatch(basename, L2A_UP_MASK) == False): stderrWrite(basename + ': identifier "*2A_*" is missing.') self.config.exitError() return False GRANULE = os.path.join(L2A_UP_DIR, 'GRANULE') if os.path.exists(GRANULE) == False: stderrWrite('directory "' + GRANULE + '" does not exist.') self.config.exitError() return False # # the product (directory) structure: #------------------------------------------------------- L3_TARGET_ID = basename L3_TARGET_ID = L3_TARGET_ID.replace('L2A_', 'L03_') L3_TARGET_ID = L3_TARGET_ID.replace(L3_TARGET_ID[47:62], self.config.minTime) L3_TARGET_ID = L3_TARGET_ID.replace(L3_TARGET_ID[63:78], self.config.maxTime) if self.config.targetDir != 'DEFAULT': targetDir = self.config.targetDir if os.name == 'nt' and not '\\\\?\\' in targetDir: targetDir = u'\\'.join([u'\\\\?', targetDir]) if (os.path.exists(targetDir) == False): os.mkdir(targetDir) else: targetDir = dirname self.config.targetDir = targetDir L3_TARGET_DIR = os.path.join(targetDir, L3_TARGET_ID) self.config.L3_TARGET_DIR = L3_TARGET_DIR self.config.L3_TARGET_ID = L3_TARGET_ID L2A_INSPIRE_XML = os.path.join(L2A_UP_DIR, 'INSPIRE.xml') L2A_MANIFEST_SAFE = os.path.join(L2A_UP_DIR, 'manifest.safe') L3_INSPIRE_XML = os.path.join(L3_TARGET_DIR, 'INSPIRE.xml') L3_MANIFEST_SAFE = os.path.join(L3_TARGET_DIR, 'manifest.safe') AUX_DATA = 'AUX_DATA' DATASTRIP = 'DATASTRIP' GRANULE = 'GRANULE' HTML = 'HTML' REP_INFO = 'rep_info' self.config.L2A_DS_MTD_XML = L2A_DS_MTD_XML xp = L3_XmlParser(self.config, 'DS2A') xp.export() xp.validate() copy_tree(os.path.join(L2A_UP_DIR, AUX_DATA), os.path.join(L3_TARGET_DIR, AUX_DATA)) copy_tree(os.path.join(L2A_UP_DIR, DATASTRIP), os.path.join(L3_TARGET_DIR, DATASTRIP)) copy_tree(os.path.join(L2A_UP_DIR, HTML), os.path.join(L3_TARGET_DIR, HTML)) copy_tree(os.path.join(L2A_UP_DIR, REP_INFO), os.path.join(L3_TARGET_DIR, REP_INFO)) copy_file(L2A_INSPIRE_XML, L3_INSPIRE_XML) copy_file(L2A_MANIFEST_SAFE, L3_MANIFEST_SAFE) if not os.path.exists(os.path.join(L3_TARGET_DIR, GRANULE)): os.mkdir(os.path.join(L3_TARGET_DIR, GRANULE)) self.config.L3_INSPIRE_XML = L2A_INSPIRE_XML self.config.L3_MANIFEST_SAFE = L2A_MANIFEST_SAFE self.config.L3_TARGET_DIR = L3_TARGET_DIR #create user product: S2A_mask = 'S2A_*' filelist = sorted(os.listdir(L2A_UP_DIR)) found = False for filename in filelist: if (fnmatch.fnmatch(filename, S2A_mask) == True): found = True break if found == False: stderrWrite('No metadata for user product') self.config.exitError() # prepare L3 User Product metadata file fn_L2A = os.path.join(L2A_UP_DIR, filename) fn_L3 = filename[:4] + 'USER' + filename[8:] fn_L3 = fn_L3.replace('L2A_', 'L03_') fn_L3 = os.path.join(L3_TARGET_DIR, fn_L3) self.config.L2A_UP_MTD_XML = fn_L2A xp = L3_XmlParser(self.config, 'UP2A') xp.export() xp.validate() self.config.L3_TARGET_MTD_XML = fn_L3 # copy L2A schemes from config_dir into rep_info: xp = L3_XmlParser(self.config, 'GIPP') cs = xp.getRoot('Common_Section') upScheme2a = cs.UP_Scheme_2A.text basename = os.path.basename(upScheme2a) copy_file(os.path.join(self.config.configDir, upScheme2a), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) tileScheme2a = cs.Tile_Scheme_2A.text basename = os.path.basename(tileScheme2a) copy_file(os.path.join(self.config.configDir, tileScheme2a), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) dsScheme2a = cs.DS_Scheme_2A.text basename = os.path.basename(dsScheme2a) copy_file(os.path.join(self.config.configDir, dsScheme2a), os.path.join(L3_TARGET_DIR, REP_INFO, basename)) # copy L3 User Product metadata file: copy_file(fn_L2A, fn_L3) # remove old L2A entries from L3_TARGET_MTD_XML: xp = L3_XmlParser(self.config, 'UP03') if (xp.convert() == False): self.logger.fatal( 'error in converting user product metadata to level 3') stderrWrite('Error in converting user product metadata to level 3') self.config.exitError() xp = L3_XmlParser(self.config, 'UP03') pi = xp.getTree('General_Info', 'L3_Product_Info') # update L2A entries from L2A_UP_MTD_XML: # 2015-07-30T10:39:14.021Z tmin = self.config.minTime tmax = self.config.maxTime pi.PRODUCT_START_TIME = tmin[:4] + '-' + tmin[4:6] + '-' + tmin[ 6:11] + ':' + tmin[11:13] + ':' + tmin[13:15] + '.000Z' pi.PRODUCT_STOP_TIME = tmax[:4] + '-' + tmax[4:6] + '-' + tmax[ 6:11] + '-' + tmax[11:13] + ':' + tmax[13:15] + '.000Z' pi.PROCESSING_LEVEL = 'Level-3p' pi.PRODUCT_TYPE = 'S2MSI3p' pi.PROCESSING_ALGORITHM = self.config.algorithm pi.RADIOMETRIC_PREFERENCE = self.config.radiometricPreference dt = datetime.utcnow() pi.GENERATION_TIME = strftime('%Y-%m-%dT%H:%M:%SZ', dt.timetuple()) qo = pi.Query_Options del qo[:] del pi.L3_Product_Organisation.Granule_List[:] aux = xp.getRoot('Auxiliary_Data_Info') del aux[:] l3auxData = xp.getTree('L3_Auxiliary_Data_Info', 'Aux_Data') l3auxData.clear() qii = xp.getRoot('Quality_Indicators_Info') del qii[:] l3icqi = xp.getTree('L3_Quality_Indicators_Info', 'Image_Content_QI') del l3icqi[:] l3qii = xp.getRoot('L3_Quality_Indicators_Info') tree = objectify.Element('L3_Classification_QI') tree.attrib['resolution'] = str(self.config.resolution) l3qii.append(tree) xp.export() #create datastrip ID: L3_DS_DIR = os.path.join(L3_TARGET_DIR, DATASTRIP) dirlist = sorted(os.listdir(L3_DS_DIR)) found = False for dirname in dirlist: if (fnmatch.fnmatch(dirname, S2A_mask) == True): found = True break if found == False: stderrWrite('No subdirectory in datastrip') self.config.exitError() L2A_DS_ID = dirname L3_DS_ID = L2A_DS_ID[:4] + 'USER' + L2A_DS_ID[8:] L3_DS_ID = L3_DS_ID.replace('L2A_', 'L03_') self.config.L3_DS_ID = L3_DS_ID olddir = os.path.join(L3_DS_DIR, L2A_DS_ID) newdir = os.path.join(L3_DS_DIR, L3_DS_ID) os.rename(olddir, newdir) #find datastrip metadada, rename and change it: L3_DS_DIR = newdir filelist = sorted(os.listdir(L3_DS_DIR)) found = False for filename in filelist: if (fnmatch.fnmatch(filename, S2A_mask) == True): found = True break if found == False: stderrWrite('No metadata in datastrip') self.config.exitError() LXX_DS_MTD_XML = filename L3_DS_MTD_XML = LXX_DS_MTD_XML[:4] + 'USER' + LXX_DS_MTD_XML[8:] L3_DS_MTD_XML = L3_DS_MTD_XML.replace('L2A_', 'L03_') oldfile = os.path.join(L3_DS_DIR, LXX_DS_MTD_XML) newfile = os.path.join(L3_DS_DIR, L3_DS_MTD_XML) self.config.L3_DS_MTD_XML = newfile os.rename(oldfile, newfile) xp = L3_XmlParser(self.config, 'DS03') if (xp.convert() == False): stderrWrite('Error in converting datastrip metadata to level 3.') self.logger.fatal( 'error in converting datastrip metadata to level 3.') self.config.exitError() xp = L3_XmlParser(self.config, 'DS03') ti = xp.getTree('Image_Data_Info', 'Tiles_Information') del ti.Tile_List.Tile[:] xp.export() self.createTable() return True