def process_10(self): if (self.scOnly == True): stdoutWrite( 'No scene classification processing for 10 m resolution in sc_only mode.\n' ) self.logger.info( 'no scene classification processing for 10 m resolution in sc_only mode' ) tMeasure = time() - self.config.tStart self.config.writeTimeEstimation(tMeasure) self._msgQueue.put(FAILURE) return False res20 = 20 if self.tables.checkAotMapIsPresent(res20) == False: stdoutWrite('20 m resolution must be processed first.\n') self.logger.info('20 m resolution must be processed first') if self.process_20() == False: return False self.config.resolution = 10 self.config.readPreferences() self.config.createOrUpdateL2A_UserProduct() self.tables = L2A_Tables(self.config) return self.process()
def selectAndProcess(self, tile): if(self.config.resolution == 10): self.config.tracer.info('selected resolution is 10m') self.config.logger.info('selected resolution is 10m') if(self._processed20 == False): self.config.resolution = 20 stdoutWrite('check if 20m table exists ...\n') self.config.tracer.info('20m resolution must be processed first') self.config.logger.info('20m resolution must be processed first') self.selectAndProcess(tile) self.config.resolution = 10 self.config.readPreferences() self.tables = L2A_Tables(self.config, tile) self.config.readTileMetadata() self._processed10 = self.process() if(self._processed10 == False): return False elif(self.config.resolution == 20): self.config.tracer.info('selected resolution is 20m') self.config.logger.info('selected resolution is 20m') if(self._processed60 == False): self.config.resolution = 60 stdoutWrite('check if 60m table exists...\n') self.config.tracer.info('60m resolution must be processed first') self.config.logger.info('60m resolution must be processed first') self.selectAndProcess(tile) self.config.resolution = 20 self.config.readPreferences() self.tables = L2A_Tables(self.config, tile) self.config.readTileMetadata() self._processed20 = self.process() if(self._processed20 == False): return False elif(self.config.resolution == 60): self.config.tracer.info('selected resolution is 60m') self.config.logger.info('selected resolution is 60m') self.config.readPreferences() self.tables = L2A_Tables(self.config, tile) self.config.readTileMetadata() self._processed60 = self.process() if(self._processed60 == False): return False else: self.config.logger.debug('wrong resolution for processing configured: ', str(self.config.resolution)) return False
def validate(self): fn = os.path.basename(self._xmlFn) self._logger.info('validating metadata file %s against scheme' % fn) l.acquire() try: schema = etree.XMLSchema(file = os.path.join(self._config.configDir, self._scheme)) parser = etree.XMLParser(schema = schema) objectify.parse(self._xmlFn, parser) self._logger.info('metadata file is valid') ret = True except etree.XMLSyntaxError, err: stdoutWrite('Metadata file is invalid, see report file for details.\n') self._logger.error('Schema file: %s' % self._scheme) self._logger.error('Details: %s' % str(err)) ret = False
def convert(self): import codecs objectify.deannotate(self._root, xsi_nil=True, cleanup_namespaces=True) outstr = etree.tostring(self._root, pretty_print=True) outstr = outstr.replace('-1C', '-2A') outstr = outstr.replace('<TILE_ID', '<TILE_ID_2A') outstr = outstr.replace('<DATASTRIP_ID', '<DATASTRIP_ID_2A') outstr = outstr.replace('<Product_Info', '<L2A_Product_Info') outstr = outstr.replace('<Product_Organisation', '<L2A_Product_Organisation') outstr = outstr.replace('<Product_Image_Characteristics', '<L2A_Product_Image_Characteristics') outstr = outstr.replace('<Pixel_Level_QI', '<L1C_Pixel_Level_QI') outstr = outstr.replace('</TILE_ID', '</TILE_ID_2A') outstr = outstr.replace('</DATASTRIP_ID', '</DATASTRIP_ID_2A') outstr = outstr.replace('</Product_Info', '</L2A_Product_Info') outstr = outstr.replace('</Product_Organisation', '</L2A_Product_Organisation') outstr = outstr.replace('</Product_Image_Characteristics', '</L2A_Product_Image_Characteristics') outstr = outstr.replace('</Pixel_Level_QI', '</L1C_Pixel_Level_QI') if self._product == 'T2A': outstr = outstr.replace('Image_Content_QI>', 'L1C_Image_Content_QI>') if self._product == 'UP2A': outstr = outstr.replace('QUANTIFICATION_VALUE', 'L1C_L2A_Quantification_Values_List') outstr = outstr.replace('</n1:Auxiliary_Data_Info>', '</n1:Auxiliary_Data_Info>\n'\ '<n1:L2A_Auxiliary_Data_Info/>') outstr = outstr.replace('</n1:Quality_Indicators_Info>', '</n1:Quality_Indicators_Info>\n'\ '<n1:L2A_Quality_Indicators_Info/>') l.acquire() try: outfile = codecs.open(self._xmlFn, 'w', 'utf-8') outfile.write('<?xml version="1.0" encoding="UTF-8"?>\n') outfile.write(outstr) outfile.close() except: stdoutWrite('Error in writing file: %s\n' % self._xmlFn) self._logger.error('error in writing file: %s\n' % self._xmlFn) finally: l.release() return self.setRoot()
def validate(self): fn = os.path.basename(self._xmlFn) self._logger.info('validating metadata file %s against scheme' % fn) l.acquire() try: schema = etree.XMLSchema( file=os.path.join(self._config.configDir, self._scheme)) parser = etree.XMLParser(schema=schema) objectify.parse(self._xmlFn, parser) self._logger.info('metadata file is valid') ret = True except etree.XMLSyntaxError, err: stdoutWrite( 'Metadata file is invalid, see report file for details.\n') self._logger.error('Schema file: %s' % self._scheme) self._logger.error('Details: %s' % str(err)) ret = False
def process_10(self): if (self.scOnly == True): stdoutWrite('No scene classification processing for 10 m resolution in sc_only mode.\n') self.logger.info('no scene classification processing for 10 m resolution in sc_only mode') tMeasure = time() - self.config.tStart self.config.writeTimeEstimation(tMeasure) self._msgQueue.put(FAILURE) return False res20 = 20 if self.tables.checkAotMapIsPresent(res20) == False: stdoutWrite('20 m resolution must be processed first.\n') self.logger.info('20 m resolution must be processed first') if self.process_20() == False: return False self.config.resolution = 10 self.config.readPreferences() self.config.createOrUpdateL2A_UserProduct() self.tables = L2A_Tables(self.config) return self.process()
def validate(self): dummy, fn = os.path.split(self._xmlFn) stdoutWrite('Validating metadata file %s against scheme ...\n' % fn) try: schema = etree.XMLSchema(file = self._config.configDir + '/' + self._scheme) parser = etree.XMLParser(schema = schema) objectify.parse(self._xmlFn, parser) stdoutWrite('Metadata file is valid.\n') return True except etree.XMLSyntaxError, err: stdoutWrite('Metadata file is invalid, see logfile for details.\n') self._config.logger.error('Schema file: %s' % self._scheme) self._config.logger.error('Details: %s' % str(err)) return False
def run(self): if self.config.resolution == 0: stdoutWrite( 'no resolution specified, will process all resolutions.\n') else: stdoutWrite('selected resolution is %s m.\n' % self.config.resolution) self.setupLogger() logger = self.logger logger.level = getLevel(self.config.logLevel) self.config.logger = logger if (self.config.resolution == 0): if self.process_60() == False: return False if self.process_20() == False: return False if self.process_10() == False: return False return True elif (self.config.resolution == 60): if self.process_60() == False: return False elif (self.config.resolution == 20): if self.process_20() == False: return False elif (self.config.resolution == 10): if (self.scOnly == True): stdoutWrite( 'No scene classification processing for 10 m resolution in sc_only mode.\n' ) self.logger.info( 'no scene classification processing for 10 m resolution in sc_only mode' ) tMeasure = time() - self.config.tStart self.config.writeTimeEstimation(tMeasure) self._msgQueue.put(FAILURE) return False if self.process_20() == False: return False elif self.process_10() == False: return False return True
def run(self): if self.config.resolution == 0: stdoutWrite('no resolution specified, will process all resolutions.\n') else: stdoutWrite('selected resolution is %s m.\n' % self.config.resolution) self.setupLogger() logger = self.logger logger.level = getLevel(self.config.logLevel) self.config.logger = logger if(self.config.resolution == 0): if self.process_60() == False: return False if self.process_20() == False: return False if self.process_10() == False: return False return True elif(self.config.resolution == 60): if self.process_60() == False: return False elif(self.config.resolution == 20): if self.process_20() == False: return False elif(self.config.resolution == 10): if (self.scOnly == True): stdoutWrite('No scene classification processing for 10 m resolution in sc_only mode.\n') self.logger.info('no scene classification processing for 10 m resolution in sc_only mode') tMeasure = time() - self.config.tStart self.config.writeTimeEstimation(tMeasure) self._msgQueue.put(FAILURE) return False if self.process_20() == False: return False elif self.process_10() == False: return False return True
fn = os.path.basename(self._xmlFn) self._logger.info('validating metadata file %s against scheme' % fn) l.acquire() try: schema = etree.XMLSchema(file = os.path.join(self._config.configDir, self._scheme)) parser = etree.XMLParser(schema = schema) objectify.parse(self._xmlFn, parser) self._logger.info('metadata file is valid') ret = True except etree.XMLSyntaxError, err: stdoutWrite('Metadata file is invalid, see report file for details.\n') self._logger.error('Schema file: %s' % self._scheme) self._logger.error('Details: %s' % str(err)) ret = False except: stdoutWrite('Unspecific Error in metadata.\n') self._logger.error('unspecific error in metadata') ret = False finally: l.release() return ret def append(self, key, value): ret = False l.acquire() try: e = etree.Element(key) e.text = value self._tree.append(e)
l.acquire() try: schema = etree.XMLSchema( file=os.path.join(self._config.configDir, self._scheme)) parser = etree.XMLParser(schema=schema) objectify.parse(self._xmlFn, parser) self._logger.info('metadata file is valid') ret = True except etree.XMLSyntaxError, err: stdoutWrite( 'Metadata file is invalid, see report file for details.\n') self._logger.error('Schema file: %s' % self._scheme) self._logger.error('Details: %s' % str(err)) ret = False except: stdoutWrite('Unspecific Error in metadata.\n') self._logger.error('unspecific error in metadata') ret = False finally: l.release() return ret def append(self, key, value): ret = False l.acquire() try: e = etree.Element(key) e.text = value self._tree.append(e) ret = True
def main(args=None): import argparse config = L2A_Config() descr = config.processorName +'. Version: '+ config.processorVersion + ', created: '+ config.processorDate + \ ', supporting Level-1C product version: ' + config.productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument('directory', help='Directory where the Level-1C input files are located') parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, must be 10, 20 or 60 [m]') parser.add_argument('--sc_only', action='store_true', help='Performs only the scene classification at 60m resolution') parser.add_argument('--profile', action='store_true', help='Profiles the processor\'s performance') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if(os.path.isabs(directory)) == False: cwd = os.getcwd() directory = os.path.join(cwd, directory) elif os.path.exists(args.directory) == False: stderrWrite('directory "%s" does not exist\n.' % args.directory) return False processor = L2A_Process(directory) processor.scOnly = args.sc_only HelloWorld = config.processorName +', '+ config.processorVersion +', created: '+ config.processorDate stdoutWrite('\n%s started ...\n' % HelloWorld) tStart = time() S2A_mask = 'S2A_*' # next statement creates L2A product Structure: config.initLogAndTrace() config.readPreferences() tiles = config.createL2A_UserProduct() for tile in tiles: if(fnmatch.fnmatch(tile, S2A_mask) == False): continue processor.resetProcessingStatus() config.initLogAndTrace() config.tracer.info(HelloWorld) config.logger.info(HelloWorld) # config.calcEarthSunDistance2(tile) if args.resolution == None: resolution = 60 else: resolution = args.resolution config.resolution = resolution config.setTimeEstimation(resolution) config.logger.debug('Module L2A_Process initialized') if(args.profile == True): import cProfile, pstats, StringIO pr = cProfile.Profile() pr.enable() result = processor.selectAndProcess(tile) pr.disable() s = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby).print_stats(.25, 'L2A_') ps.print_stats() profile = s.getvalue() s.close() with open(os.environ['SEN2COR_HOME'] + '/log/profile', 'w') as textFile: textFile.write(profile) textFile.close() else: result = processor.selectAndProcess(tile) if(result == False): stderrWrite('Application terminated with errors, see log file and traces.\n') return False tMeasure = time() - tStart config.writeTimeEstimation(resolution, tMeasure) stdoutWrite('\nApplication terminated successfully.\n') return True
def main(args=None): import argparse config = L2A_Config(None) descr = config.processorName +'. Version: '+ config.processorVersion + ', created: '+ config.processorDate + \ ', supporting Level-1C product version: ' + config.productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument( 'directory', help='Directory where the Level-1C input files are located') parser.add_argument( '--resolution', type=int, choices=[10, 20, 60], help= 'Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed' ) parser.add_argument( '--sc_only', action='store_true', help='Performs only the scene classification at 60 or 20m resolution') parser.add_argument( '--cr_only', action='store_true', help='Performs only the creation of the L2A product tree, no processing' ) # parser.add_argument('--profile', action='store_true', help='Profiles the processor\'s performance') parser.add_argument( '--refresh', action='store_true', help='Performs a refresh of the persistent configuration before start') parser.add_argument('--GIP_L2A', help='Select the user GIPP') parser.add_argument('--GIP_L2A_SC', help='Select the scene classification GIPP') parser.add_argument('--GIP_L2A_AC', help='Select the atmospheric correction GIPP') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/' or directory[-1] == '\\': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if (os.path.isabs(directory)) == False: cwd = os.getcwd() directory = os.path.join(cwd, directory) directory = os.path.normpath(directory) if os.path.exists(directory) == False: stderrWrite('directory "%s" does not exist\n.' % directory) return FAILURE # check if directory argument contains a tile. If yes, split the tile from path, # put the tile in the config object created below as selected tile, # create the new path for the user directory. selectedTile = None if 'GRANULE' in directory: dirname, selectedTile = os.path.split(directory) directory = os.path.dirname(dirname) test = os.path.basename(directory) S2A_L1C_mask = 'S2A_????_???_???L1C*' if (fnmatch.fnmatch(test, S2A_L1C_mask) == False): stderrWrite( 'L1C user product directory must match the following mask: %s\n' % S2A_L1C_mask) stderrWrite('but is: %s\n' % test) return FAILURE config = L2A_Config(None, directory) HelloWorld = config.processorName + ', ' + config.processorVersion + ', created: ' + config.processorDate stdoutWrite('\n%s started ...\n' % HelloWorld) # if(args.profile == True): # import cProfile, pstats, StringIO # pr = cProfile.Profile() # pr.enable() if args.resolution == None: resolution = 0 else: resolution = args.resolution # create and initialize the base log system: dirname, basename = os.path.split(directory) L2A_UP_ID = basename[:4] + 'USER' + basename[8:] L2A_UP_ID = L2A_UP_ID.replace('1C_', '2A_') logName = L2A_UP_ID + '_report.xml' logDir = config.logDir logLevel = config.logLevel fnLog = os.path.join(logDir, logName) if not os.path.exists(logDir): os.mkdir(logDir) try: f = open(config.processingStatusFn, 'w') f.write('0.0\n') f.close() except: stderrWrite('cannot create process status file: %s\n' % config.processingStatusFn) return FAILURE try: f = open(fnLog, 'w') f.write('<?xml version="1.0" encoding="UTF-8"?>\n') f.write('<Sen2Cor_Level-2A_Report_File>\n') f.close() except: stderrWrite('cannot update the report file: %s\n' % fnLog) return FAILURE # Just a normal logger logger = logging.getLogger('sen2cor') handler = logging.FileHandler(fnLog) handler.setFormatter(formatter) logger.addHandler(handler) logger.level = getLevel(logLevel) logger.info('logging for the main process initialized') config.logger = logger CFG = 'cfg' if args.GIP_L2A != None: config._configFn = os.path.join(config.home, CFG, args.GIP_L2A) if args.GIP_L2A_SC != None: config.configSC = os.path.join(config.home, CFG, args.GIP_L2A_SC) if args.GIP_L2A_AC != None: config.configAC = os.path.join(config.home, CFG, args.GIP_L2A_AC) config.workDir = directory config.resolution = resolution config.scOnly = args.sc_only config.crOnly = args.cr_only config.refresh = args.refresh config.selectedTile = selectedTile result = config.readPreferences() if result == False: return FAILURE config.tStart = time() config.setTimeEstimation(resolution) L2A_TILES = updateTiles(config) if L2A_TILES == False: return FAILURE result = SUCCESS if config.crOnly == False: scheduler = L2A_Schedule(config, L2A_TILES) result = scheduler.sync() config.logger = logger # validate the meta data on user product level: try: xp = L2A_XmlParser(config, 'UP2A') xp.validate() except: logger.error('parsing error for user product') result = FAILURE # if(args.profile == True): # pr.disable() # s = StringIO.StringIO() # sortby = 'cumulative' # ps = pstats.Stats(pr, stream=s).sort_stats(sortby).print_stats(.25, 'L2A_') # ps.print_stats() # profile = s.getvalue() # s.close() # with open(os.path.join(getScriptDir(), 'log', 'profile'), 'w') as textFile: # textFile.write(profile) # textFile.close() else: config.logger = logger #Create the manifest.safe (L2A) mn = L2A_Manifest(config) mn.generate(config.L2A_UP_DIR, config.L2A_MANIFEST_SAFE) try: xp = L2A_XmlParser(config, 'Manifest') xp.validate() except: logger.error('parsing error for manifest') result = FAILURE if postprocess(config) == False: result = FAILURE if result == FAILURE: stdoutWrite( 'Progress[%]: 100.00 : Application terminated with at least one error.\n' ) else: stdoutWrite( 'Progress[%]: 100.00 : Application terminated successfully.\n') return result
def main(args=None): import argparse config = L2A_Config(None) descr = config.processorName +'. Version: '+ config.processorVersion + ', created: '+ config.processorDate + \ ', supporting Level-1C product version: ' + config.productVersion + '.' parser = argparse.ArgumentParser(description=descr) parser.add_argument('directory', help='Directory where the Level-1C input files are located') parser.add_argument('--resolution', type=int, choices=[10, 20, 60], help='Target resolution, can be 10, 20 or 60m. If omitted, all resolutions will be processed') parser.add_argument('--sc_only', action='store_true', help='Performs only the scene classification at 60 or 20m resolution') parser.add_argument('--cr_only', action='store_true', help='Performs only the creation of the L2A product tree, no processing') # parser.add_argument('--profile', action='store_true', help='Profiles the processor\'s performance') parser.add_argument('--refresh', action='store_true', help='Performs a refresh of the persistent configuration before start') parser.add_argument('--GIP_L2A', help='Select the user GIPP') parser.add_argument('--GIP_L2A_SC', help='Select the scene classification GIPP') parser.add_argument('--GIP_L2A_AC', help='Select the atmospheric correction GIPP') args = parser.parse_args() # SIITBX-49: directory should not end with '/': directory = args.directory if directory[-1] == '/' or directory[-1] == '\\': directory = directory[:-1] # check if directory argument starts with a relative path. If not, expand: if(os.path.isabs(directory)) == False: cwd = os.getcwd() directory = os.path.join(cwd, directory) directory = os.path.normpath(directory) if os.path.exists(directory) == False: stderrWrite('directory "%s" does not exist\n.' % directory) return FAILURE # check if directory argument contains a tile. If yes, split the tile from path, # put the tile in the config object created below as selected tile, # create the new path for the user directory. selectedTile = None if 'GRANULE' in directory: dirname, selectedTile = os.path.split(directory) directory = os.path.dirname(dirname) test = os.path.basename(directory) S2A_L1C_mask = 'S2A_????_???_???L1C*' if(fnmatch.fnmatch(test, S2A_L1C_mask) == False): stderrWrite('L1C user product directory must match the following mask: %s\n' % S2A_L1C_mask) stderrWrite('but is: %s\n' % test) return FAILURE config = L2A_Config(None, directory) HelloWorld = config.processorName +', '+ config.processorVersion +', created: '+ config.processorDate stdoutWrite('\n%s started ...\n' % HelloWorld) # if(args.profile == True): # import cProfile, pstats, StringIO # pr = cProfile.Profile() # pr.enable() if args.resolution == None: resolution = 0 else: resolution = args.resolution # create and initialize the base log system: dirname, basename = os.path.split(directory) L2A_UP_ID = basename[:4] + 'USER' + basename[8:] L2A_UP_ID = L2A_UP_ID.replace('1C_', '2A_') logName = L2A_UP_ID + '_report.xml' logDir = config.logDir logLevel = config.logLevel fnLog = os.path.join(logDir, logName) if not os.path.exists(logDir): os.mkdir(logDir) try: f = open(config.processingStatusFn, 'w') f.write('0.0\n') f.close() except: stderrWrite('cannot create process status file: %s\n' % config.processingStatusFn) return FAILURE try: f = open(fnLog, 'w') f.write('<?xml version="1.0" encoding="UTF-8"?>\n') f.write('<Sen2Cor_Level-2A_Report_File>\n') f.close() except: stderrWrite('cannot update the report file: %s\n' % fnLog) return FAILURE # Just a normal logger logger = logging.getLogger('sen2cor') handler = logging.FileHandler(fnLog) handler.setFormatter(formatter) logger.addHandler(handler) logger.level = getLevel(logLevel) logger.info('logging for the main process initialized') config.logger = logger CFG = 'cfg' if args.GIP_L2A != None: config._configFn = os.path.join(config.home, CFG, args.GIP_L2A) if args.GIP_L2A_SC != None: config.configSC = os.path.join(config.home, CFG, args.GIP_L2A_SC) if args.GIP_L2A_AC != None: config.configAC = os.path.join(config.home , CFG, args.GIP_L2A_AC) config.workDir = directory config.resolution = resolution config.scOnly = args.sc_only config.crOnly = args.cr_only config.refresh = args.refresh config.selectedTile = selectedTile result = config.readPreferences() if result == False: return FAILURE config.tStart = time() config.setTimeEstimation(resolution) L2A_TILES = updateTiles(config) if L2A_TILES == False: return FAILURE result = SUCCESS if config.crOnly == False: scheduler = L2A_Schedule(config, L2A_TILES) result = scheduler.sync() config.logger = logger # validate the meta data on user product level: try: xp = L2A_XmlParser(config, 'UP2A') xp.validate() except: logger.error('parsing error for user product') result = FAILURE # if(args.profile == True): # pr.disable() # s = StringIO.StringIO() # sortby = 'cumulative' # ps = pstats.Stats(pr, stream=s).sort_stats(sortby).print_stats(.25, 'L2A_') # ps.print_stats() # profile = s.getvalue() # s.close() # with open(os.path.join(getScriptDir(), 'log', 'profile'), 'w') as textFile: # textFile.write(profile) # textFile.close() else: config.logger = logger #Create the manifest.safe (L2A) mn = L2A_Manifest(config) mn.generate(config.L2A_UP_DIR, config.L2A_MANIFEST_SAFE) try: xp = L2A_XmlParser(config, 'Manifest') xp.validate() except: logger.error('parsing error for manifest') result = FAILURE if postprocess(config) == False: result = FAILURE if result == FAILURE: stdoutWrite('Progress[%]: 100.00 : Application terminated with at least one error.\n') else: stdoutWrite('Progress[%]: 100.00 : Application terminated successfully.\n') return result