def main(args): # create a parser instance parser = argparse.ArgumentParser( add_help=True, fromfile_prefix_chars="@", argument_default=argparse.SUPPRESS, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.description = textwrap.dedent( """\ Takes <infile> raster data and pre-processes it into <outfiles_basename>.tif, a GeoTIFF converted to RGB using default internal tiling, internal overviews, no compression, and 0 as no-data value, and <outfiles_basename>.xml, a EOxServer simple XML EO metadata file. The outfiles are ready to be used with the eoxs_register command of an EOxServer instance. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!Caution, you will loose data quality!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! The data quality of <outfiles_basename>.tif is most likely different from <infile> because of the pre-processing applied. Reasons include re-projection, compression, conversion to RGB, indexing, etc. Examples: # basic usage with no creation of metadata eoxserver-preprocess.py --no-metadata input.tif # basic usage with creation of metadata with specific basename eoxserver-preprocess.py --coverage-id=a --begin-time=2008-03-01T13:00:00Z \\ --end-time=2008-03-01T13:00:00Z input.tif output # with RGBA band selection eoxserver-preprocess.py --bands 1:0:255,2:0:255,3:0:100,4 --rgba \\ --no-metadata input.tif # with DEFLATE compression and color index from a palette file eoxserver-preprocess.py --compression=DEFLATE --zlevel=2 --indexed \\ --pct palette.vrt --no-metadata input.tif # (re-)setting the extent of the file eoxserver-preprocess.py --extent 0,0,10,10,4326 --no-metadata input.tif # Using GCPs eoxserver-preprocess.py --no-metadata --gcp 0,0,10,10 --gcp 2560,0,50,10 \\ --gcp 0,1920,10,50 --gcp 2560,1920,50,50 \\ --georef-crs 4326 input.tif # reading arguments from a file (1 line per argument), with overrides eoxserver-preprocess.py @args.txt --crs=3035 --no-tiling input.tif """ ) # =========================================================================== # Metadata parsing group # =========================================================================== # TODO: won't work with mutual exclusive groups. Bug in Argparse? # md_g = parser.add_mutually_exclusive_group(required=True) # md_g.add_argument("--no-metadata", dest="generate_metadata", # action="store_false", # help="Explicitly turn off the creation of a metadata " # "file.") # md_g_data = md_g.add_argument_group() # md_g_data.add_argument("--begin-time", dest="begin_time", # type=_parse_datetime, # help="The ISO 8601 timestamp of the begin time.") # md_g_data.add_argument("--end-time", dest="end_time", # type=_parse_datetime, # help="The ISO 8601 timestamp of the end time.") # md_g_data.add_argument("--coverage-id", dest="coverage_id", # type=_parse_coverage_id, # help="The ID of the coverage, must be a valid " # "NCName.") # should be mutually exclusive parser.add_argument( "--no-metadata", dest="generate_metadata", action="store_false", help="Explicitly turn off the creation of a metadata " "file.", ) parser.add_argument( "--begin-time", dest="begin_time", type=_parse_datetime, help="The ISO 8601 timestamp of the begin time." ) parser.add_argument( "--end-time", dest="end_time", type=_parse_datetime, help="The ISO 8601 timestamp of the end time." ) parser.add_argument( "--coverage-id", dest="coverage_id", type=_parse_coverage_id, help="The ID of the coverage, must be a valid NCName.", ) # =========================================================================== # Georeference group # =========================================================================== georef_g = parser.add_mutually_exclusive_group() georef_g.add_argument( "--extent", dest="extent", type=_parse_extent, help="The extent of the dataset, as a 4-tuple of " "floats." ) # georef_g.add_argument("--footprint", dest="footprint", # type=_parse_footprint, # help="The footprint of the dataset, as a Polygon WKT.") georef_g.add_argument( "--gcp", dest="gcps", type=_parse_gcp, action="append", help="A Ground Control Point in the format: " "'pixel,line,easting,northing[,elevation]'.", ) parser.add_argument("--georef-crs", dest="georef_crs") # =========================================================================== # Arbitraries # =========================================================================== parser.add_argument("--crs", dest="crs", type=int, help="The desired output CRS ID of the dataset.") parser.add_argument( "--no-tiling", dest="tiling", action="store_false", default=True, help="Explicitly turn of the tiling of the output " "dataset.", ) parser.add_argument( "--no-overviews", dest="overviews", action="store_false", default=True, help="Explicitly turn of the creation of overview " "images of the output dataset.", ) # =========================================================================== # Bands group # =========================================================================== bands_g = parser.add_mutually_exclusive_group() bands_g.add_argument( "--rgba", dest="bandmode", action="store_const", const=RGBA, help="Convert the image to RGBA, using the first four " "bands.", ) bands_g.add_argument( "--orig-bands", dest="bandmode", action="store_const", const=ORIG_BANDS, help="Explicitly keep all original bands.", ) parser.add_argument( "--bands", dest="bands", type=_parse_bands, help="A comma separated list of bands with optional " "subsets in the form: 'no[:low:high]'. Either " "three bands, or four when --rgba is requested.", ) parser.add_argument( "--footprint-alpha", dest="footprint_alpha", action="store_true", default=False, help="A comma separated list of bands with optional " "subsets in the form: 'no[:low:high]'. Either " "three bands, or four when --rgba is requested.", ) parser.add_argument( "--compression", dest="compression", choices=GeoTIFFFormatSelection.SUPPORTED_COMPRESSIONS, help="The desired compression technique.", ) parser.add_argument( "--jpeg-quality", dest="jpeg_quality", type=int, help="The JPEG algorithm quality when JPEG compression " "is requested. Default is 75.", ) parser.add_argument( "--zlevel", dest="zlevel", type=int, help="The zlevel quality setting when DEFLATE " "compression is requested." ) indexed_group = parser.add_argument_group() # TODO: pct depends on indexed parser.add_argument("--indexed", dest="color_index", action="store_true", help="Create a paletted (indexed) image.") parser.add_argument( "--pct", dest="palette_file", help="Use the given file as color palette. If not " "given, a new palette will be calculated.", ) parser.add_argument( "--no-data-value", dest="no_data_value", type=_parse_nodata_values, help="Either one, or a list of no-data values.", ) parser.add_argument( "--co", dest="creation_options", type=int, action="append", help="Additional GDAL dataset creation options. " "See http://www.gdal.org/frmt_gtiff.html", ) parser.add_argument("--traceback", action="store_true", default=False) parser.add_argument( "--force", "-f", dest="force", action="store_true", default=False, help="Override files, if they already exist." ) parser.add_argument( "--verbosity", "-v", dest="verbosity", type=int, default=1, help="Set the verbosity (0, 1, 2). Default is 1." ) parser.add_argument("input_filename", metavar="infile", nargs=1, help="The input raster file to be processed.") parser.add_argument( "output_basename", metavar="outfiles_basename", nargs="?", help="The base name of the outputfile(s) to be " "generated.", ) values = vars(parser.parse_args(args)) # check metadata values if "generate_metadata" in values and ("begin_time" in values or "end_time" in values or "coverage_id" in values): parser.error("--no-metadata is mutually exclusive with --begin-time, " "--end-time and --coverage-id.") elif "generate_metadata" not in values and not ( "begin_time" in values and "end_time" in values and "coverage_id" in values ): parser.error("Enter the full metadata with --begin-time, --end-time " "and --coverage-id.") # hack to flatten the list values["input_filename"] = values["input_filename"][0] georef_crs = values.pop("georef_crs", None) if "extent" in values: values["geo_reference"] = Extent(*values.pop("extent"), srid=georef_crs or 4326) if "gcps" in values: values["geo_reference"] = GCPList(values.pop("gcps"), georef_crs or 4326) if "palette_file" in values and not "color_index" in values: parser.error("--pct can only be used with --indexed") # Extract format and execution specific values format_values = _extract(values, ("tiling", "compression", "jpeg_quality", "zlevel", "creation_options")) exec_values = _extract(values, ("input_filename", "geo_reference", "generate_metadata")) other_values = _extract(values, ("traceback",)) metadata_values = _extract(values, ("coverage_id", "begin_time", "end_time")) force = values.pop("force", True) verbosity = values.pop("verbosity") output_basename = values.pop("output_basename", None) input_filename = exec_values.get("input_filename") # setup logging if verbosity > 0: if verbosity == 1: level = logging.WARN elif verbosity == 2: level = logging.INFO elif verbosity >= 3: level = logging.DEBUG logging.basicConfig(format="%(levelname)s: %(message)s", stream=sys.stderr, level=level) try: # create a format selection format_selection = get_format_selection("GTiff", **format_values) # TODO: make 'tif' dependant on format selection # check files exist if not output_basename: output_filename = splitext(input_filename)[0] + "_proc" + format_selection.extension output_md_filename = splitext(input_filename)[0] + "_proc.xml" else: output_filename = output_basename + format_selection.extension output_md_filename = output_basename + ".xml" if not force: check_file_existence(output_filename) exec_values["output_filename"] = output_filename # create and run the preprocessor preprocessor = WMSPreProcessor(format_selection, **values) result = preprocessor.process(**exec_values) if exec_values.get("generate_metadata", True): if not force: check_file_existence(output_md_filename) encoder = NativeMetadataFormatEncoder() xml = DOMElementToXML( encoder.encodeMetadata( metadata_values["coverage_id"], metadata_values["begin_time"], metadata_values["end_time"], result.footprint_raw, ) ) with open(output_md_filename, "w+") as f: f.write(xml) except Exception, e: # error wrapping if other_values["traceback"]: traceback.print_exc() sys.stderr.write("%s: %s\n" % (type(e).__name__, str(e)))
def ingest_browse_report(parsed_browse_report, do_preprocessing=True, config=None): """ Ingests a browse report. reraise_exceptions if errors shall be handled externally """ # initialize the EOxServer system/registry/configuration System.init() try: # get the according browse layer browse_type = parsed_browse_report.browse_type browse_layer = models.BrowseLayer.objects.get(browse_type=browse_type) except models.BrowseLayer.DoesNotExist: logger.warn("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) raise IngestionException( "Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type ) # generate a browse report model browse_report = create_browse_report(parsed_browse_report, browse_layer) # initialize the preprocessor with configuration values crs = None if browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible": crs = "EPSG:3857" elif browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad": crs = "EPSG:4326" logger.debug("Using CRS '%s' ('%s')." % (crs, browse_layer.grid)) # create the required preprocessor/format selection format_selection = get_format_selection("GTiff", **get_format_config(config)) if do_preprocessing: # add config parameters and custom params params = get_optimization_config(config) # add radiometric interval rad_min = browse_layer.radiometric_interval_min if rad_min is not None: params["radiometric_interval_min"] = rad_min else: rad_min = "min" rad_max = browse_layer.radiometric_interval_max if rad_max is not None: params["radiometric_interval_max"] = rad_max else: rad_max = "max" # add band selection if browse_layer.r_band is not None and browse_layer.g_band is not None and browse_layer.b_band is not None: bands = [ (browse_layer.r_band, rad_min, rad_max), (browse_layer.g_band, rad_min, rad_max), (browse_layer.b_band, rad_min, rad_max), ] if params["bandmode"] == RGBA: # RGBA bands.append((0, 0, 0)) params["bands"] = bands preprocessor = NGEOPreProcessor(format_selection, crs=crs, **params) else: preprocessor = None # TODO: CopyPreprocessor report_result = IngestBrowseReportResult() succeded = [] failed = [] timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S%f") browse_dirname = _valid_path( "%s_%s_%s_%s" % ( browse_type, browse_report.responsible_org_name, browse_report.date_time.strftime("%Y%m%d%H%M%S%f"), timestamp, ) ) success_dir = join(get_success_dir(config), browse_dirname) failure_dir = join(get_failure_dir(config), browse_dirname) if exists(success_dir): logger.warn("Success directory '%s' already exists.") else: makedirs(success_dir) if exists(failure_dir): logger.warn("Failure directory '%s' already exists.") else: makedirs(failure_dir) # iterate over all browses in the browse report for parsed_browse in parsed_browse_report: # transaction management per browse with transaction.commit_manually(): with transaction.commit_manually(using="mapcache"): try: seed_areas = [] # try ingest a single browse and log success result = ingest_browse( parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=config, ) report_result.add(result) succeded.append(parsed_browse) # commit here to allow seeding transaction.commit() transaction.commit(using="mapcache") logger.info("Committed changes to database.") for minx, miny, maxx, maxy, start_time, end_time in seed_areas: try: # seed MapCache synchronously # TODO: maybe replace this with an async solution seed_mapcache( tileset=browse_layer.id, grid=browse_layer.grid, minx=minx, miny=miny, maxx=maxx, maxy=maxy, minzoom=browse_layer.lowest_map_level, maxzoom=browse_layer.highest_map_level, start_time=start_time, end_time=end_time, delete=False, **get_mapcache_seed_config(config) ) logger.info("Successfully finished seeding.") except Exception, e: logger.warn("Seeding failed: %s" % str(e)) # log ingestions for report generation # date/browseType/browseLayerId/start/end report_logger.info( "/\\/\\".join( ( datetime.utcnow().isoformat("T") + "Z", parsed_browse_report.browse_type, browse_layer.id, ( parsed_browse.start_time.replace(tzinfo=None) - parsed_browse.start_time.utcoffset() ).isoformat("T") + "Z", ( parsed_browse.end_time.replace(tzinfo=None) - parsed_browse.end_time.utcoffset() ).isoformat("T") + "Z", ) ) ) except Exception, e: # report error logger.error("Failure during ingestion of browse '%s'." % parsed_browse.browse_identifier) logger.error("Exception was '%s': %s" % (type(e).__name__, str(e))) logger.debug(traceback.format_exc() + "\n") # undo latest changes, append the failure and continue report_result.add( IngestBrowseFailureResult( parsed_browse.browse_identifier, getattr(e, "code", None) or type(e).__name__, str(e) ) ) failed.append(parsed_browse) transaction.rollback() transaction.rollback(using="mapcache")
def ingest_browse_report(parsed_browse_report, do_preprocessing=True, config=None): """ Ingests a browse report. reraise_exceptions if errors shall be handled externally """ # initialize the EOxServer system/registry/configuration System.init() try: # get the according browse layer browse_type = parsed_browse_report.browse_type browse_layer = models.BrowseLayer.objects.get(browse_type=browse_type) except models.BrowseLayer.DoesNotExist: logger.warn("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) raise IngestionException("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) # generate a browse report model browse_report = create_browse_report(parsed_browse_report, browse_layer) # initialize the preprocessor with configuration values crs = None if browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible": crs = "EPSG:3857" elif browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad": crs = "EPSG:4326" logger.debug("Using CRS '%s' ('%s')." % (crs, browse_layer.grid)) # create the required preprocessor/format selection format_selection = get_format_selection("GTiff", **get_format_config(config)) if do_preprocessing: # add config parameters and custom params params = get_optimization_config(config) # add radiometric interval rad_min = browse_layer.radiometric_interval_min if rad_min is not None: params["radiometric_interval_min"] = rad_min else: rad_min = "min" rad_max = browse_layer.radiometric_interval_max if rad_max is not None: params["radiometric_interval_max"] = rad_max else: rad_max = "max" # add band selection if (browse_layer.r_band is not None and browse_layer.g_band is not None and browse_layer.b_band is not None): bands = [(browse_layer.r_band, rad_min, rad_max), (browse_layer.g_band, rad_min, rad_max), (browse_layer.b_band, rad_min, rad_max)] if params["bandmode"] == RGBA: # RGBA bands.append((0, 0, 0)) params["bands"] = bands preprocessor = NGEOPreProcessor(format_selection, crs=crs, **params) else: preprocessor = None # TODO: CopyPreprocessor report_result = IngestBrowseReportResult() succeded = [] failed = [] timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S%f") browse_dirname = _valid_path("%s_%s_%s_%s" % ( browse_type, browse_report.responsible_org_name, browse_report.date_time.strftime("%Y%m%d%H%M%S%f"), timestamp )) success_dir = join(get_success_dir(config), browse_dirname) failure_dir = join(get_failure_dir(config), browse_dirname) if exists(success_dir): logger.warn("Success directory '%s' already exists.") else: makedirs(success_dir) if exists(failure_dir): logger.warn("Failure directory '%s' already exists.") else: makedirs(failure_dir) # iterate over all browses in the browse report for parsed_browse in parsed_browse_report: # transaction management per browse with transaction.commit_manually(): with transaction.commit_manually(using="mapcache"): try: seed_areas = [] # try ingest a single browse and log success result = ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=config) report_result.add(result) succeded.append(parsed_browse) # commit here to allow seeding transaction.commit() transaction.commit(using="mapcache") logger.info("Committed changes to database.") for minx, miny, maxx, maxy, start_time, end_time in seed_areas: try: # seed MapCache synchronously # TODO: maybe replace this with an async solution seed_mapcache(tileset=browse_layer.id, grid=browse_layer.grid, minx=minx, miny=miny, maxx=maxx, maxy=maxy, minzoom=browse_layer.lowest_map_level, maxzoom=browse_layer.highest_map_level, start_time=start_time, end_time=end_time, delete=False, **get_mapcache_seed_config(config)) logger.info("Successfully finished seeding.") except Exception, e: logger.warn("Seeding failed: %s" % str(e)) # log ingestions for report generation # date/browseType/browseLayerId/start/end report_logger.info("/\\/\\".join(( datetime.utcnow().isoformat("T") + "Z", parsed_browse_report.browse_type, browse_layer.id, (parsed_browse.start_time.replace(tzinfo=None)-parsed_browse.start_time.utcoffset()).isoformat("T") + "Z", (parsed_browse.end_time.replace(tzinfo=None)-parsed_browse.end_time.utcoffset()).isoformat("T") + "Z" ))) except Exception, e: # report error logger.error("Failure during ingestion of browse '%s'." % parsed_browse.browse_identifier) logger.error("Exception was '%s': %s" % (type(e).__name__, str(e))) logger.debug(traceback.format_exc() + "\n") # undo latest changes, append the failure and continue report_result.add(IngestBrowseFailureResult( parsed_browse.browse_identifier, getattr(e, "code", None) or type(e).__name__, str(e)) ) failed.append(parsed_browse) transaction.rollback() transaction.rollback(using="mapcache")
def main(args): # create a parser instance parser = argparse.ArgumentParser(add_help=True, fromfile_prefix_chars='@', argument_default=argparse.SUPPRESS, formatter_class=argparse.RawDescriptionHelpFormatter) parser.description = textwrap.dedent("""\ Takes <infile> raster data and pre-processes it into <outfiles_basename>.tif, a GeoTIFF converted to RGB using default internal tiling, internal overviews, no compression, and 0 as no-data value, and <outfiles_basename>.xml, a EOxServer simple XML EO metadata file. The outfiles are ready to be used with the eoxs_register command of an EOxServer instance. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!Caution, you will loose data quality!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! The data quality of <outfiles_basename>.tif is most likely different from <infile> because of the pre-processing applied. Reasons include re-projection, compression, conversion to RGB, indexing, etc. Examples: # basic usage with no creation of metadata eoxserver-preprocess.py --no-metadata input.tif # basic usage with creation of metadata with specific basename eoxserver-preprocess.py --coverage-id=a --begin-time=2008-03-01T13:00:00Z \\ --end-time=2008-03-01T13:00:00Z input.tif output # with RGBA band selection eoxserver-preprocess.py --bands 1:0:255,2:0:255,3:0:100,4 --rgba \\ --no-metadata input.tif # with DEFLATE compression and color index from a palette file eoxserver-preprocess.py --compression=DEFLATE --zlevel=2 --indexed \\ --pct palette.vrt --no-metadata input.tif # (re-)setting the extent of the file eoxserver-preprocess.py --extent 0,0,10,10,4326 --no-metadata input.tif # Using GCPs eoxserver-preprocess.py --no-metadata --gcp 0,0,10,10 --gcp 2560,0,50,10 \\ --gcp 0,1920,10,50 --gcp 2560,1920,50,50 \\ --georef-crs 4326 input.tif # reading arguments from a file (1 line per argument), with overrides eoxserver-preprocess.py @args.txt --crs=3035 --no-tiling input.tif """) #=========================================================================== # Metadata parsing group #=========================================================================== # TODO: won't work with mutual exclusive groups. Bug in Argparse? #md_g = parser.add_mutually_exclusive_group(required=True) #md_g.add_argument("--no-metadata", dest="generate_metadata", # action="store_false", # help="Explicitly turn off the creation of a metadata " # "file.") #md_g_data = md_g.add_argument_group() #md_g_data.add_argument("--begin-time", dest="begin_time", # type=_parse_datetime, # help="The ISO 8601 timestamp of the begin time.") #md_g_data.add_argument("--end-time", dest="end_time", # type=_parse_datetime, # help="The ISO 8601 timestamp of the end time.") #md_g_data.add_argument("--coverage-id", dest="coverage_id", # type=_parse_coverage_id, # help="The ID of the coverage, must be a valid " # "NCName.") # should be mutually exclusive parser.add_argument("--no-metadata", dest="generate_metadata", action="store_false", help="Explicitly turn off the creation of a metadata " "file.") parser.add_argument("--begin-time", dest="begin_time", type=_parse_datetime, help="The ISO 8601 timestamp of the begin time.") parser.add_argument("--end-time", dest="end_time", type=_parse_datetime, help="The ISO 8601 timestamp of the end time.") parser.add_argument("--coverage-id", dest="coverage_id", type=_parse_coverage_id, help="The ID of the coverage, must be a valid NCName.") #=========================================================================== # Georeference group #=========================================================================== georef_g = parser.add_mutually_exclusive_group() georef_g.add_argument("--extent", dest="extent", type=_parse_extent, help="The extent of the dataset, as a 4-tuple of " "floats.") #georef_g.add_argument("--footprint", dest="footprint", # type=_parse_footprint, # help="The footprint of the dataset, as a Polygon WKT.") georef_g.add_argument("--gcp", dest="gcps", type=_parse_gcp, action="append", help="A Ground Control Point in the format: " "'pixel,line,easting,northing[,elevation]'.") parser.add_argument("--georef-crs", dest="georef_crs") #=========================================================================== # Arbitraries #=========================================================================== parser.add_argument("--crs", dest="crs", type=int, help="The desired output CRS ID of the dataset.") parser.add_argument("--no-tiling", dest="tiling", action="store_false", default=True, help="Explicitly turn of the tiling of the output " "dataset.") parser.add_argument("--no-overviews", dest="overviews", action="store_false", default=True, help="Explicitly turn of the creation of overview " "images of the output dataset.") #=========================================================================== # Bands group #=========================================================================== bands_g = parser.add_mutually_exclusive_group() bands_g.add_argument("--rgba", dest="bandmode", action="store_const", const=RGBA, help="Convert the image to RGBA, using the first four " "bands.") bands_g.add_argument("--orig-bands", dest="bandmode", action="store_const", const=ORIG_BANDS, help="Explicitly keep all original bands.") parser.add_argument("--bands", dest="bands", type=_parse_bands, help="A comma separated list of bands with optional " "subsets in the form: 'no[:low:high]'. Either " "three bands, or four when --rgba is requested.") parser.add_argument("--footprint-alpha", dest="footprint_alpha", action="store_true", default=False, help="A comma separated list of bands with optional " "subsets in the form: 'no[:low:high]'. Either " "three bands, or four when --rgba is requested.") parser.add_argument("--compression", dest="compression", choices=GeoTIFFFormatSelection.SUPPORTED_COMPRESSIONS, help="The desired compression technique.") parser.add_argument("--jpeg-quality", dest="jpeg_quality", type=int, help="The JPEG algorithm quality when JPEG compression " "is requested. Default is 75.") parser.add_argument("--zlevel", dest="zlevel", type=int, help="The zlevel quality setting when DEFLATE " "compression is requested.") indexed_group = parser.add_argument_group() # TODO: pct depends on indexed parser.add_argument("--indexed", dest="color_index", action="store_true", help="Create a paletted (indexed) image.") parser.add_argument("--pct", dest="palette_file", help="Use the given file as color palette. If not " "given, a new palette will be calculated.") parser.add_argument("--no-data-value", dest="no_data_value", type=_parse_nodata_values, help="Either one, or a list of no-data values.") parser.add_argument("--co", dest="creation_options", type=int, action="append", help="Additional GDAL dataset creation options. " "See http://www.gdal.org/frmt_gtiff.html") parser.add_argument("--traceback", action="store_true", default=False) parser.add_argument("--force", "-f", dest="force", action="store_true", default=False, help="Override files, if they already exist.") parser.add_argument("--verbosity", "-v", dest="verbosity", type=int, default=1, help="Set the verbosity (0, 1, 2). Default is 1.") parser.add_argument("input_filename", metavar="infile", nargs=1, help="The input raster file to be processed.") parser.add_argument("output_basename", metavar="outfiles_basename", nargs="?", help="The base name of the outputfile(s) to be " "generated.") values = vars(parser.parse_args(args)) # check metadata values if "generate_metadata" in values and ("begin_time" in values or "end_time" in values or "coverage_id" in values): parser.error("--no-metadata is mutually exclusive with --begin-time, " "--end-time and --coverage-id.") elif "generate_metadata" not in values and not ("begin_time" in values and "end_time" in values and "coverage_id" in values): parser.error("Enter the full metadata with --begin-time, --end-time " "and --coverage-id.") # hack to flatten the list values["input_filename"] = values["input_filename"][0] georef_crs = values.pop("georef_crs", None) if "extent" in values: values["geo_reference"] = Extent(*values.pop("extent"), srid=georef_crs or 4326) if "gcps" in values: values["geo_reference"] = GCPList(values.pop("gcps"), georef_crs or 4326) if "palette_file" in values and not "color_index" in values: parser.error("--pct can only be used with --indexed") # Extract format and execution specific values format_values = _extract(values, ("tiling", "compression", "jpeg_quality", "zlevel", "creation_options")) exec_values = _extract(values, ("input_filename", "geo_reference", "generate_metadata")) other_values = _extract(values, ("traceback", )) metadata_values = _extract(values, ("coverage_id", "begin_time", "end_time")) force = values.pop("force", True) verbosity = values.pop("verbosity") output_basename = values.pop("output_basename", None) input_filename = exec_values.get("input_filename") # setup logging if verbosity > 0: if verbosity == 1: level = logging.WARN elif verbosity == 2: level = logging.INFO elif verbosity >= 3: level = logging.DEBUG logging.basicConfig(format="%(levelname)s: %(message)s", stream=sys.stderr, level=level) try: # create a format selection format_selection = get_format_selection("GTiff", **format_values) # TODO: make 'tif' dependant on format selection # check files exist if not output_basename: output_filename = splitext(input_filename)[0] + "_proc" + format_selection.extension output_md_filename = splitext(input_filename)[0] + "_proc.xml" else: output_filename = output_basename + format_selection.extension output_md_filename = output_basename + ".xml" if not force: check_file_existence(output_filename) exec_values["output_filename"] = output_filename # create and run the preprocessor preprocessor = WMSPreProcessor(format_selection, **values) result = preprocessor.process(**exec_values) if exec_values.get("generate_metadata", True): if not force: check_file_existence(output_md_filename) encoder = NativeMetadataFormatEncoder() xml = DOMElementToXML(encoder.encodeMetadata(metadata_values["coverage_id"], metadata_values["begin_time"], metadata_values["end_time"], result.footprint_raw)) with open(output_md_filename, "w+") as f: f.write(xml) except Exception, e: # error wrapping if other_values["traceback"]: traceback.print_exc() sys.stderr.write("%s: %s\n" % (type(e).__name__, str(e)))
def ingest_browse_report(parsed_browse_report, do_preprocessing=True, config=None): """ Ingests a browse report. reraise_exceptions if errors shall be handled externally """ try: # get the according browse layer browse_type = parsed_browse_report.browse_type browse_layer = models.BrowseLayer.objects.get(browse_type=browse_type) except models.BrowseLayer.DoesNotExist: raise IngestionException("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) # generate a browse report model browse_report = create_browse_report(parsed_browse_report, browse_layer) # initialize the preprocessor with configuration values crs = None if browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible": crs = "EPSG:3857" elif browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad": crs = "EPSG:4326" logger.debug("Using CRS '%s' ('%s')." % (crs, browse_layer.grid)) # create the required preprocessor/format selection format_selection = get_format_selection("GTiff", **get_format_config(config)) if do_preprocessing and not browse_layer.contains_vertical_curtains \ and not browse_layer.contains_volumes: # add config parameters and custom params params = get_optimization_config(config) # add radiometric interval rad_min = browse_layer.radiometric_interval_min if rad_min is not None: params["radiometric_interval_min"] = rad_min else: rad_min = "min" rad_max = browse_layer.radiometric_interval_max if rad_max is not None: params["radiometric_interval_max"] = rad_max else: rad_max = "max" # add band selection if (browse_layer.r_band is not None and browse_layer.g_band is not None and browse_layer.b_band is not None): bands = [(browse_layer.r_band, rad_min, rad_max), (browse_layer.g_band, rad_min, rad_max), (browse_layer.b_band, rad_min, rad_max)] if params["bandmode"] == RGBA: # RGBA bands.append((0, 0, 0)) params["bands"] = bands preprocessor = NGEOPreProcessor(format_selection, crs=crs, **params) elif browse_layer.contains_vertical_curtains: logger.info("Preparing Vertical Curtain Pre-Processor") params = {} # add radiometric interval rad_min = browse_layer.radiometric_interval_min if rad_min is not None: params["radiometric_interval_min"] = rad_min else: rad_min = "min" rad_max = browse_layer.radiometric_interval_max if rad_max is not None: params["radiometric_interval_max"] = rad_max else: rad_max = "max" preprocessor = VerticalCurtainPreprocessor(**params) elif browse_layer.contains_volumes: preprocessor = VolumePreProcessor() else: preprocessor = None # TODO: CopyPreprocessor report_result = IngestBrowseReportResult() succeded = [] failed = [] timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S%f") browse_dirname = _valid_path("%s_%s_%s_%s" % ( browse_type, browse_report.responsible_org_name, browse_report.date_time.strftime("%Y%m%d%H%M%S%f"), timestamp )) success_dir = join(get_success_dir(config), browse_dirname) failure_dir = join(get_failure_dir(config), browse_dirname) if exists(success_dir): logger.warn("Success directory '%s' already exists.") else: makedirs(success_dir) if exists(failure_dir): logger.warn("Failure directory '%s' already exists.") else: makedirs(failure_dir) # iterate over all browses in the browse report for parsed_browse in parsed_browse_report: # transaction management per browse with transaction.commit_manually(): with transaction.commit_manually(using="mapcache"): try: seed_areas = [] # try ingest a single browse and log success result = ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=config) report_result.add(result) succeded.append(parsed_browse) # commit here to allow seeding transaction.commit() transaction.commit(using="mapcache") logger.info("Commited changes to database.") if not browse_layer.contains_vertical_curtains and not browse_layer.contains_volumes: for minx, miny, maxx, maxy, start_time, end_time in seed_areas: try: # seed MapCache synchronously # TODO: maybe replace this with an async solution seed_mapcache(tileset=browse_layer.id, grid=browse_layer.grid, minx=minx, miny=miny, maxx=maxx, maxy=maxy, minzoom=browse_layer.lowest_map_level, maxzoom=browse_layer.highest_map_level, start_time=start_time, end_time=end_time, delete=False, **get_mapcache_seed_config(config)) logger.info("Successfully finished seeding.") except Exception, e: logger.warn("Seeding failed: %s" % str(e)) elif not browse_layer.contains_volumes: host = "http://localhost/browse/ows" level_0_num_tiles_y = 2 # rows level_0_num_tiles_x = 4 # cols seed_level = range(browse_layer.lowest_map_level, browse_layer.highest_map_level) for tileLevel in seed_level: tiles_x = level_0_num_tiles_x * pow(2, tileLevel); tiles_y = level_0_num_tiles_y * pow(2, tileLevel) #find which tiles are crossed by extent tile_width = 360 / (tiles_x) tile_height = 180 / (tiles_y) coverage = eoxs_models.Coverage.objects.get(identifier=result.identifier) #cycle through tiles for col in range(tiles_x): for row in range(tiles_y): west = -180 + (col * tile_width) east = west + tile_width north = 90 - (row * tile_height) south = north - tile_height if (coverage.footprint.intersects(Polygon.from_bbox( (west,south,east,north) ))): try: # NOTE: The MeshFactory ignores time time = (isoformat(result.time_interval[0]) + "/" + isoformat(result.time_interval[1])) baseurl = host + '?service=W3DS&request=GetTile&version=1.0.0&crs=EPSG:4326&layer={0}&style=default&format=model/gltf'.format(browse_layer.id) url = '{0}&tileLevel={1}&tilecol={2}&tilerow={3}&time={4}'.format(baseurl, tileLevel, col, row, time) logger.info('Seeding call to URL: %s' % (url,)) response = urllib2.urlopen(url) response.close() except Exception, e: logger.warn("Seeding failed: %s" % str(e)) transaction.commit() else: pass