def ingest_browse_report(parsed_browse_report, do_preprocessing=True, config=None): """ Ingests a browse report. reraise_exceptions if errors shall be handled externally """ # initialize the EOxServer system/registry/configuration System.init() try: # get the according browse layer browse_type = parsed_browse_report.browse_type browse_layer = models.BrowseLayer.objects.get(browse_type=browse_type) except models.BrowseLayer.DoesNotExist: logger.warn("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) raise IngestionException("Browse layer with browse type '%s' does not " "exist." % parsed_browse_report.browse_type) # generate a browse report model browse_report = create_browse_report(parsed_browse_report, browse_layer) # initialize the preprocessor with configuration values crs = None if browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible": crs = "EPSG:3857" elif browse_layer.grid == "urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad": crs = "EPSG:4326" logger.debug("Using CRS '%s' ('%s')." % (crs, browse_layer.grid)) # create the required preprocessor/format selection format_selection = get_format_selection("GTiff", **get_format_config(config)) if do_preprocessing: # add config parameters and custom params params = get_optimization_config(config) # add radiometric interval rad_min = browse_layer.radiometric_interval_min if rad_min is not None: params["radiometric_interval_min"] = rad_min else: rad_min = "min" rad_max = browse_layer.radiometric_interval_max if rad_max is not None: params["radiometric_interval_max"] = rad_max else: rad_max = "max" # add band selection if (browse_layer.r_band is not None and browse_layer.g_band is not None and browse_layer.b_band is not None): bands = [(browse_layer.r_band, rad_min, rad_max), (browse_layer.g_band, rad_min, rad_max), (browse_layer.b_band, rad_min, rad_max)] if params["bandmode"] == RGBA: # RGBA bands.append((0, 0, 0)) params["bands"] = bands preprocessor = NGEOPreProcessor(format_selection, crs=crs, **params) else: preprocessor = None # TODO: CopyPreprocessor report_result = IngestBrowseReportResult() succeded = [] failed = [] timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S%f") browse_dirname = _valid_path("%s_%s_%s_%s" % ( browse_type, browse_report.responsible_org_name, browse_report.date_time.strftime("%Y%m%d%H%M%S%f"), timestamp )) success_dir = join(get_success_dir(config), browse_dirname) failure_dir = join(get_failure_dir(config), browse_dirname) if exists(success_dir): logger.warn("Success directory '%s' already exists.") else: makedirs(success_dir) if exists(failure_dir): logger.warn("Failure directory '%s' already exists.") else: makedirs(failure_dir) # iterate over all browses in the browse report for parsed_browse in parsed_browse_report: # transaction management per browse with transaction.commit_manually(): with transaction.commit_manually(using="mapcache"): try: seed_areas = [] # try ingest a single browse and log success result = ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=config) report_result.add(result) succeded.append(parsed_browse) # commit here to allow seeding transaction.commit() transaction.commit(using="mapcache") logger.info("Committed changes to database.") for minx, miny, maxx, maxy, start_time, end_time in seed_areas: try: # seed MapCache synchronously # TODO: maybe replace this with an async solution seed_mapcache(tileset=browse_layer.id, grid=browse_layer.grid, minx=minx, miny=miny, maxx=maxx, maxy=maxy, minzoom=browse_layer.lowest_map_level, maxzoom=browse_layer.highest_map_level, start_time=start_time, end_time=end_time, delete=False, **get_mapcache_seed_config(config)) logger.info("Successfully finished seeding.") except Exception, e: logger.warn("Seeding failed: %s" % str(e)) # log ingestions for report generation # date/browseType/browseLayerId/start/end report_logger.info("/\\/\\".join(( datetime.utcnow().isoformat("T") + "Z", parsed_browse_report.browse_type, browse_layer.id, (parsed_browse.start_time.replace(tzinfo=None)-parsed_browse.start_time.utcoffset()).isoformat("T") + "Z", (parsed_browse.end_time.replace(tzinfo=None)-parsed_browse.end_time.utcoffset()).isoformat("T") + "Z" ))) except Exception, e: # report error logger.error("Failure during ingestion of browse '%s'." % parsed_browse.browse_identifier) logger.error("Exception was '%s': %s" % (type(e).__name__, str(e))) logger.debug(traceback.format_exc() + "\n") # undo latest changes, append the failure and continue report_result.add(IngestBrowseFailureResult( parsed_browse.browse_identifier, getattr(e, "code", None) or type(e).__name__, str(e)) ) failed.append(parsed_browse) transaction.rollback() transaction.rollback(using="mapcache")
def import_browse_report(p, browse_report_file, browse_layer_model, crs, seed_cache_levels, import_cache_levels, config): """ """ seed_areas = [] report_result = IngestBrowseReportResult() browse_report = decode_browse_report(etree.parse(browse_report_file)) browse_report_model = create_browse_report(browse_report, browse_layer_model) for browse in browse_report: with transaction.commit_manually(): with transaction.commit_manually(using="mapcache"): try: result = import_browse(p, browse, browse_report_model, browse_layer_model, crs, seed_areas, config) report_result.add(result) transaction.commit() transaction.commit(using="mapcache") except Exception, e: logger.error("Failure during import of browse '%s'." % browse.browse_identifier) logger.debug(traceback.format_exc() + "\n") transaction.rollback() transaction.rollback(using="mapcache") report_result.add( IngestBrowseFailureResult(browse.browse_identifier, type(e).__name__, str(e))) continue tileset_name = browse_layer_model.id dim = isotime(browse.start_time) + "/" + isotime(browse.end_time) ts = tileset.open(get_tileset_path(browse_layer_model.browse_type, config), mode="w") grid = URN_TO_GRID[browse_layer_model.grid] tile_num = 0 # import cache for minzoom, maxzoom in import_cache_levels: logger.info("Importing cached tiles from zoom level %d to %d." % (minzoom, maxzoom)) for x, y, z, f in p.get_cache_files(tileset_name, grid, dim): if z < minzoom or z > maxzoom: continue ts.add_tile(tileset_name, grid, dim, x, y, z, f) tile_num += 1 logger.info("Imported %d cached tiles." % tile_num) # seed cache for minzoom, maxzoom in seed_cache_levels: logger.info("Re-seeding tile cache from zoom level %d to %d." % (minzoom, maxzoom)) seed_mapcache(tileset=browse_layer_model.id, grid=browse_layer_model.grid, minx=result.extent[0], miny=result.extent[1], maxx=result.extent[2], maxy=result.extent[3], minzoom=minzoom, maxzoom=maxzoom, start_time=result.time_interval[0], end_time=result.time_interval[1], delete=False, **get_mapcache_seed_config(config)) logger.info("Successfully finished seeding.")