Ejemplo n.º 1
0
    def _handle(self, browse_layer_id, browse_type):
        from ngeo_browse_server.control.queries import remove_browse

        # query the browse layer
        if browse_layer_id:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    id=browse_layer_id)
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer '%s' does not exist" %
                             browse_layer_id)
                raise CommandError("Browse layer '%s' does not exist" %
                                   browse_layer_id)
        else:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    browse_type=browse_type)
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer with browse type '%s' does "
                             "not exist" % browse_type)
                raise CommandError("Browse layer with browse type '%s' does "
                                   "not exist" % browse_type)

        # get all browses of browse layer
        browses_qs = Browse.objects.all().filter(
            browse_layer=browse_layer_model)

        paths_to_delete = []
        seed_areas = []

        with transaction.commit_on_success():
            with transaction.commit_on_success(using="mapcache"):
                logger.info("Deleting '%d' browse%s from database." %
                            (browses_qs.count(),
                             "s" if browses_qs.count() > 1 else ""))
                # go through all browses to be deleted
                for browse_model in browses_qs:
                    _, filename = remove_browse(browse_model,
                                                browse_layer_model,
                                                browse_model.coverage_id,
                                                seed_areas,
                                                unseed=False)

                    paths_to_delete.append(filename)

        # loop through optimized browse images and delete them
        # This is done at this point to make sure a rollback is possible
        # if there is an error while deleting the browses and coverages
        for file_path in paths_to_delete:
            if exists(file_path):
                remove(file_path)
                logger.info("Optimized browse image deleted: %s" % file_path)
            else:
                logger.warning(
                    "Optimized browse image to be deleted not found "
                    "in path: %s" % file_path)

        delete_browse_layer(browse_layer_model, purge=True)
Ejemplo n.º 2
0
    def _handle(self, browse_layer_id, browse_type):
        from ngeo_browse_server.control.queries import remove_browse

        # query the browse layer
        if browse_layer_id:
            try:
                browse_layer_model = BrowseLayer.objects.get(id=browse_layer_id)
            except BrowseLayer.DoesNotExist:
                logger.error(
                    "Browse layer '%s' does not exist" % browse_layer_id
                )
                raise CommandError(
                    "Browse layer '%s' does not exist" % browse_layer_id
                )
        else:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    browse_type=browse_type
                )
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer with browse type '%s' does "
                             "not exist" % browse_type)
                raise CommandError("Browse layer with browse type '%s' does "
                                   "not exist" % browse_type)

        # get all browses of browse layer
        browses_qs = Browse.objects.all().filter(browse_layer=browse_layer_model)

        paths_to_delete = []
        seed_areas = []

        with transaction.commit_on_success():
            with transaction.commit_on_success(using="mapcache"):
                logger.info("Deleting '%d' browse%s from database."
                            % (browses_qs.count(),
                               "s" if browses_qs.count() > 1 else ""))
                # go through all browses to be deleted
                for browse_model in browses_qs:
                    _, filename = remove_browse(
                        browse_model, browse_layer_model,
                        browse_model.coverage_id, seed_areas,
                        unseed=False
                    )

                    paths_to_delete.append(filename)

        # loop through optimized browse images and delete them
        # This is done at this point to make sure a rollback is possible
        # if there is an error while deleting the browses and coverages
        for file_path in paths_to_delete:
            if exists(file_path):
                remove(file_path)
                logger.info("Optimized browse image deleted: %s" % file_path)
            else:
                logger.warning("Optimized browse image to be deleted not found "
                               "in path: %s" % file_path)

        delete_browse_layer(browse_layer_model, purge=True)
Ejemplo n.º 3
0
    def _handle(self, start, end, browse_layer_id, browse_type):
        from ngeo_browse_server.control.queries import remove_browse

        # query the browse layer
        if browse_layer_id:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    id=browse_layer_id
                )
            except BrowseLayer.DoesNotExist:
                logger.error(
                    "Browse layer '%s' does not exist" % browse_layer_id
                )
                raise CommandError(
                    "Browse layer '%s' does not exist" % browse_layer_id
                )
        else:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    browse_type=browse_type
                )
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer with browse type'%s' does "
                             "not exist" % browse_type)
                raise CommandError("Browse layer with browse type'%s' does "
                                   "not exist" % browse_type)

        # get all browses of browse layer
        browses_qs = Browse.objects.all().filter(
            browse_layer=browse_layer_model
        )

        # apply start/end filter
        if start and not end:
            browses_qs = browses_qs.filter(start_time__gte=start)
        elif end and not start:
            browses_qs = browses_qs.filter(end_time__lte=end)
        elif start and end:
            browses_qs = browses_qs.filter(
                start_time__gte=start, end_time__lte=end
            )

        paths_to_delete = []
        seed_areas = []

        with transaction.commit_on_success():
            with transaction.commit_on_success(using="mapcache"):
                logger.info("Deleting '%d' browse%s from database."
                            % (browses_qs.count(),
                               "s" if browses_qs.count() > 1 else ""))
                # go through all browses to be deleted
                for browse_model in browses_qs:

                    _, filename = remove_browse(
                        browse_model, browse_layer_model,
                        browse_model.coverage_id, seed_areas
                    )

                    paths_to_delete.append(filename)

        # loop through optimized browse images and delete them
        # This is done at this point to make sure a rollback is possible
        # if there is an error while deleting the browses and coverages
        for file_path in paths_to_delete:
            if exists(file_path):
                remove(file_path)
                logger.info("Optimized browse image deleted: %s" % file_path)
            else:
                logger.warning("Optimized browse image to be deleted not found"
                               " in path: %s" % file_path)

        # only if either start or end is present browses are left
        if start or end:
            if start:
                if end:
                    seed_areas = [
                        area for area in seed_areas
                        if not (area[4] >= start and area[5] <= end)
                    ]
                else:
                    seed_areas = [
                        area for area in seed_areas if not (area[4] >= start)
                    ]
            else:
                seed_areas = [
                    area for area in seed_areas if not (area[5] <= end)
                ]

            for minx, miny, maxx, maxy, start_time, end_time in seed_areas:
                try:

                    # seed MapCache synchronously
                    # TODO: maybe replace this with an async solution
                    seed_mapcache(tileset=browse_layer_model.id,
                                  grid=browse_layer_model.grid,
                                  minx=minx, miny=miny,
                                  maxx=maxx, maxy=maxy,
                                  minzoom=browse_layer_model.lowest_map_level,
                                  maxzoom=browse_layer_model.highest_map_level,
                                  start_time=start_time,
                                  end_time=end_time,
                                  delete=False,
                                  **get_mapcache_seed_config())
                    logger.info("Successfully finished seeding.")

                except Exception, e:
                    logger.warn("Seeding failed: %s" % str(e))
Ejemplo n.º 4
0
            if strategy == "merge" and timedelta < threshold:

                if previous_time > current_time:
                    # TODO: raise exception?
                    pass

                rect_ds = System.getRegistry().getFromFactory(
                    "resources.coverages.wrappers.EOCoverageFactory", {"obj_id": existing_browse_model.coverage_id}
                )
                merge_footprint = rect_ds.getFootprint()
                merge_with = rect_ds.getData().getLocation().getPath()

                replaced_time_interval = (existing_browse_model.start_time, existing_browse_model.end_time)

                _, _ = remove_browse(existing_browse_model, browse_layer, coverage_id, seed_areas, config=config)
                replaced = False
                logger.debug("Existing browse found, merging it.")
            else:
                # perform replacement

                replaced_time_interval = (existing_browse_model.start_time, existing_browse_model.end_time)

                replaced_extent, replaced_filename = remove_browse(
                    existing_browse_model, browse_layer, coverage_id, seed_areas, config=config
                )
                replaced = True
                logger.info("Existing browse found, replacing it.")

        else:
            # A browse with that identifier does not exist, so just create a new one
Ejemplo n.º 5
0
    def _handle(self, start, end, browse_layer_id, browse_type):
        from ngeo_browse_server.control.queries import remove_browse

        # query the browse layer
        if browse_layer_id:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    id=browse_layer_id)
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer '%s' does not exist" %
                             browse_layer_id)
                raise CommandError("Browse layer '%s' does not exist" %
                                   browse_layer_id)
        else:
            try:
                browse_layer_model = BrowseLayer.objects.get(
                    browse_type=browse_type)
            except BrowseLayer.DoesNotExist:
                logger.error("Browse layer with browse type'%s' does "
                             "not exist" % browse_type)
                raise CommandError("Browse layer with browse type'%s' does "
                                   "not exist" % browse_type)

        # get all browses of browse layer
        browses_qs = Browse.objects.all().filter(
            browse_layer=browse_layer_model)

        # apply start/end filter
        if start and not end:
            browses_qs = browses_qs.filter(start_time__gte=start)
        elif end and not start:
            browses_qs = browses_qs.filter(end_time__lte=end)
        elif start and end:
            browses_qs = browses_qs.filter(start_time__gte=start,
                                           end_time__lte=end)

        paths_to_delete = []
        seed_areas = []

        with transaction.commit_on_success():
            with transaction.commit_on_success(using="mapcache"):
                logger.info("Deleting '%d' browse%s from database." %
                            (browses_qs.count(),
                             "s" if browses_qs.count() > 1 else ""))
                # go through all browses to be deleted
                for browse_model in browses_qs:

                    _, filename = remove_browse(browse_model,
                                                browse_layer_model,
                                                browse_model.coverage_id,
                                                seed_areas)

                    paths_to_delete.append(filename)

        # loop through optimized browse images and delete them
        # This is done at this point to make sure a rollback is possible
        # if there is an error while deleting the browses and coverages
        for file_path in paths_to_delete:
            if exists(file_path):
                remove(file_path)
                logger.info("Optimized browse image deleted: %s" % file_path)
            else:
                logger.warning("Optimized browse image to be deleted not found"
                               " in path: %s" % file_path)

        # only if either start or end is present browses are left
        if start or end:
            if start:
                if end:
                    seed_areas = [
                        area for area in seed_areas
                        if not (area[4] >= start and area[5] <= end)
                    ]
                else:
                    seed_areas = [
                        area for area in seed_areas if not (area[4] >= start)
                    ]
            else:
                seed_areas = [
                    area for area in seed_areas if not (area[5] <= end)
                ]

            for minx, miny, maxx, maxy, start_time, end_time in seed_areas:
                try:

                    # seed MapCache synchronously
                    # TODO: maybe replace this with an async solution
                    seed_mapcache(tileset=browse_layer_model.id,
                                  grid=browse_layer_model.grid,
                                  minx=minx,
                                  miny=miny,
                                  maxx=maxx,
                                  maxy=maxy,
                                  minzoom=browse_layer_model.lowest_map_level,
                                  maxzoom=browse_layer_model.highest_map_level,
                                  start_time=start_time,
                                  end_time=end_time,
                                  delete=False,
                                  **get_mapcache_seed_config())
                    logger.info("Successfully finished seeding.")

                except Exception, e:
                    logger.warn("Seeding failed: %s" % str(e))
Ejemplo n.º 6
0
Archivo: imp.py Proyecto: EOX-A/ngeo-b
def import_browse(p, browse, browse_report_model, browse_layer_model, crs,
                  seed_areas, config):
    filename = browse.file_name
    coverage_id = splitext(filename)[0]
    footprint_filename = coverage_id + ".wkb"

    logger.info("Importing browse with data file '%s' and metadata file '%s'."
                % (filename, footprint_filename))
    replaced = False
    replaced_filename = None

    existing_browse_model = get_existing_browse(browse.browse_identifier, coverage_id, browse_layer_model.id)
    if existing_browse_model:
        logger.info("Existing browse found, replacing it.")

        replaced_extent, replaced_filename = remove_browse(
            existing_browse_model, browse_layer_model, coverage_id, seed_areas,
            config=config
        )
        replaced = True

    else:
        # A browse with that identifier does not exist, so just create a new one
        logger.info("Creating new browse.")

    output_filename = get_optimized_path(filename, browse_layer_model.id, config=config)

    if (exists(output_filename) and
        ((replaced_filename and
          not samefile(output_filename, replaced_filename))
         or not replaced_filename)):
        raise ImportException("Output file '%s' already exists and is not to "
                              "be replaced." % output_filename)

    with FileTransaction((output_filename, replaced_filename)):
        if not exists(dirname(output_filename)):
            makedirs(dirname(output_filename))

        p.extract_browse_file(filename, output_filename)

        # TODO: find out num bands and footprint
        ds = gdal.Open(output_filename)
        num_bands = ds.RasterCount

        footprint = p.get_footprint(footprint_filename)

        extent, time_interval = create_browse(
            browse, browse_report_model, browse_layer_model,
            coverage_id, crs, replaced, footprint, num_bands,
            output_filename, seed_areas, config=config
        )

    if not replaced:
        return IngestBrowseResult(browse.browse_identifier, extent,
                                  time_interval)
    else:
        replaced_time_interval = (existing_browse_model.start_time,
                                  existing_browse_model.end_time)
        return IngestBrowseReplaceResult(browse.browse_identifier,
                                         extent, time_interval, replaced_extent,
                                         replaced_time_interval)
Ejemplo n.º 7
0
                if previous_time > current_time:
                    # TODO: raise exception?
                    pass

                rect_ds = System.getRegistry().getFromFactory(
                    "resources.coverages.wrappers.EOCoverageFactory",
                    {"obj_id": existing_browse_model.coverage_id}
                )
                merge_footprint = rect_ds.getFootprint()
                merge_with = rect_ds.getData().getLocation().getPath()

                replaced_time_interval = (existing_browse_model.start_time,
                                          existing_browse_model.end_time)

                _, _ = remove_browse(
                    existing_browse_model, browse_layer, coverage_id,
                    seed_areas, config=config
                )
                replaced = False
                logger.debug("Existing browse found, merging it.")

            elif strategy == "skip" and current_time <= previous_time:
                logger.debug("Existing browse found and not older, skipping.")
                return IngestBrowseSkipResult(parsed_browse.browse_identifier)

            else:
                # perform replacement

                replaced_time_interval = (existing_browse_model.start_time,
                                          existing_browse_model.end_time)

                replaced_extent, replaced_filename = remove_browse(
Ejemplo n.º 8
0
def import_browse(p, browse, browse_report_model, browse_layer_model, crs,
                  seed_areas, config):
    filename = browse.file_name
    coverage_id = splitext(filename)[0]
    footprint_filename = coverage_id + ".wkb"

    logger.info(
        "Importing browse with data file '%s' and metadata file '%s'." %
        (filename, footprint_filename))
    replaced = False
    replaced_filename = None

    existing_browse_model = get_existing_browse(browse.browse_identifier,
                                                coverage_id,
                                                browse_layer_model.id)
    if existing_browse_model:
        logger.info("Existing browse found, replacing it.")

        replaced_extent, replaced_filename = remove_browse(
            existing_browse_model,
            browse_layer_model,
            coverage_id,
            seed_areas,
            config=config)
        replaced = True

    else:
        # A browse with that identifier does not exist, so just create a new one
        logger.info("Creating new browse.")

    output_filename = get_optimized_path(filename,
                                         browse_layer_model.id,
                                         config=config)

    if (exists(output_filename)
            and ((replaced_filename
                  and not samefile(output_filename, replaced_filename))
                 or not replaced_filename)):
        raise ImportException("Output file '%s' already exists and is not to "
                              "be replaced." % output_filename)

    with FileTransaction((output_filename, replaced_filename)):
        if not exists(dirname(output_filename)):
            makedirs(dirname(output_filename))

        p.extract_browse_file(filename, output_filename)

        # TODO: find out num bands and footprint
        ds = gdal.Open(output_filename)
        num_bands = ds.RasterCount

        footprint = p.get_footprint(footprint_filename)

        extent, time_interval = create_browse(browse,
                                              browse_report_model,
                                              browse_layer_model,
                                              coverage_id,
                                              crs,
                                              replaced,
                                              footprint,
                                              num_bands,
                                              output_filename,
                                              seed_areas,
                                              config=config)

    if not replaced:
        return IngestBrowseResult(browse.browse_identifier, extent,
                                  time_interval)
    else:
        replaced_time_interval = (existing_browse_model.start_time,
                                  existing_browse_model.end_time)
        return IngestBrowseReplaceResult(browse.browse_identifier, extent,
                                         time_interval, replaced_extent,
                                         replaced_time_interval)