# get the input and output filenames storage_path = get_storage_path() input_filename = abspath(get_storage_path(parsed_browse.file_name, config=config)) # check that the input filename is valid -> somewhere under the storage dir if commonprefix((input_filename, storage_path)) != storage_path: raise IngestionException("Input path '%s' points to an invalid " "location." % parsed_browse.file_name) try: models.FileNameValidator(input_filename) except ValidationError, e: raise IngestionException("%s" % str(e), "ValidationError") # Get filename to store preprocessed image output_filename = "%s_%s" % (uuid.uuid4().hex, parsed_browse.file_name) output_filename = _valid_path( get_optimized_path(output_filename, browse_layer.id + "/" + str(parsed_browse.start_time.year), config=config) ) output_filename = preprocessor.generate_filename(output_filename) try: ingest_config = get_ingest_config(config) # check if a browse already exists and delete it in order to replace it existing_browse_model = get_existing_browse(parsed_browse.browse_identifier, coverage_id, browse_layer.id) if existing_browse_model: previous_time = existing_browse_model.browse_report.date_time current_time = browse_report.date_time timedelta = current_time - previous_time # get strategy and merge threshold
# check that the input filename is valid -> somewhere under the storage dir if commonprefix((input_filename, storage_path)) != storage_path: raise IngestionException("Input path '%s' points to an invalid " "location." % parsed_browse.file_name) try: models.FileNameValidator(input_filename) except ValidationError, e: raise IngestionException("%s" % str(e), "ValidationError") # Get filename to store preprocessed image output_filename = "%s_%s" % ( uuid.uuid4().hex, basename(parsed_browse.file_name) ) output_filename = _valid_path( get_optimized_path( output_filename, browse_layer.id + "/" + str(parsed_browse.start_time.year), config=config ) ) output_filename = preprocessor.generate_filename(output_filename) try: ingest_config = get_ingest_config(config) # check if a browse already exists and decide how to deal with it existing_browse_model = get_existing_browse( parsed_browse.browse_identifier, coverage_id, browse_layer.id) if existing_browse_model: previous_time = existing_browse_model.browse_report.date_time current_time = browse_report.date_time timedelta = current_time - previous_time
def import_browse(p, browse, browse_report_model, browse_layer_model, crs, seed_areas, config): filename = browse.file_name coverage_id = splitext(filename)[0] footprint_filename = coverage_id + ".wkb" logger.info("Importing browse with data file '%s' and metadata file '%s'." % (filename, footprint_filename)) replaced = False replaced_filename = None existing_browse_model = get_existing_browse(browse.browse_identifier, coverage_id, browse_layer_model.id) if existing_browse_model: logger.info("Existing browse found, replacing it.") replaced_extent, replaced_filename = remove_browse( existing_browse_model, browse_layer_model, coverage_id, seed_areas, config=config ) replaced = True else: # A browse with that identifier does not exist, so just create a new one logger.info("Creating new browse.") output_filename = get_optimized_path(filename, browse_layer_model.id, config=config) if (exists(output_filename) and ((replaced_filename and not samefile(output_filename, replaced_filename)) or not replaced_filename)): raise ImportException("Output file '%s' already exists and is not to " "be replaced." % output_filename) with FileTransaction((output_filename, replaced_filename)): if not exists(dirname(output_filename)): makedirs(dirname(output_filename)) p.extract_browse_file(filename, output_filename) # TODO: find out num bands and footprint ds = gdal.Open(output_filename) num_bands = ds.RasterCount footprint = p.get_footprint(footprint_filename) extent, time_interval = create_browse( browse, browse_report_model, browse_layer_model, coverage_id, crs, replaced, footprint, num_bands, output_filename, seed_areas, config=config ) if not replaced: return IngestBrowseResult(browse.browse_identifier, extent, time_interval) else: replaced_time_interval = (existing_browse_model.start_time, existing_browse_model.end_time) return IngestBrowseReplaceResult(browse.browse_identifier, extent, time_interval, replaced_extent, replaced_time_interval)
def import_browse(p, browse, browse_report_model, browse_layer_model, crs, seed_areas, config): filename = browse.file_name coverage_id = splitext(filename)[0] footprint_filename = coverage_id + ".wkb" logger.info( "Importing browse with data file '%s' and metadata file '%s'." % (filename, footprint_filename)) replaced = False replaced_filename = None existing_browse_model = get_existing_browse(browse.browse_identifier, coverage_id, browse_layer_model.id) if existing_browse_model: logger.info("Existing browse found, replacing it.") replaced_extent, replaced_filename = remove_browse( existing_browse_model, browse_layer_model, coverage_id, seed_areas, config=config) replaced = True else: # A browse with that identifier does not exist, so just create a new one logger.info("Creating new browse.") output_filename = get_optimized_path(filename, browse_layer_model.id, config=config) if (exists(output_filename) and ((replaced_filename and not samefile(output_filename, replaced_filename)) or not replaced_filename)): raise ImportException("Output file '%s' already exists and is not to " "be replaced." % output_filename) with FileTransaction((output_filename, replaced_filename)): if not exists(dirname(output_filename)): makedirs(dirname(output_filename)) p.extract_browse_file(filename, output_filename) # TODO: find out num bands and footprint ds = gdal.Open(output_filename) num_bands = ds.RasterCount footprint = p.get_footprint(footprint_filename) extent, time_interval = create_browse(browse, browse_report_model, browse_layer_model, coverage_id, crs, replaced, footprint, num_bands, output_filename, seed_areas, config=config) if not replaced: return IngestBrowseResult(browse.browse_identifier, extent, time_interval) else: replaced_time_interval = (existing_browse_model.start_time, existing_browse_model.end_time) return IngestBrowseReplaceResult(browse.browse_identifier, extent, time_interval, replaced_extent, replaced_time_interval)