def ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=None): """ Ingests a single browse report, performs the preprocessing of the data file and adds the generated browse model to the browse report model. Returns a boolean value, indicating whether or not the browse has been inserted or replaced a previous browse entry. """ # TODO: if curtain: check that layer allows curtains # TODO: same for volumes logger.info("Ingesting browse '%s'." % (parsed_browse.browse_identifier or "<<no ID>>")) replaced = False replaced_extent = None replaced_filename = None merge_with = None merge_footprint = None config = config or get_ngeo_config() coverage_id = parsed_browse.browse_identifier if not coverage_id: # no identifier given, generate a new one coverage_id = _generate_coverage_id(parsed_browse, browse_layer) logger.info("No browse identifier given, generating coverage ID '%s'." % coverage_id) else: try: models.NCNameValidator(coverage_id) except ValidationError: # given ID is not valid, generate a new identifier old_id = coverage_id coverage_id = _generate_coverage_id(parsed_browse, browse_layer) logger.info("Browse ID '%s' is not a valid coverage ID. Using " "generated ID '%s'." % (old_id, coverage_id)) # get the `leave_original` setting leave_original = False try: leave_original = config.getboolean("control.ingest", "leave_original") except: pass # get the input and output filenames storage_path = get_storage_path() input_filename = abspath(get_storage_path(parsed_browse.file_name, config=config)) # check that the input filename is valid -> somewhere under the storage dir if commonprefix((input_filename, storage_path)) != storage_path: raise IngestionException("Input path '%s' points to an invalid " "location." % parsed_browse.file_name) try: models.FileNameValidator(input_filename) except ValidationError, e: raise IngestionException("%s" % str(e), "ValidationError")
config=config, ) except: # save exception info to re-raise it exc_info = sys.exc_info() logger.error( "Error during ingestion of Browse '%s'. Moving " "original image to `failure_dir` '%s'." % (parsed_browse.browse_identifier, failure_dir) ) # move the file to failure folder try: if not leave_original: storage_dir = get_storage_path() relative = relpath(input_filename, storage_dir) dst_dirname = join(failure_dir, dirname(relative)) safe_makedirs(dst_dirname) shutil.move(input_filename, dst_dirname) except Exception, e: logger.warn( "Could not move '%s' to configured " "`failure_dir` '%s'. Error was: '%s'." % (input_filename, failure_dir, str(e)) ) # re-raise the exception raise exc_info[0], exc_info[1], exc_info[2] else: # move the file to success folder, or delete it right away
def ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs, success_dir, failure_dir, seed_areas, config=None): """ Ingests a single browse report, performs the preprocessing of the data file and adds the generated browse model to the browse report model. Returns a boolean value, indicating whether or not the browse has been inserted or replaced a previous browse entry. """ logger.info("Ingesting browse '%s'." % (parsed_browse.browse_identifier or "<<no ID>>")) replaced = False replaced_extent = None replaced_filename = None merge_with = None merge_footprint = None config = config or get_ngeo_config() coverage_id = parsed_browse.browse_identifier if not coverage_id: # no identifier given, generate a new one coverage_id = _generate_coverage_id(parsed_browse, browse_layer) logger.info("No browse identifier given, generating coverage ID '%s'." % coverage_id) else: coverage_id = browse_layer.id + "_" + coverage_id try: NCNameValidator(coverage_id) except ValidationError: # given ID is not valid, generate a new identifier old_id = coverage_id coverage_id = _generate_coverage_id(parsed_browse, browse_layer) logger.info("Browse ID '%s' is not a valid coverage ID. Using " "generated ID '%s'." % (old_id, coverage_id)) # get the `leave_original` setting leave_original = False try: leave_original = config.getboolean("control.ingest", "leave_original") except: pass # get the input and output filenames storage_path = get_storage_path() # if file_name is a URL download browse first and store it locally validate = URLValidator() try: validate(parsed_browse.file_name) input_filename = abspath(get_storage_path( basename(parsed_browse.file_name), config=config)) logger.info("URL given, downloading browse image from '%s' to '%s'." % (parsed_browse.file_name, input_filename)) if not exists(input_filename): try: remote_browse = urlopen(parsed_browse.file_name) with open(input_filename, "wb") as local_browse: local_browse.write(remote_browse.read()) except HTTPError, e: raise IngestionException("HTTP error downloading '%s': %s" % (parsed_browse.file_name, e.code)) except URLError, e: raise IngestionException("URL error downloading '%s': %s" % (parsed_browse.file_name, e.reason))
try: remote_browse = urlopen(parsed_browse.file_name) with open(input_filename, "wb") as local_browse: local_browse.write(remote_browse.read()) except HTTPError, e: raise IngestionException("HTTP error downloading '%s': %s" % (parsed_browse.file_name, e.code)) except URLError, e: raise IngestionException("URL error downloading '%s': %s" % (parsed_browse.file_name, e.reason)) else: raise IngestionException("File do download already exists locally " "as '%s'" % input_filename) except ValidationError: input_filename = abspath(get_storage_path(parsed_browse.file_name, config=config)) logger.info("Filename given, using local browse image '%s'." % input_filename) # check that the input filename is valid -> somewhere under the storage dir if commonprefix((input_filename, storage_path)) != storage_path: raise IngestionException("Input path '%s' points to an invalid " "location." % parsed_browse.file_name) try: models.FileNameValidator(input_filename) except ValidationError, e: raise IngestionException("%s" % str(e), "ValidationError") # Get filename to store preprocessed image output_filename = "%s_%s" % ( uuid.uuid4().hex, basename(parsed_browse.file_name)