def ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs,
success_dir, failure_dir, seed_areas, config=None):
""" Ingests a single browse report, performs the preprocessing of the data
file and adds the generated browse model to the browse report model. Returns
a boolean value, indicating whether or not the browse has been inserted or
replaced a previous browse entry.
"""
# TODO: if curtain: check that layer allows curtains
# TODO: same for volumes
logger.info("Ingesting browse '%s'."
% (parsed_browse.browse_identifier or "<<no ID>>"))
replaced = False
replaced_extent = None
replaced_filename = None
merge_with = None
merge_footprint = None
config = config or get_ngeo_config()
coverage_id = parsed_browse.browse_identifier
if not coverage_id:
# no identifier given, generate a new one
coverage_id = _generate_coverage_id(parsed_browse, browse_layer)
logger.info("No browse identifier given, generating coverage ID '%s'."
% coverage_id)
else:
try:
models.NCNameValidator(coverage_id)
except ValidationError:
# given ID is not valid, generate a new identifier
old_id = coverage_id
coverage_id = _generate_coverage_id(parsed_browse, browse_layer)
logger.info("Browse ID '%s' is not a valid coverage ID. Using "
"generated ID '%s'." % (old_id, coverage_id))
# get the `leave_original` setting
leave_original = False
try:
leave_original = config.getboolean("control.ingest", "leave_original")
except: pass
# get the input and output filenames
storage_path = get_storage_path()
input_filename = abspath(get_storage_path(parsed_browse.file_name, config=config))
# check that the input filename is valid -> somewhere under the storage dir
if commonprefix((input_filename, storage_path)) != storage_path:
raise IngestionException("Input path '%s' points to an invalid "
"location." % parsed_browse.file_name)
try:
models.FileNameValidator(input_filename)
except ValidationError, e:
raise IngestionException("%s" % str(e), "ValidationError")
config=config,
)
except:
# save exception info to re-raise it
exc_info = sys.exc_info()
logger.error(
"Error during ingestion of Browse '%s'. Moving "
"original image to `failure_dir` '%s'." % (parsed_browse.browse_identifier, failure_dir)
)
# move the file to failure folder
try:
if not leave_original:
storage_dir = get_storage_path()
relative = relpath(input_filename, storage_dir)
dst_dirname = join(failure_dir, dirname(relative))
safe_makedirs(dst_dirname)
shutil.move(input_filename, dst_dirname)
except Exception, e:
logger.warn(
"Could not move '%s' to configured "
"`failure_dir` '%s'. Error was: '%s'." % (input_filename, failure_dir, str(e))
)
# re-raise the exception
raise exc_info[0], exc_info[1], exc_info[2]
else:
# move the file to success folder, or delete it right away
def ingest_browse(parsed_browse, browse_report, browse_layer, preprocessor, crs,
success_dir, failure_dir, seed_areas, config=None):
""" Ingests a single browse report, performs the preprocessing of the data
file and adds the generated browse model to the browse report model. Returns
a boolean value, indicating whether or not the browse has been inserted or
replaced a previous browse entry.
"""
logger.info("Ingesting browse '%s'."
% (parsed_browse.browse_identifier or "<<no ID>>"))
replaced = False
replaced_extent = None
replaced_filename = None
merge_with = None
merge_footprint = None
config = config or get_ngeo_config()
coverage_id = parsed_browse.browse_identifier
if not coverage_id:
# no identifier given, generate a new one
coverage_id = _generate_coverage_id(parsed_browse, browse_layer)
logger.info("No browse identifier given, generating coverage ID '%s'."
% coverage_id)
else:
coverage_id = browse_layer.id + "_" + coverage_id
try:
NCNameValidator(coverage_id)
except ValidationError:
# given ID is not valid, generate a new identifier
old_id = coverage_id
coverage_id = _generate_coverage_id(parsed_browse, browse_layer)
logger.info("Browse ID '%s' is not a valid coverage ID. Using "
"generated ID '%s'." % (old_id, coverage_id))
# get the `leave_original` setting
leave_original = False
try:
leave_original = config.getboolean("control.ingest", "leave_original")
except:
pass
# get the input and output filenames
storage_path = get_storage_path()
# if file_name is a URL download browse first and store it locally
validate = URLValidator()
try:
validate(parsed_browse.file_name)
input_filename = abspath(get_storage_path(
basename(parsed_browse.file_name), config=config))
logger.info("URL given, downloading browse image from '%s' to '%s'."
% (parsed_browse.file_name, input_filename))
if not exists(input_filename):
try:
remote_browse = urlopen(parsed_browse.file_name)
with open(input_filename, "wb") as local_browse:
local_browse.write(remote_browse.read())
except HTTPError, e:
raise IngestionException("HTTP error downloading '%s': %s"
% (parsed_browse.file_name, e.code))
except URLError, e:
raise IngestionException("URL error downloading '%s': %s"
% (parsed_browse.file_name, e.reason))
try:
remote_browse = urlopen(parsed_browse.file_name)
with open(input_filename, "wb") as local_browse:
local_browse.write(remote_browse.read())
except HTTPError, e:
raise IngestionException("HTTP error downloading '%s': %s"
% (parsed_browse.file_name, e.code))
except URLError, e:
raise IngestionException("URL error downloading '%s': %s"
% (parsed_browse.file_name, e.reason))
else:
raise IngestionException("File do download already exists locally "
"as '%s'" % input_filename)
except ValidationError:
input_filename = abspath(get_storage_path(parsed_browse.file_name,
config=config))
logger.info("Filename given, using local browse image '%s'."
% input_filename)
# check that the input filename is valid -> somewhere under the storage dir
if commonprefix((input_filename, storage_path)) != storage_path:
raise IngestionException("Input path '%s' points to an invalid "
"location." % parsed_browse.file_name)
try:
models.FileNameValidator(input_filename)
except ValidationError, e:
raise IngestionException("%s" % str(e), "ValidationError")
# Get filename to store preprocessed image
output_filename = "%s_%s" % (
uuid.uuid4().hex, basename(parsed_browse.file_name)
