def _get_or_create_file_model(model, file, name=None):
# For models that point to a `ChecksumFile`
file_entry = _get_or_create_checksum_file(file, name=name)
# No commit in case we need to skip the signal
entry, created = get_or_create_no_commit(model, file=file_entry)
_save_signal(entry, created)
return entry
def load_kwcoco_archives(archives):
ids = []
for fspec, farch in archives:
spec = _get_or_create_checksum_file(fspec)
arch = _get_or_create_checksum_file(farch)
ds, created = get_or_create_no_commit(models.KWCOCOArchive,
spec_file=spec,
image_archive=arch)
_save_signal(ds, created)
ids.append(ds.id)
return ids
def load_raster(pks, raster_dict, footprint=False):
if not isinstance(pks, (list, tuple)):
pks = [
pks,
]
imset, _ = models.get_or_create_image_set(pks)
# Make raster of that image set
raster, created = get_or_create_no_commit(models.Raster, image_set=imset)
_save_signal(raster, created)
# Add optional metadata
date = raster_dict.get('acquisition_date', None)
cloud_cover = raster_dict.get('cloud_cover', None)
name = raster_dict.get('name', None)
ancillary = raster_dict.get('ancillary_files', None)
instrumentation = raster_dict.get('instrumentation', None)
if date:
adt = dateutil.parser.isoparser().isoparse(date)
try:
raster.rastermeta.acquisition_date = make_aware(adt)
except ValueError:
raster.rastermeta.acquisition_date = adt
raster.rastermeta.save(update_fields=[
'acquisition_date',
])
if cloud_cover:
raster.rastermeta.cloud_cover = cloud_cover
raster.rastermeta.save(update_fields=[
'cloud_cover',
])
if name:
raster.name = name
raster.save(update_fields=[
'name',
])
if ancillary:
[
raster.ancillary_files.add(
_get_or_create_checksum_file_url(af, name=path))
for path, af in ancillary
]
if instrumentation:
raster.rastermeta.instrumentation = instrumentation
raster.rastermeta.save(update_fields=[
'instrumentation',
])
if footprint:
etl.populate_raster_footprint(raster.id)
return raster
def read_fmv_file(fmv_file_id):
fmv_file = FMV.objects.get(id=fmv_file_id)
fmv_file.skip_signal = True
validation = True # TODO: use `fmv_file.file.validate()`
# Only extract the KLV data if it does not exist or the checksum of the video has changed
if not fmv_file.klv_file or not validation:
_extract_klv(fmv_file)
if not fmv_file.web_video_file or not validation:
_convert_video_to_mp4(fmv_file)
# create a model entry for that shapefile
entry, created = get_or_create_no_commit(FMVMeta, fmv_file=fmv_file)
_populate_fmv_entry(entry)
def load_spatial_image_sets(image_sets):
for value in image_sets:
image_files, loc_file = value
path = datastore.fetch(loc_file)
# Load JSON image
with open(path, 'r') as f:
geom = shape(json.loads(f.read())['geometry'])
feature = GEOSGeometry(memoryview(dumps(geom)))
# Load image entries
image_ids = load_images(
list(zip([None] * len(image_files), image_files)))
imset, _ = models.get_or_create_image_set(image_ids)
# Make an ImageSetSpatial
imset_spatial, _ = get_or_create_no_commit(models.ImageSetSpatial,
image_set=imset)
imset_spatial.footprint = feature
imset_spatial.outline = feature.convex_hull
imset_spatial.save()
def populate_raster(raster):
"""Autopopulate the fields of the raster."""
if not isinstance(raster, Raster):
raster = Raster.objects.get(id=raster)
# Has potential to error with failure reason
meta = _validate_image_set_is_raster(raster.image_set)
if not raster.name:
raster.name = raster.image_set.name
raster.save(update_fields=[
'name',
])
raster_meta, created = get_or_create_no_commit(RasterMeta,
parent_raster=raster)
# Not using `defaults` here because we want `meta` to always get updated.
for k, v in meta.items():
# Yeah. This is sketchy, but it works.
setattr(raster_meta, k, v)
raster_meta.save()
return True
def read_point_cloud_file(pc_file):
"""Read a PointCloud object and create a new PointCloudMeta."""
if not isinstance(pc_file, PointCloud):
pc_file = PointCloud.objects.get(id=pc_file)
pc_entry, _ = get_or_create_no_commit(PointCloudMeta, source=pc_file)
pc_entry.name = pc_file.file.name
# Parse the point cloud file format and convert
ext = pc_file.file.name.split('.')[-1].strip().lower()
try:
method = _get_readers()[ext]
except KeyError:
raise ValueError(f'Extension `{ext}` unsupported for point cloud conversion.')
try:
pc_entry.vtp_data
except ChecksumFile.DoesNotExist:
pc_entry.vtp_data = ChecksumFile()
if not pc_entry.vtp_data.collection:
pc_entry.vtp_data.collection = pc_file.file.collection
_file_conversion_helper(pc_file.file, pc_entry.vtp_data.file, method, extension='.vtp')
# Save the record
pc_entry.save()
def load_image(image):
"""Image ingestion routine.
This helper will open an image file from ``Image`` and create a
``ImageMeta`` and collection of ``BandMeta`` entries.
"""
# Fetch the image file this Layer corresponds to
if not isinstance(image, Image):
image = Image.objects.get(id=image)
with image.file.yield_local_path(vsi=True) as file_path:
image_meta, created = get_or_create_no_commit(ImageMeta,
parent_image=image)
if not created:
# Clear out associated entries because they could be invalid
BandMeta.objects.filter(parent_image=image).delete()
ConvertedImage.objects.filter(source_image=image).delete()
_populate_image_meta_models(image_meta, file_path)
return image_meta
def read_geometry_archive(archive_id):
"""Read an archive of a single shapefile (and associated) files.
This will load zipped archives of shape files and create entries
for a single shapefile (basename of files).
A single shapefile will consist of a collection of one or many features
of varying types. We produce a single ``GeometryCollection`` of those
data. Hence, we associate a single shapefile with a single
``GeometryCollection``.
We may need to do more checks/validation to make sure the user only
added a single shape file or provide a more explicit upload interface
where they upload the ``shp``, ``dbf``, etc. files individually and
we assert that they match.
"""
archive = GeometryArchive.objects.get(id=archive_id)
# TODO: add a setting like this:
workdir = getattr(settings, 'GEODATA_WORKDIR', None)
with tempfile.TemporaryDirectory(dir=workdir) as tmpdir:
with archive.file.yield_local_path() as archive_path:
logger.info(f'The geometry archive: {archive_path}')
# Unzip the contents to the working dir
with zipfile.ZipFile(archive_path, 'r') as zip_ref:
zip_ref.extractall(tmpdir)
msg = 'There must be one and only one shapefile in the archive. Found ({})'
shape_files = glob(os.path.join(tmpdir, '*.shp'))
if len(shape_files) > 1:
raise ValidationError(msg.format(len(shape_files)))
elif len(shape_files) == 0:
shape_files = glob(os.path.join(tmpdir, '**/*.shp'))
if len(shape_files) != 1:
raise ValidationError(msg.format(len(shape_files)))
shape_file = shape_files[0]
# load each shapefile using fiona
with fiona.open(shape_file) as shapes:
geometry_entry, created = get_or_create_no_commit(
Geometry,
defaults=dict(name=archive.file.name),
geometry_archive=archive)
shapes.meta # TODO: dump this JSON into the model entry
crs_wkt = shapes.meta['crs_wkt']
logger.info(f'Geometry crs_wkt: {crs_wkt}')
spatial_ref = SpatialReference(crs_wkt)
logger.info(f'Geometry SRID: {spatial_ref.srid}')
collection = []
for item in shapes:
geom = shape(item['geometry']) # not optimal?
# TODO: check this
collection.append(
transform_geometry(
GEOSGeometry(memoryview(
dumps(geom, srid=spatial_ref.srid)),
srid=spatial_ref.srid),
crs_wkt,
))
geometry_entry.data = GeometryCollection(*collection)
geometry_entry.footprint = geometry_entry.data.convex_hull
bounds = geometry_entry.footprint.extent # (xmin, ymin, xmax, ymax)
coords = [
(bounds[0], bounds[3]), # (xmin, ymax)
(bounds[2], bounds[3]), # (xmax, ymax)
(bounds[2], bounds[1]), # (xmax, ymin)
(bounds[0], bounds[1]), # (xmin, ymin)
(bounds[0], bounds[3]), # Close the loop
]
geometry_entry.outline = Polygon(coords)
geometry_entry.save()
return True
def create(self, data):
item = pystac.Item.from_dict(data)
image_ids, ancillary = [], []
for name in item.assets:
asset = item.assets[name]
checksum_file, _ = ChecksumFile.objects.get_or_create(
type=FileSourceType.URL,
url=asset.href,
)
if len(item.assets) == 1 or (asset.roles
and 'data' in asset.roles):
image, _ = models.Image.objects.get_or_create(
file=checksum_file)
image_ids.append(image.pk)
for eo_band in item.ext.eo.bands:
if eo_band.name.startswith(
'B') and eo_band.name[1:].isdigit():
eo_band_number = int(eo_band.name[1:])
else:
eo_band_number = 0 # TODO: confirm reasonable default here
if eo_band.common_name in BAND_RANGE_BY_COMMON_NAMES:
eo_band_spectral_lower, eo_band_spectral_upper = BAND_RANGE_BY_COMMON_NAMES[
eo_band.common_name]
elif eo_band.center_wavelength and eo_band.full_width_half_max:
eo_band_spectral_lower = (
eo_band.eo_band_spectral_upper -
eo_band.full_width_half_max / 2)
eo_band_spectral_upper = (
eo_band.center_wavelength +
eo_band.full_width_half_max / 2)
bandmeta = models.BandMeta.objects.get_or_create(
parent_image=image,
band_number=eo_band_number,
)
bandmeta.description = eo_band.description
bandmeta.band_range = (
Decimal(eo_band_spectral_lower),
Decimal(eo_band_spectral_upper),
)
bandmeta.save()
else:
ancillary.append(checksum_file)
image_set, _ = models.get_or_create_image_set(
image_ids, defaults=dict(name=item.id))
raster, raster_created = get_or_create_no_commit(
models.Raster, image_set=image_set, defaults=dict(name=item.id))
raster.skip_signal = True
raster.save()
[raster.ancillary_files.add(af) for af in ancillary]
raster.save()
outline = Polygon((
[item.bbox[0], item.bbox[1]],
[item.bbox[0], item.bbox[3]],
[item.bbox[2], item.bbox[3]],
[item.bbox[2], item.bbox[1]],
[item.bbox[0], item.bbox[1]],
))
raster_meta = dict(
footprint=json.dumps(item.geometry),
crs=f'+init=epsg:{item.ext.projection.epsg}',
cloud_cover=item.ext.eo.cloud_cover,
transform=item.ext.projection.transform,
extent=item.bbox,
origin=(item.bbox[0], item.bbox[1]),
resolution=(0, 0), # TODO: fix
outline=outline,
acquisition_date=dateutil.parser.isoparser().isoparse(
item.properties['datetime']),
instrumentation=item.properties['platform'],
)
if raster_created:
instance = models.RasterMeta(**raster_meta)
instance.parent_raster = raster
else:
models.RasterMeta.objects.filter(parent_raster=raster).update(
**raster_meta)
instance = models.RasterMeta.objects.get(parent_raster=raster)
instance.save()
return instance
def convert_images(modeladmin, request, queryset):
for image in queryset.all():
entry, created = get_or_create_no_commit(ConvertedImage,
source_image=image)
entry.save()
