def _run_kwcoco_import(demo):
f_image_archive = demo['archive']
f_spec_file = demo['spec']
return factories.KWCOCOArchiveFactory(
image_archive__file__filename=f_image_archive,
image_archive__file__from_path=datastore.fetch(f_image_archive),
spec_file__file__filename=f_spec_file,
spec_file__file__from_path=datastore.fetch(f_spec_file),
)
def test_subsample_annotaion():
# Test with annotations
factories.KWCOCOArchiveFactory(
image_archive__file__filename='demo_rle.zip',
image_archive__file__from_path=datastore.fetch('demo_rle.zip'),
spec_file__file__filename='demo_rle.kwcoco.json',
spec_file__file__from_path=datastore.fetch('demo_rle.kwcoco.json'),
)
file = ChecksumFile.objects.get(name='000000242287.jpg') # bicycle
image = Image.objects.get(file=file)
a = Annotation.objects.get(image=image.pk) # Should be only one
sub = _create_subsampled(image, 'annotation', {'id': a.pk})
sub.refresh_from_db()
assert sub.processed_image.file
def astro_image():
name = 'astro.png'
image = factories.ImageFactory(
file__file__filename=name,
file__file__from_path=datastore.fetch(name),
)
return image
def elevation():
name = 'Elevation.tif'
image = factories.ImageFactory(
file__file__filename=name,
file__file__from_path=datastore.fetch(name),
)
return image
def test_repopulate_image_entry():
"""Only test with single image file."""
testfile = SampleFiles[0]
imagefile = factories.ImageFactory(
file__file__filename=testfile['name'],
file__file__from_path=datastore.fetch(testfile['name']),
)
# Testing that we can repopulate an image entry
load_image(imagefile.id)
def test_point_cloud_etl(sample_file):
pc_file = factories.PointCloudFactory(
file__file__filename=sample_file,
file__file__from_path=datastore.fetch(sample_file),
)
entry = models.PointCloudMeta.objects.filter(source=pc_file).first()
assert entry.vtp_data is not None
# Testing that we can repopulate a point cloud entry
read_point_cloud_file(pc_file.id)
def _make_raster_from_datastore(name):
image = factories.ImageFactory(
file__file__filename=name,
file__file__from_path=datastore.fetch(name),
)
image_set = factories.ImageSetFactory(images=[image.id], )
raster = factories.RasterFactory(
name=name,
image_set=image_set,
)
return raster
def sample_raster_b():
image = factories.ImageFactory(
file__file__filename='cclc_schu_100.tif',
file__file__from_path=datastore.fetch('cclc_schu_100.tif'),
)
image_set = factories.ImageSetFactory(images=[image.id], )
raster = factories.RasterFactory(
name='cclc_schu_100.tif',
image_set=image_set,
)
return raster.rastermeta
def test_geometry_etl(sample_file):
geom_archive = factories.GeometryArchiveFactory(
file__file__filename=sample_file,
file__file__from_path=datastore.fetch(sample_file),
)
entry = models.Geometry.objects.filter(
geometry_archive=geom_archive).first()
assert entry.data is not None
# Testing that we can repopulate a geometry archive again
read_geometry_archive(geom_archive.id)
# test the field file validator
models.validate_archive(geom_archive.file.file)
def test_cog_image_conversion():
image = factories.ImageFactory(
file__file__filename='20091021202517-01000100-VIS_0001.ntf',
file__file__from_path=datastore.fetch(
'20091021202517-01000100-VIS_0001.ntf'),
)
c = ConvertedImage()
c.source_image = image
c.save()
# Task should complete synchronously
c.refresh_from_db()
assert c.processed_image
def sample_raster_c():
image = factories.ImageFactory(
file__file__filename='RomanColosseum_WV2mulitband_10.tif',
file__file__from_path=datastore.fetch(
'RomanColosseum_WV2mulitband_10.tif'),
)
image_set = factories.ImageSetFactory(images=[image.id], )
raster = factories.RasterFactory(
name='RomanColosseum_WV2mulitband_10.tif',
image_set=image_set,
)
return raster.rastermeta
def sample_raster_a():
image = factories.ImageFactory(
file__file__filename='20091021202517-01000100-VIS_0001.ntf',
file__file__from_path=datastore.fetch(
'20091021202517-01000100-VIS_0001.ntf'),
)
image_set = factories.ImageSetFactory(images=[image.id], )
raster = factories.RasterFactory(
name='20091021202517-01000100-VIS_0001.ntf',
image_set=image_set,
)
return raster.rastermeta
def _get_or_create_checksum_file_datastore(file, name=None):
try:
file_entry = models.ChecksumFile.objects.get(name=file)
_save_signal(file_entry, False)
except models.ChecksumFile.DoesNotExist:
path = datastore.fetch(file)
file_entry = models.ChecksumFile()
if name:
file_entry.name = name
else:
file_entry.name = file
with open(path, 'rb') as f:
file_entry.file.save(os.path.basename(path), f)
file_entry.type = models.FileSourceType.FILE_FIELD
_save_signal(file_entry, True)
return file_entry
def load_spatial_image_sets(image_sets):
for value in image_sets:
image_files, loc_file = value
path = datastore.fetch(loc_file)
# Load JSON image
with open(path, 'r') as f:
geom = shape(json.loads(f.read())['geometry'])
feature = GEOSGeometry(memoryview(dumps(geom)))
# Load image entries
image_ids = load_images(
list(zip([None] * len(image_files), image_files)))
imset, _ = models.get_or_create_image_set(image_ids)
# Make an ImageSetSpatial
imset_spatial, _ = get_or_create_no_commit(models.ImageSetSpatial,
image_set=imset)
imset_spatial.footprint = feature
imset_spatial.outline = feature.convex_hull
imset_spatial.save()
def sample_raster_multi():
# These test files are dramatically downsampled for rapid testing
landsat_files = [
'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band1.tif',
'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band2.tif',
'LC08_L1TP_034032_20200429_20200509_01_T1_sr_band3.tif',
]
images = [
factories.ImageFactory(
file__file__filename=landsat_files[0],
file__file__from_path=datastore.fetch(f),
) for f in landsat_files
]
image_set = factories.ImageSetFactory(images=images, )
# Create a Raster from the three band image entries
raster = factories.RasterFactory(
name='Multi File Test',
image_set=image_set,
)
return raster.rastermeta
def file_path():
return datastore.fetch(FILENAME)
def geotiff_image_entry():
image = factories.ImageFactory(
file__file__filename='paris_france_10.tiff',
file__file__from_path=datastore.fetch('paris_france_10.tiff'),
)
return image