def test_file_exists(self):
path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt')
s3_path = 's3://{}/xxx/lorem.txt'.format(self.bucket_name)
s3_path_prefix = 's3://{}/xxx/lorem'.format(self.bucket_name)
s3_directory = 's3://{}/xxx/'.format(self.bucket_name)
make_dir(path, check_empty=False, use_dirname=True)
str_to_file(self.lorem, path)
upload_or_copy(path, s3_path)
self.assertTrue(file_exists(s3_directory, include_dir=True))
self.assertTrue(file_exists(s3_path, include_dir=False))
self.assertFalse(file_exists(s3_path_prefix, include_dir=True))
self.assertFalse(file_exists(s3_directory, include_dir=False))
self.assertFalse(
file_exists(s3_directory + 'NOTPOSSIBLE', include_dir=False))
def __init__(self,
uri,
extent,
crs_transformer,
tmp_dir,
vector_output=None,
class_config=None):
"""Constructor.
Args:
uri: (str) URI of GeoTIFF file used for storing predictions as RGB values
extent: (Box) The extent of the scene
crs_transformer: (CRSTransformer)
tmp_dir: (str) temp directory to use
vector_output: (None or array of dicts) containing vectorifiction
configuration information
class_config: (ClassConfig) with color values used to convert
class ids to RGB value
"""
self.uri = uri
self.vector_output = vector_output
self.extent = extent
self.crs_transformer = crs_transformer
self.tmp_dir = tmp_dir
# Note: can't name this class_transformer due to Python using that attribute
if class_config:
self.class_trans = SegmentationClassTransformer(class_config)
else:
self.class_trans = None
self.source = None
if file_exists(uri):
self.source = RasterioSourceConfig(uris=[uri]).build(tmp_dir)
def test_file_exists_local_true(self):
path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt')
directory = os.path.dirname(path)
make_dir(directory, check_empty=False)
str_to_file(self.lorem, path)
self.assertTrue(file_exists(path))
def test_file_exists_s3_true(self):
path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt')
directory = os.path.dirname(path)
make_dir(directory, check_empty=False)
str_to_file(self.lorem, path)
s3_path = 's3://{}/lorem.txt'.format(self.bucket_name)
upload_or_copy(path, s3_path)
self.assertTrue(file_exists(s3_path))
def test_copy_from_http(self):
http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
'17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
expected = os.path.join(
self.tmp_dir.name, 'http', 'raw.githubusercontent.com',
'tensorflow/models',
'17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
download_if_needed(http_path, self.tmp_dir.name)
self.assertTrue(file_exists(expected))
os.remove(expected)
def test_file_exists_s3_false(self):
s3_path = 's3://{}/hello.txt'.format(self.bucket_name)
self.assertFalse(file_exists(s3_path))
def test_file_exists_local_false(self):
path = os.path.join(self.tmp_dir.name, 'hello', 'hello.txt')
directory = os.path.dirname(path)
make_dir(directory, check_empty=False)
self.assertFalse(file_exists(path))
def test_file_exists_http_false(self):
http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
'17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/XXX')
self.assertFalse(file_exists(http_path))
def save_image_crop(image_uri,
image_crop_uri,
label_uri=None,
label_crop_uri=None,
size=600,
min_features=10,
vector_labels=True,
class_config=None):
"""Save a crop of an image to use for testing.
If label_uri is set, the crop needs to cover >= min_features.
Args:
image_uri: URI of original image
image_crop_uri: URI of cropped image to save
label_uri: optional URI of label file
label_crop_uri: optional URI of cropped labels to save
size: height and width of crop
Raises:
ValueError if cannot find a crop satisfying min_features constraint.
"""
if not file_exists(image_crop_uri):
print('Saving test crop to {}...'.format(image_crop_uri))
old_environ = os.environ.copy()
try:
request_payer = S3FileSystem.get_request_payer()
if request_payer == 'requester':
os.environ['AWS_REQUEST_PAYER'] = request_payer
im_dataset = rasterio.open(image_uri)
h, w = im_dataset.height, im_dataset.width
extent = Box(0, 0, h, w)
windows = extent.get_windows(size, size)
if label_uri and vector_labels:
crs_transformer = RasterioCRSTransformer.from_dataset(
im_dataset)
geojson_vs_config = GeoJSONVectorSourceConfig(uri=label_uri)
vs = geojson_vs_config.build(class_config, crs_transformer)
geojson = vs.get_geojson()
geoms = []
for f in geojson['features']:
g = shape(f['geometry'])
geoms.append(g)
tree = STRtree(geoms)
def p2m(x, y, z=None):
return crs_transformer.pixel_to_map((x, y))
for w in windows:
use_window = True
if label_uri and vector_labels:
w_polys = tree.query(w.to_shapely())
use_window = len(w_polys) >= min_features
if use_window and label_crop_uri is not None:
print('Saving test crop labels to {}...'.format(
label_crop_uri))
label_crop_features = [
mapping(transform(p2m, wp)) for wp in w_polys
]
label_crop_json = {
'type':
'FeatureCollection',
'features': [{
'geometry': f
} for f in label_crop_features]
}
json_to_file(label_crop_json, label_crop_uri)
if use_window:
crop_image(image_uri, w, image_crop_uri)
if not vector_labels and label_uri and label_crop_uri:
crop_image(label_uri, w, label_crop_uri)
break
if not use_window:
raise ValueError('Could not find a good crop.')
finally:
os.environ.clear()
os.environ.update(old_environ)