def check_images(test, mast):
"""Read in a generic (non-geographic) image, like JPEG, and do a diff
Return diff raster if actually different
Args:
test <str>: path to test image
mast <str>: path to master image
"""
try:
from scipy.misc import imread
except ImportError:
from scipy.ndimage import imread
# read images
try:
test_im = imread(test)
mast_im = imread(mast)
except ImportError:
logger.warning("Likely missing Python Image Library (PIL).")
# try Scikit Image
from skimage.io import imread
try:
mast_im = imread(mast)
test_im = imread(test)
except (ValueError, TypeError, ImportError):
logger.warning("Not able to open image with skimag.io. Likely missing image library.")
return None
# check diff
try:
diff_im = do_diff(test_im, mast_im)
if len(np.nonzero(diff_im)) > 3:
logger.error("Values differ between {0} and {1}.".format(test, mast))
return diff_im
else:
logger.info("Values equivalent between {0} and {1}.".format(test, mast))
return None
except ValueError:
logger.error("Image {0} and {1} are not the same dimensions.".format(test, mast))
def place_order(espa_env: str,
username: str,
outdir: str = None,
order_key: str = None):
"""
Place the order with the appropriate ESPA environment
:param order_key: Optionally specify a keyword pointing to a specific order
:param outdir: Optionally specify full path to the output directory, otherwise os.getcwd() is used
:param espa_env: The name of the ESPA environment
:param username: ESPA username
:return:
"""
espa_url = espa_orders_api.get_espa_env(espa_env)
orders = load_order(order_key)
passwd = espa_orders_api.espa_login()
order_length = len(orders)
response = list()
for i, order in enumerate(orders):
logger.info("Requesting order %d of %d", i + 1, order_length)
r = requests.post(espa_url + api_config.api_urls["order"],
auth=(username, passwd),
json=order)
try:
result = r.json()
if 'orderid' not in result:
logger.error('Order "%s" %d/%d failed: %s', order_key, i + 1,
order_length, result)
continue
response.append(result)
except json.decoder.JSONDecodeError: # inherits from ValueError
# This error seems to occur when trying to decode a None-type object
logger.error(
"There was likely a problem connecting with the host. "
"Check to see if ESPA is down for maintenance.")
filename = order_text(outdir)
logger.warning('Store ordering results: %s', filename)
with open(filename, "a") as f:
f.write(json.dumps(response, indent=4))
def check_jpeg_files(test: list, mast: list, dir_out: str) -> None:
"""
Check JPEG files (i.e., Gverify or preview images) for diffs in file size or file contents. Plot difference
image if applicable
:param test: List of paths to test jpg files
:param mast: List of paths to master jpg files
:param dir_out: Full path to output directory
:return:
"""
test, mast = Cleanup.remove_nonmatching_files(test, mast)
logger.info("Checking JPEG preview/gverify files...")
if mast is None or test is None:
logger.error("No JPEG files to check in test and/or mast "
"directories.")
else:
for i, j in zip(test, mast):
# Compare file sizes
if os.path.getsize(i) != os.path.getsize(j):
logger.warning("JPEG file sizes do not match for "
"Master {0} and Test {1}...\n".format(j, i))
logger.warning("{0} size: {1}".format(
i, os.path.getsize(i)))
logger.warning("{0} size: {1}".format(
j, os.path.getsize(j)))
else:
logger.info("JPEG files {0} and {1} are the same "
"size".format(j, i))
# diff images
result = ArrayImage.check_images(i, j)
if result:
ImWrite.plot_diff_image(test=i,
mast=j,
diff_raster=result,
fn_out=i.split(os.sep)[-1],
fn_type="diff_",
dir_out=dir_out)
def check_text_files(test, mast, ext):
"""Check master and test text-based files (headers, XML, etc.)
line-by-line for differences.
Sort all the lines to attempt to capture new entries.
Args:
test <str>: path to test text file
mast <str>: path to master text file
ext <str>: file extension (should be .txt, .xml or .gtf
"""
logger.info("Checking {0} files...".format(ext))
test, mast = Cleanup.remove_nonmatching_files(test, mast)
# Do some checks to make sure files are worth testing
if mast is None or test is None:
logger.warning("No {0} files to check in test and/or mast "
"directories.".format(ext))
return
if len(mast) != len(test):
logger.error("{0} file lengths differ. Master: {1} | Test:"
" {2}".format(ext, len(mast), len(test)))
return
for i, j in zip(test, mast):
topen = open(i)
mopen = open(j)
# Read text line-by-line from file
file_topen = topen.readlines()
file_mopen = mopen.readlines()
# Close files
topen.close()
mopen.close()
# Check file names for name differences.
# Print non-matching names in details.
# get file names
i_fn = i.split(os.sep)[-1]
j_fn = j.split(os.sep)[-1]
if i_fn != j_fn:
logger.error(
"{0} file names differ. Master: {1} | Test: {2}".format(
ext, j, i))
return
else:
logger.info("{0} file names equivalent. Master: {1} | Test: "
"{2}".format(ext, j, i))
# Check open files line-by-line (sorted) for changes.
# Print non-matching lines in details.
txt_diffs = set(file_topen).difference(set(file_mopen))
if len(txt_diffs) > 0:
for k in txt_diffs:
logger.error("{0} changes: {1}".format(ext, k))
else:
logger.info("No differences between {0} and {1}.".format(i, j))
def load_order(order_key: str = 'original.test1') -> dict:
"""
Load in a pre-constructed order by default if None is specified. Otherwise, load the order from a .yaml file
:param order_key: A string containing the order key, otherwise "original" will be used
:return:
"""
if len(order_key.split('.')) == 1:
order_key += '.*'
file_search = os.path.abspath(
os.path.join(
__location__, 'retrieve_data/espa/orders/{0}/{1}.json'.format(
*order_key.split('.'))))
files = glob.glob(file_search)
if not len(files):
raise IOError('File does not exist: {}'.format(file_search))
try:
return json.load(open(files[0]))
except Exception as exc:
logger.error("Problem reading file: {}".format(files[0]))
raise
def extent_diff_rows(test, mast):
"""Make sure number of rows are the same between GDAL objects.
Args:
test <osgeo.gdal.Dataset>: test raster
mast <osgeo.gdal.Dataset>: master raster
"""
rows_diff = test.RasterYSize - mast.RasterYSize
if rows_diff == 0:
logger.info("Rows match.")
status = True
else:
logger.error("Rows do not match.")
logger.error("Test row: {0}".format(test.RasterYSize))
logger.error("Master row: {0}".format(mast.RasterYSize))
status = False
return status
def compare_geo_trans(test, mast):
"""Make sure geographic transforms are the same between GDAL objects.
Args:
test <osgeo.gdal.Dataset>: test raster
mast <osgeo.gdal.Dataset>: master raster
"""
tst_gt = test.GetGeoTransform()
mst_gt = mast.GetGeoTransform()
gt_diff = (tst_gt == mst_gt)
if gt_diff:
logger.info("Geo transforms match.")
status = True
else:
logger.error("Geo transforms match.")
logger.error("Test transform: {0}".format(tst_gt))
logger.error("Master transform: {0}".format(mst_gt))
status = False
return status
def compare_proj_ref(test, mast):
"""Make sure projections are the same between two GDAL objects.
Args:
test <osgeo.gdal.Dataset>: test raster
mast <osgeo.gdal.Dataset>: master raster
"""
tst_proj = test.GetProjectionRef()
mst_proj = mast.GetProjectionRef()
proj_diff = (tst_proj == mst_proj)
if proj_diff:
logger.info("Projections match.")
status = True
else:
logger.error("Projections do not match.")
logger.error("Test transform: {0}".format(tst_proj))
logger.error("Master transform: {0}".format(mst_proj))
status = False
return status
def check_images(test, mast, dir_out, ext, include_nd=False):
"""Compare the test and master images, both for their raw contents and
geographic parameters. If differences exist, produce diff plot + CSV
stats file.
Args:
test <str>: path to test image
mast <str>: path to master image
dir_out <str>: path to output directory
ext <str>: file extension
include_nd <bool>: incl. nodata values in file cmp (default=False)
"""
logger.warning("Checking {0} files...".format(ext))
# clean up non-matching files
test, mast = Cleanup.remove_nonmatching_files(test, mast)
# make sure there are actually files to check
if mast is None or test is None:
logger.error("No {0} files to check in test and/or mast directories.".format(ext))
return False
# do other comparison checks, return stats + plots if diffs exist
for i, j in zip(test, mast):
logger.info("Checking Test {0} against Master {1}".format(i, j))
# Open each raster
ds_test = RasterIO.open_raster(i)
ds_mast = RasterIO.open_raster(j)
# Compare various raster parameters
status = list()
status.append(RasterCmp.compare_proj_ref(ds_test, ds_mast))
status.append(RasterCmp.compare_geo_trans(ds_test, ds_mast))
status.append(RasterCmp.extent_diff_cols(ds_test, ds_mast))
status.append(RasterCmp.extent_diff_rows(ds_test, ds_mast))
# If any above tests fail, go to next iteration
if any(stat is False for stat in status):
continue
# Count number of sub-bands in the files
d_range = Find.count(i, ds_test, j, ds_mast, ext)
if d_range is None:
logger.critical("Number of files different; data cannot be tested successfully.")
continue
# if sub-bands exist, read them one-by-one and do diffs + stats
if d_range > 1:
for ii in range(0, d_range):
# Get the first band from each raster
if ext == ".img":
logger.info("Reading sub-band {0} from .img {1}...".format(ii, i))
ds_tband = RasterIO.read_band_as_array(ds_test, ii)
ds_mband = RasterIO.read_band_as_array(ds_mast, ii)
else:
logger.info("Reading .hdf/.nc SDS {0} from file {0}...".format(ii, i))
sds_tband = RasterIO.open_raster(RasterIO.get_sds(ds_test)[ii][0])
sds_mband = RasterIO.open_raster(RasterIO.get_sds(ds_mast)[ii][0])
ds_tband, t_nd = RasterIO.read_band_as_array(sds_tband)
ds_mband, m_nd = RasterIO.read_band_as_array(sds_mband)
# do image differencing without masking NoData
if isinstance(t_nd, type(None)) or include_nd:
diff = do_diff(ds_tband, ds_mband)
# do image differencing with NoData masked
else:
diff = do_diff(ds_tband, ds_mband, nodata=int(t_nd))
# call stats functions to write out results/plots/etc.
call_stats(i, j, diff, i, dir_out, rast_num=ii)
else: # else it's a singleband raster
logger.info("Reading {0}...".format(i))
# read in bands as array
ds_tband, t_nd = RasterIO.read_band_as_array(ds_test)
ds_mband, m_nd = RasterIO.read_band_as_array(ds_mast)
# do diff
if isinstance(t_nd, type(None)) or include_nd:
diff = do_diff(ds_tband, ds_mband)
else:
diff = do_diff(ds_tband, ds_mband, nodata=int(t_nd))
# call stats functions to write out results/plots/etc.
call_stats(i, j, diff, i, dir_out)