def create_crop_files_coordinated(box_list, sub_cdim, img):
sys.path.append('../../DeepMoon/')
import utils.transform as trf
df = pd.DataFrame(box_list,
columns=['x_start', 'y_start', 'x_end', 'y_end'])
long_start_vec = []
long_end_vec = []
lat_start_vec = []
lat_end_vec = []
for box in box_list:
ix = box[::2]
iy = box[1::2]
llong, llat = trf.pix2coord(ix,
iy,
sub_cdim,
list(img.size),
origin="upper")
long_start_vec.append(llong[0])
long_end_vec.append(llong[1])
lat_start_vec.append(llat[0])
lat_end_vec.append(llat[1])
df['long_start'] = long_start_vec
df['long_end'] = long_end_vec
df['lat_start'] = lat_start_vec
df['lat_end'] = lat_end_vec
return df
def test_pix2coord(self, origin):
x, y = trf.coord2pix(self.cx,
self.cy,
self.cdim,
self.imgdim,
origin=origin)
cx, cy = trf.pix2coord(x, y, self.cdim, self.imgdim, origin=origin)
cxy = np.r_[cx, cy]
cxy_gt = np.r_[self.cx, self.cy]
assert np.all(np.isclose(cxy, cxy_gt, rtol=1e-7, atol=1e-10))
def GenDataset(img,
craters,
outhead,
rawlen_range=[1000, 2000],
rawlen_dist='log',
ilen=256,
cdim=[-180., 180., -60., 60.],
arad=3390,
minpix=0,
tglen=256,
binary=True,
rings=True,
ringwidth=1,
truncate=True,
amt=100,
istart=0,
seed=None,
verbose=False):
"""Generates random dataset from a global DEM and crater catalogue.
The function randomly samples small images from a global digital elevation
map (DEM) that uses a Plate Carree projection, and converts the small
images to Orthographic projection. Pixel coordinates and radii of craters
from the catalogue that fall within each image are placed in a
corresponding Pandas dataframe. Images and dataframes are saved to disk in
hdf5 format.
Parameters
----------
img : PIL.Image.Image
Source image.
craters : pandas.DataFrame
Crater catalogue .csv.
outhead : str
Filepath and file prefix of the image and crater table hdf5 files.
rawlen_range : list-like, optional
Lower and upper bounds of raw image widths, in pixels, to crop from
source. To always crop the same sized image, set lower bound to the
same value as the upper. Default is [300, 4000].
rawlen_dist : 'uniform' or 'log'
Distribution from which to randomly sample image widths. 'uniform' is
uniform sampling, and 'log' is loguniform sampling.
ilen : int, optional
Input image width, in pixels. Cropped images will be downsampled to
this size. Default is 256.
cdim : list-like, optional
Coordinate limits (x_min, x_max, y_min, y_max) of image. Default is
LRO-Kaguya's [-180., 180., -60., 60.].
arad : float. optional
World radius in km. Defaults to Moon radius (1737.4 km).
minpix : int, optional
Minimum crater diameter in pixels to be included in crater list.
Useful when the smallest craters in the catalogue are smaller than 1
pixel in diameter.
tglen : int, optional
Target image width, in pixels.
binary : bool, optional
If True, returns a binary image of crater masks.
rings : bool, optional
If True, mask uses hollow rings rather than filled circles.
ringwidth : int, optional
If rings is True, ringwidth sets the width (dr) of the ring.
truncate : bool
If True, truncate mask where image truncates.
amt : int, optional
Number of images to produce. 100 by default.
istart : int
Output file starting number, when creating datasets spanning multiple
files.
seed : int or None
np.random.seed input (for testing purposes).
verbose : bool
If True, prints out number of image being generated.
"""
# just in case we ever make this user-selectable...
origin = "upper"
# Seed random number generator.
np.random.seed(seed)
# Get craters.
AddPlateCarree_XY(craters, list(img.size), cdim=cdim, origin=origin)
iglobe = ccrs.Globe(semimajor_axis=arad * 1000.,
semiminor_axis=arad * 1000.,
ellipse=None)
# Create random sampler (either uniform or loguniform).
if rawlen_dist == 'log':
rawlen_min = np.log10(rawlen_range[0])
rawlen_max = np.log10(rawlen_range[1])
def random_sampler():
return int(10**np.random.uniform(rawlen_min, rawlen_max))
else:
def random_sampler():
return np.random.randint(rawlen_range[0], rawlen_range[1] + 1)
# Initialize output hdf5s.
imgs_h5 = h5py.File(outhead + '_images.hdf5', 'w')
imgs_h5_inputs = imgs_h5.create_dataset("input_images", (amt, ilen, ilen),
dtype='uint8')
imgs_h5_inputs.attrs['definition'] = "Input image dataset."
imgs_h5_tgts = imgs_h5.create_dataset("target_masks", (amt, tglen, tglen),
dtype='float32')
imgs_h5_tgts.attrs['definition'] = "Target mask dataset."
imgs_h5_llbd = imgs_h5.create_group("longlat_bounds")
imgs_h5_llbd.attrs['definition'] = ("(long min, long max, lat min, "
"lat max) of the cropped image.")
imgs_h5_box = imgs_h5.create_group("pix_bounds")
imgs_h5_box.attrs['definition'] = ("Pixel bounds of the Global DEM region"
" that was cropped for the image.")
imgs_h5_dc = imgs_h5.create_group("pix_distortion_coefficient")
imgs_h5_dc.attrs['definition'] = ("Distortion coefficient due to "
"projection transformation.")
imgs_h5_cll = imgs_h5.create_group("cll_xy")
imgs_h5_cll.attrs['definition'] = ("(x, y) pixel coordinates of the "
"central long / lat.")
craters_h5 = pd.HDFStore(outhead + '_craters.hdf5', 'w')
# Zero-padding for hdf5 keys.
zeropad = int(np.log10(amt)) + 1
i = -1
while i < amt:
i = i + 1
if (i == amt):
break
print(
i,
"------------------------------------------------------------------",
i)
# Current image number.
img_number = "img_{i:0{zp}d}".format(i=istart + i, zp=zeropad)
if verbose:
print("Generating {0}".format(img_number))
# Determine image size to crop.
rawlen = random_sampler()
xc = np.random.randint(0, img.size[0] - rawlen)
try:
yc = np.random.randint(0, img.size[1] - rawlen)
except:
i = i - 1
continue
box = np.array([xc, yc, xc + rawlen, yc + rawlen], dtype='int32')
# Load necessary because crop may be a lazy operation; im.load() should
# copy it. See <http://pillow.readthedocs.io/en/3.1.x/
# reference/Image.html>.
im = img.crop(box)
im.load()
# Obtain long/lat bounds for coordinate transform.
ix = box[::2]
iy = box[1::2]
llong, llat = trf.pix2coord(ix,
iy,
cdim,
list(img.size),
origin=origin)
llbd = np.r_[llong, llat[::-1]]
# Downsample image.
im = im.resize([ilen, ilen], resample=Image.NEAREST)
# Remove all craters that are too small to be seen in image.
ctr_sub = ResampleCraters(craters,
llbd,
im.size[1],
arad=arad,
minpix=minpix)
# Convert Plate Carree to Orthographic.
try:
[imgo, ctr_xy, distortion_coefficient,
clonglat_xy] = (PlateCarree_to_Orthographic(im,
llbd,
ctr_sub,
iglobe=iglobe,
ctr_sub=True,
arad=arad,
origin=origin,
rgcoeff=1.2,
slivercut=0.5))
# [imgo, ctr_xy, distortion_coefficient, clonglat_xy] = (
# PlateCarree_to_Orthographic(im, llbd, ctr_sub, iglobe=iglobe, ctr_sub=True,arad=arad, origin=origin, rgcoeff=1.2, slivercut=0.5))
except:
print('less values found ')
i = i - 1
continue
if imgo is None:
print("Discarding narrow image")
i = i - 1
continue
imgo_arr = np.asanyarray(imgo)
if imgo_arr.sum() <= 0:
print("assertion failed, movivng to i= i-1")
i = i - 1
continue
# Make target mask. Used Image.BILINEAR resampling because
# Image.NEAREST creates artifacts. Try Image.LANZCOS if BILINEAR still\
#update- By @ Karan
#remve complete black images.
if not imgo.getbbox():
print("Image is completly black , so ignoring it ")
i = i - 1
continue
print(" printing the length ", len(ctr_xy))
if len(ctr_xy["x"]) == 0:
print("No crater found , moving onto next ")
i = i - 1
continue
# leaves artifacts).
tgt = np.asanyarray(
imgo.resize((tglen, tglen), resample=Image.BILINEAR))
mask = make_mask(ctr_xy,
tgt,
binary=binary,
rings=rings,
ringwidth=ringwidth,
truncate=truncate)
# Output everything to file.
imgs_h5_inputs[i, ...] = imgo_arr
imgs_h5_tgts[i, ...] = mask
sds_box = imgs_h5_box.create_dataset(img_number, (4, ), dtype='int32')
sds_box[...] = box
sds_llbd = imgs_h5_llbd.create_dataset(img_number, (4, ),
dtype='float')
sds_llbd[...] = llbd
sds_dc = imgs_h5_dc.create_dataset(img_number, (1, ), dtype='float')
sds_dc[...] = np.array([distortion_coefficient])
sds_cll = imgs_h5_cll.create_dataset(img_number, (2, ), dtype='float')
sds_cll[...] = clonglat_xy.loc[:, ['x', 'y']].as_matrix().ravel()
craters_h5[img_number] = ctr_xy
imgs_h5.flush()
craters_h5.flush()
imgs_h5.close()
craters_h5.close()
def GenDataset(box_list, img, craters, outhead, arad, cdim):
truncate = True
ringwidth = 1
rings = True
binary = True
minpix = 1.
tglen = 256
ilen = 256
amt = len(box_list)
origin = "upper"
# Get craters.
igen.AddPlateCarree_XY(craters, list(img.size), cdim=cdim, origin=origin)
iglobe = ccrs.Globe(semimajor_axis=arad * 1000.,
semiminor_axis=arad * 1000.,
ellipse=None)
# Initialize output hdf5s.
[
imgs_h5, imgs_h5_inputs, imgs_h5_tgts, imgs_h5_llbd, imgs_h5_box,
imgs_h5_dc, imgs_h5_cll, craters_h5
] = init_files(outhead, amt, ilen, tglen)
for i in range(amt):
# Current image number.
img_number = "img_{:05d}".format(i)
# Determine image size to crop.
box = box_list[i]
#print("Generating {} current crop: ({})".format(img_number,box))
# Load necessary because crop may be a lazy operation; im.load() should
# copy it. See <http://pillow.readthedocs.io/en/3.1.x/
# reference/Image.html>.
im = img.crop(box)
im.load()
# Obtain long/lat bounds for coordinate transform.
ix = box[::2]
iy = box[1::2]
llong, llat = trf.pix2coord(ix,
iy,
cdim,
list(img.size),
origin=origin)
llbd = np.r_[llong, llat[::-1]]
# Downsample image.
im = im.resize([ilen, ilen], resample=Image.NEAREST)
# Remove all craters that are too small to be seen in image.
ctr_sub = igen.ResampleCraters(craters,
llbd,
im.size[1],
arad=arad,
minpix=minpix)
# Convert Plate Carree to Orthographic.
[imgo, ctr_xy, distortion_coefficient,
clonglat_xy] = (igen.PlateCarree_to_Orthographic(im,
llbd,
ctr_sub,
iglobe=iglobe,
ctr_sub=True,
arad=arad,
origin=origin,
rgcoeff=1.2,
slivercut=0.5))
if imgo is None:
print("Discarding narrow image")
continue
imgo_arr = np.asanyarray(imgo)
assert imgo_arr.sum() > 0, ("Sum of imgo is zero! There likely was "
"an error in projecting the cropped "
"image.")
# Make target mask. Used Image.BILINEAR resampling because
# Image.NEAREST creates artifacts. Try Image.LANZCOS if BILINEAR still
# leaves artifacts).
tgt = np.asanyarray(
imgo.resize((tglen, tglen), resample=Image.BILINEAR))
mask = igen.make_mask(ctr_xy,
tgt,
binary=binary,
rings=rings,
ringwidth=ringwidth,
truncate=truncate)
# Output everything to file.
output_to_file(img_number, i, imgs_h5_inputs, imgo_arr, imgs_h5_tgts,
mask, imgs_h5_box, box, imgs_h5_llbd, llbd, imgs_h5_dc,
distortion_coefficient, imgs_h5_cll, clonglat_xy,
craters_h5, ctr_xy, imgs_h5)
imgs_h5.close()
craters_h5.close()
def get_llbd(self, box):
llong, llat = trf.pix2coord(box[::2], box[1::2], self.cdim,
self.imgsize, origin="upper")
return np.r_[llong, llat[::-1]]
def test_gendataset(self, tmpdir, ringwidth):
amt = 10
zeropad = 2
outhead = str(tmpdir.join('gentest'))
igen.GenDataset(self.img, self.craters, outhead,
rawlen_range=[300, 1000], rawlen_dist='log',
ilen=self.imlen, tglen=self.imlen, cdim=self.cdim,
minpix=1, ringwidth=ringwidth, amt=amt, istart=0,
seed=self.seed)
imgs_h5 = h5py.File(outhead + '_images.hdf5', 'r')
craters_h5 = pd.HDFStore(outhead + '_craters.hdf5', 'r')
for i in range(amt):
# Find image number.
img_number = "img_{i:0{zp}d}".format(i=i, zp=zeropad)
# Load box.
box = np.array(imgs_h5['pix_bounds'][img_number][...])
im = self.img.crop(box)
im.load()
# Obtain long/lat bounds for coordinate transform.
ix = box[::2]
iy = box[1::2]
llong, llat = trf.pix2coord(ix, iy, self.cdim, list(self.img.size),
origin='upper')
llbd = np.r_[llong, llat[::-1]]
# Downsample image.
im = im.resize([self.imlen, self.imlen], resample=Image.NEAREST)
# Remove all craters that are too small to be seen in image.
ctr_sub = igen.ResampleCraters(self.craters, llbd, im.size[1],
minpix=1)
# Convert Plate Carree to Orthographic.
[imgo, ctr_xy, distortion_coefficient, clonglat_xy] = (
igen.PlateCarree_to_Orthographic(
im, llbd, ctr_sub, iglobe=self.iglobe, ctr_sub=True,
slivercut=0.5))
imgo_arr = np.asanyarray(imgo)
# Make target mask.
tgt = np.asanyarray(imgo.resize((self.imlen, self.imlen),
resample=Image.BILINEAR))
mask = igen.make_mask(ctr_xy, tgt, binary=True, rings=True,
ringwidth=ringwidth, truncate=True)
assert np.all(imgo_arr == imgs_h5['input_images'][i, ...])
assert np.all(mask == imgs_h5['target_masks'][i, ...])
assert np.all(llbd == imgs_h5['longlat_bounds'][img_number][...])
assert (distortion_coefficient ==
imgs_h5['pix_distortion_coefficient'][img_number][0])
assert np.all(clonglat_xy[["x", "y"]] ==
imgs_h5['cll_xy'][img_number][...])
assert np.all(ctr_xy == craters_h5[img_number])
imgs_h5.close()
craters_h5.close()