global n_idstr_train

global n_idstr_valid

global val_split

global idstr_train

global idstr_valid

# only load once

if n_idstr_train > 0: return

# Get train ID Strings from directory names

train_path = arg_config.cfg['train_path']

idstrs = next(os.walk(train_path))[1]

# remove duff training examples

if '7b38c9173ebe69b4c6ba7e703c0c27f39305d9b2910f46405993d2ea7a963b80' in idstrs:

print('ignoring 7b38c9173ebe69b4c6ba7e703c0c27f39305d9b2910f46405993d2ea7a963b80')

idstrs.remove('7b38c9173ebe69b4c6ba7e703c0c27f39305d9b2910f46405993d2ea7a963b80')

do_short = False

# short version on mac (always)

if arg_config.args.machine == 'james_mac':

do_short = True

if do_short:

idstrs = idstrs[:10]

# stats

n_idstr = len(idstrs)

# build the valid and train dataset lists

for idx in range(n_idstr):

if idx % val_split == 0:

# for the valid set we don't want augmented data

idstr_valid.append(idstrs[idx])

else:

# we want all the angles x24

for angle in range(0,360,15):

# all the shears x5

for shear in shear_tag_list:

# and all filps x2

for flip in range(2):

idstr_train.append((idstrs[idx],angle,shear,flip))

n_idstr_train = len(idstr_train)

n_idstr_valid = len(idstr_valid)

# shuffle the valid and train dataset lists

random.Random(2018).shuffle(idstr_train)

random.Random(2018).shuffle(idstr_valid)

# print stats

print('load_data()...')

print("n_train_ids / train / val",n_idstr,n_idstr_train,n_idstr_valid)

global idstr_train

global idstr_valid

load_data()

train_path = arg_config.cfg['train_path']

if not is_train:

# VALIDATIONS SET

for idstr in idstr_valid:

path_X = os.path.join(train_path,idstr,'images','aug_0_sh_' + idstr + '.png')

path_Y = os.path.join(train_path,idstr,'masks','aug_0_sh_' + idstr + '.png')

img_X = imread(path_X)[:,:,:3]

img_Y = imread(path_Y)

# convert uint8 to float64

img_X = np.array(img_X) / 255.0

img_Y = np.array(img_Y) / 255.0

(xh,xw,_) = img_X.shape

(yh,yw) = img_Y.shape

assert yh == xh, "mask is not same shape as image"

assert yw == xw, "mask is not same shape as image"

# last field is about flipping - ignore

yield img_X,img_Y,-1

return

while(True):

# TRAINING SET

for idstr,angle,shear,do_flip, in idstr_train:

path_X = os.path.join(train_path,idstr,

'images',

'aug_' + str(angle) + '_' + shear + '_' + idstr + '.png')

path_Y = os.path.join(train_path,idstr,

'masks',

'aug_' + str(angle) + '_' + shear + '_' + idstr + '.png')

img_X = imread(path_X)[:,:,:3]

img_Y = imread(path_Y)

img_X = np.array(img_X) / 255.0

img_Y = np.array(img_Y) / 255.0

(xh,xw,_) = img_X.shape

(yh,yw) = img_Y.shape

# convert uint8 to float64

assert yh == xh, "mask is not same shape as image"

assert yw == xw, "mask is not same shape as image"

if len(img_X) > 0:

h,w = img_X.shape[:2]

else:

h,w = img_Y.shape[:2]

crop_x = (w - crop_w) // 2

crop_y = (h - crop_h) // 2

crop_x += random.randint(-shift,shift)

crop_y += random.randint(-shift,shift)

if len(img_X) > 0:

img_X = img_X[crop_y:crop_y+crop_h,

crop_x:crop_x+crop_w,

:]

if len(img_Y) > 0:

img_Y = img_Y[crop_y:crop_y+crop_h,

crop_x:crop_x+crop_w]

img_X = rotate(img_X, rotation_angle, order=3)

img_Y = rotate(img_Y, rotation_angle, order=0)

if len(img_X) > 0:

h,w = img_X.shape[:2]

else:

h,w = img_Y.shape[:2]

matrix = np.zeros((3,3))

matrix[0,0] = 1.0

matrix[1,1] = 1.0

matrix[2,2] = 1.0

if horz:

matrix[0,1] = sh

matrix[0,2] = -np.sin(sh)* w / 2

else:

matrix[1,0] = sh

matrix[1,2] = -np.sin(sh)* h / 2

# Create Afine transform

afine_tf = transform.AffineTransform(matrix)

#print(afine_tf.params)

# Apply transform to image data

if len(img_X) > 0:

img_X = transform.warp(img_X, inverse_map=afine_tf,mode='edge', order=3)

if len(img_Y) > 0:

img_Y = transform.warp(img_Y, inverse_map=afine_tf,mode='edge', order=0)

XY_gen = gen_XY(is_train=is_train)

for img_X,img_Y,do_flip in XY_gen:

(h,w,_) = img_X.shape

# width of the original center rounded to the nearest 16

# then add a border of 16 (32)

# we like a bit of random jitter of the traing data

# consequently we can't use the whole 32 pixel border

# contained in the augmented images

# we can only have a SAFE border of 16 pixels

w_crop = int(round((w-64)/16))*16 + 32

h_crop = int(round((h-64)/16))*16 + 32

if is_train:

# we like a bit of random jitter of the traing data

aug_X, aug_Y = do_random_crop(img_X,img_Y,w_crop,h_crop,16)

# flipping of requested

if do_flip:

img_X = np.fliplr(img_X)

img_Y = np.fliplr(img_Y)

else:

# return cropped images with no jitter or augmentation

aug_X,aug_Y = do_random_crop(img_X,img_Y,w_crop,h_crop,0)

# get the dimentions ready for the NN

aug_X = np.expand_dims(aug_X, axis=0)

aug_Y = np.expand_dims(aug_Y, axis=0)

aug_Y = np.expand_dims(aug_Y, axis=-1)

aug_X = np.array(aug_X,dtype=np.float32)

aug_Y = np.array(aug_Y,dtype=np.float32)