def merge_masks_diff_inputs(groupkeys, batchid=''):
os.makedirs('/tmp/results/{}/inputmerged/'.format(batchid),
exist_ok=True)
masks = []
for file in groupkeys:
binarymask = cv2.resize(pngread(file), (1024, 1024),
interpolation=cv2.INTER_AREA)
distance = ndi.distance_transform_edt(binarymask)
local_maxi = peak_local_max(distance,
labels=binarymask,
footprint=np.ones((3, 3)),
indices=False)
markers = ndi.label(local_maxi)[0]
masks.append(watershed(-distance, markers, mask=binarymask))
try:
binarymask = merge_two_masks(outpaths)
distance = ndi.distance_transform_edt(binarymask)
local_maxi = peak_local_max(distance,
labels=binarymask,
footprint=np.ones((3, 3)),
indices=False)
markers = ndi.label(local_maxi)[0]
except:
mask = watershed(-distance, markers, mask=binarymask)
pass
savepath = os.path.join(
'/tmp/results/' + batchid + '/inputmerged/',
file.split('/')[-1].split('__')[0].replace(
'merged_', 'inputmerged_') + '.jpg')
pngsave(savepath, np.uint8(mask > 0))
def savemerge(masklist, outpath=outpath, modelids=modelids):
mask = np.uint8(merge_two_masks(masklist)) > 0
savepath = os.path.join(
outpath,
'merged_' + masklist[0].split('/')[-1].split(modelids[0])[-1])
pngsave(savepath, mask)
num_cores = multiprocessing.cpu_count()
Parallel(n_jobs=num_cores)(
delayed(savemerge)([modelres[0][idx], modelres[1][idx]])
for idx in range(0, len(modelres[0])))
def merge_masks(s3_object, model_ids, batchid=''):
os.makedirs('/tmp/results/merge/merged/', exist_ok=True)
outpaths = []
ismodelresult = any([m for m in model_ids if m in s3_object])
if ismodelresult:
for model_id in model_ids:
if not s3_object.endswith("/") and "masks" in s3_object:
outpath = os.path.join(
'/tmp/results/' + batchid + '/merge/',
model_id + '_' + s3_object.split('/')[-1])
s3.meta.client.download_file(
'sagemaker-eu-west-1-102554356212', s3_object, outpath)
outpaths.append(outpath)
if outpaths:
mask = merge_two_masks(outpaths)
pngsave(
os.path.join('/tmp/results/' + batchid + '/merge/merged/',
'merged_' + s3_object.split('/')[-1]),
np.uint8(mask > 0))
def batch2masks(model_id, batchid=''):
s3 = boto3.resource('s3')
s3_resource = boto3.resource('s3')
sess = sagemaker.Session()
bucket = sess.default_bucket()
s3results = s3_resource.Bucket(name=bucket)
keys = [obj.key for obj in s3results.objects.all()]
os.makedirs('/tmp/results/{}'.format(batchid), exist_ok=True)
savepaths = []
for s3_object in keys:
if "results_" + model_id in s3_object and "out" in s3_object:
s3.meta.client.download_file(
bucket, s3_object, '/tmp/' + s3_object.split('/')[-1])
with open('/tmp/' + s3_object.split('/')[-1], 'rb') as image:
img = image.read()
img = bytearray(img)
mask = np.array(Image.open(io.BytesIO(img)))
savepath = '/tmp/results/' + batchid + '/' + model_id + '.'.join(
s3_object.split('/')[-1].split('.')[:-1])
pngsave(savepath, mask)
os.remove('/tmp/' + s3_object.split('/')[-1])
savepaths.append(savepath)
return (savepaths)
def proccesshelafiles(f, s3=s3, sess=sess, bucket=bucket, prefix=prefix):
train_channel = prefix + '/train'
validation_channel = prefix + '/validation'
train_annotation_channel = prefix + '/train_annotation'
validation_annotation_channel = prefix + '/validation_annotation'
if 'jpg' in f:
file = f
if 'segproj/hela_dataset_training_data/train/' in f:
jpgpath = '/tmp/' + 'hela_' + file.split('/')[-1].split(
'.')[0] + '_' + 'raw.jpg'
s3.meta.client.download_file('meadata', file, jpgpath)
inverted_img = pngread(jpgpath)
num = int(''.join(filter(str.isdigit, str(
inverted_img.dtype)))) - 1
image = exposure.rescale_intensity(inverted_img,
out_range=(0, 2**num - 1))
image = cv2.resize(image, (1024, 1024),
interpolation=cv2.INTER_AREA)
pngsave(jpgpath, img_as_ubyte(image))
sess.upload_data(path=jpgpath,
bucket=bucket,
key_prefix=train_channel)
elif 'segproj/hela_dataset_training_data/val/' in f:
jpgpath = '/tmp/' + 'hela_' + file.split('/')[-1].split(
'.')[0] + '_' + 'raw.jpg'
s3.meta.client.download_file('meadata', file, jpgpath)
inverted_img = pngread(jpgpath)
num = int(''.join(filter(str.isdigit, str(
inverted_img.dtype)))) - 1
image = exposure.rescale_intensity(inverted_img,
out_range=(0, 2**num - 1))
image = cv2.resize(image, (1024, 1024),
interpolation=cv2.INTER_AREA)
pngsave(jpgpath, img_as_ubyte(image))
sess.upload_data(path=jpgpath,
bucket=bucket,
key_prefix=validation_channel)
elif 'png' in f:
file = f
if 'segproj/hela_dataset_training_data/train_annotation/' in f:
pngpath = '/tmp/' + 'hela_' + file.split('/')[-1].split(
'.')[0] + '_' + 'raw.png'
s3.meta.client.download_file('meadata', file, pngpath)
im1 = pngread(pngpath)
num = int(''.join(filter(str.isdigit, str(im1.dtype)))) - 1
image = exposure.rescale_intensity(im1, out_range=(0, 2**num - 1))
image = img_as_ubyte(image)
im = mark_boundaries(image,
im1,
color=[0, 0, 0],
outline_color=[0, 0, 0],
mode='outer',
background_label=0)
im2 = img_as_int(im)
im3 = np.zeros([im2.shape[0], im2.shape[1]])
im3 = im2[:, :, 0] + im2[:, :, 1] + im2[:, :, 2]
im3 = np.uint8((im3 > 0))
im3 = cv2.resize(im3, (1024, 1024), interpolation=cv2.INTER_AREA)
pngsave(pngpath, im3, check_contrast=False)
sess.upload_data(path=pngpath,
bucket=bucket,
key_prefix=train_annotation_channel)
elif 'segproj/hela_dataset_training_data/val_annotation/' in f:
pngpath = '/tmp/' + 'hela_' + file.split('/')[-1].split(
'.')[0] + '_' + 'raw.png'
s3.meta.client.download_file('meadata', file, pngpath)
im1 = pngread(pngpath)
num = int(''.join(filter(str.isdigit, str(im1.dtype)))) - 1
image = exposure.rescale_intensity(im1, out_range=(0, 2**num - 1))
image = img_as_ubyte(image)
im = mark_boundaries(image,
im1,
color=[0, 0, 0],
outline_color=[0, 0, 0],
mode='outer',
background_label=0)
im2 = img_as_int(im)
im3 = np.zeros([im2.shape[0], im2.shape[1]])
im3 = im2[:, :, 0] + im2[:, :, 1] + im2[:, :, 2]
im3 = np.uint8((im3 > 0))
im3 = cv2.resize(im3, (1024, 1024), interpolation=cv2.INTER_AREA)
pngsave(pngpath, im3, check_contrast=False)
sess.upload_data(path=pngpath,
bucket=bucket,
key_prefix=validation_annotation_channel)
def innerloop(filepath, batchid=batchid, imlabels=imlabels):
resimages = createmultipleinputs(filepath)
for idx in range(0, len(resimages)):
savepath = '/tmp/' + batchid + '/' + batchid + '_' + filepath.split('.')[0].split('/')[-1] + '__' + \
imlabels[idx] + '.jpg'
pngsave(savepath, resimages[idx])