def load_dataset_by_dir(_dir): pos_dir = _dir + "POSITIVE/*.png" ic_pos = ImageCollection(pos_dir, load_func=load_and_pp) X_pos = concatenate_images(ic_pos) y_pos = np.array([1 for _ in range(len(ic_pos))]) neg_dir = _dir + "NEGATIVE/*.png" ic_neg = ImageCollection(neg_dir, load_func=load_and_pp) X_neg = concatenate_images(ic_neg) y_neg = np.array([0 for _ in range(len(ic_neg))]) X = np.concatenate((X_pos, X_neg)) y = np.concatenate((y_pos, y_neg)) return (X, y)
def save_stacks(vis, yfp, dsred, save_path, fov_name):
vis_con = io.concatenate_images(img_as_uint(vis))
tf.imsave(save_path + '/%s_bf_stack.tif' % fov_name, vis_con)
print("BF stack saved")
yfp_con = io.concatenate_images(img_as_uint(yfp))
tf.imsave(save_path + '/%s_yfp_stack.tif' % fov_name, yfp_con)
print("yfp stack saved")
dsred_con = io.concatenate_images(img_as_uint(dsred))
tf.imsave(save_path + '/%s_dsred_stack.tif' % fov_name, dsred_con)
print("dsred stack saved")
def load_images(path, input_size, output_size):
x_ = []
y_ = []
counter, totalnumber = 1, len(os.listdir(path))
for imgpath in os.listdir(path):
if counter % 100 == 0:
print("Importing image %s of %s (%s%%)" %
(counter, totalnumber, round(counter / totalnumber * 100)))
y = imread(path + "/" + imgpath)
y = rgb2gray(resize(y, output_size, mode="constant"))
x = resize(y, input_size, mode="constant")
x_.append(x)
y_.append(y)
counter += 1
return concatenate_images(x_), concatenate_images(y_)
def __getitem__(self, vid_idx):
""" Arguments:
vid_idx(int): Video Index to be fetched form the video list
"""
vid_name_path = os.path.join(self.root_dir,
self.vot_list.iloc[vid_idx, 0], '*.jpg')
gt = pd.read_csv(os.path.join(self.root_dir,
self.vot_list.iloc[vid_idx, 0],
'groundtruth.txt'),
header=None)
im_seq = imread_collection(vid_name_path)
# Image collection to np.array
images = concatenate_images(im_seq) # Shape(Nr. of images, h, w, RGB)
# Also convert the gt to np.array
gt = gt.values
sample = {'Video': images, 'gt': gt}
# Cant tell yet what this is for
if self.transform:
sample = self.transform(sample)
return sample
def save_ee_image(base_path,
ee_image,
bands: list,
image_id: str,
bbox: tuple or list,
zoom=13,
img_size: int or None = None):
if os.path.exists(base_path):
shutil.rmtree(base_path)
scale = get_tile_pixel_scale_from_zoom(
zoom=zoom, tile_size=img_size if img_size else 256)
print(
f"Downloading image {f'of size {img_size}x{img_size} pixels ' if img_size else ''}"
f"with {scale:.2f} m resolution")
url = get_image_download_url(ee_image, bbox, scale, name=image_id)
print(url)
r = requests.get(url)
z = zipfile.ZipFile(BytesIO(r.content))
z.extractall(base_path)
imgs = [
io.imread(os.path.join(base_path, f"{image_id}.{band}.tif"))
for band in bands
]
out = io.concatenate_images(imgs)
if img_size is not None:
out = resize(out, (out.shape[0], img_size, img_size))
imsave(f"{base_path}.tif", out)
shutil.rmtree(base_path)
return out
def __getitem__(self, vid_idx, T = 10):
""" Arguments:
vid_idx(int): Video Index to be fetched form the video list
T(int): Nr of Images in sequence - default == 10
"""
gt = pd.read_csv(os.path.join(self.root_dir,
self.vot_list.iloc[vid_idx,0],
'groundtruth.txt'), header = None)
vid_name_path = os.path.join(self.root_dir,
self.vot_list.iloc[vid_idx,0],
'*.jpg')
file_names = glob.glob(vid_name_path)
rand_start = np.random.randint(0, len(file_names)-T+1)
file_names = file_names[rand_start:(rand_start+T-1)]
im_seq = imread_collection(file_names)
# Image collection to np.array
images = concatenate_images(im_seq) # Shape(Nr. of images, h, w, RGB)
# Also convert the gt to np.array
gt = gt.values
gt = gt[rand_start:(rand_start+T-1),:]
sample = {'Video': images, 'gt': gt}
# Cant tell yet what this is for
if self.transform:
sample = self.transform(sample)
return sample
def load_data_from_folder(self, dir):
# read all images into an image collection
ic = io.ImageCollection(dir + "*.jpg", load_func=self.imread_convert)
# create one large array of image data
data = io.concatenate_images(ic)
# extract labels from image names
labels = np.array(ic.files)
for i, f in enumerate(labels):
m = re.search("_", f)
labels[i] = f[len(dir):m.start()]
return (data, labels)
def read_dataset(data_txt_file, image_data_path):
"""Read data into a Python dictionary.
Args:
data_txt_file(str): path to the data txt file.
image_data_path(str): path to the image directory.
Returns:
data(dict): A Python dictionary with keys 'image' and 'label'.
The value of dict['image'] is a numpy array of dimension (N,28,28)
containing the loaded images.
The value of dict['label'] is a numpy array of dimension (N,)
containing the loaded label.
N is the number of examples in the data split, the exampels should
be stored in the same order as in the txt file.
"""
f = open(data_txt_file, 'r')
image = []
label = []
for line in f:
line.strip('\n')
items = line.split('\t')
image.append(image_data_path + '/' + items[0])
label.append(int(items[1]))
ic = io.imread_collection(image, True)
images = io.concatenate_images(ic)
data = {'image': images, 'label': np.asarray(label)}
return data
def load_frames(folder_name, offset=0, desired_fps=3, max_frames=40):
"""
:param folder_name: Filename with a gif
:param offset: How many frames into the gif we want to start at
:param desired_fps: How many fps we'll sample from the image
:return: [T, h, w, 3] GIF
"""
coll = ImageCollection(folder_name + '/out-*.jpg', mode='RGB')
try:
duration_path = folder_name + '/duration.txt'
with open(duration_path, 'r') as f:
durs = f.read().splitlines()
fps = 100.0 / durs[0]
except:
# Some error occurs
fps = 10
# want to scale it to desired_fps
keep_ratio = max(1., fps / desired_fps)
frames = np.arange(offset, len(coll), keep_ratio).astype(int)[:max_frames]
def _add_chans(img):
if img.ndim == 3:
return img
return np.stack([img] * 3, -1)
imgs_concat = concatenate_images([_add_chans(coll[f]) for f in frames])
assert imgs_concat.ndim == 4
return imgs_concat
def save_stacks(translated_images_dict, save_path, fov_name):
print("Saving stacks...")
for channel, stack in translated_images_dict.items():
print("Saving {} stack".format(channel))
concat_stack = io.concatenate_images(img_as_uint(stack))
tf.imsave(save_path + '/{}_{}_stack.tif'.format(fov_name, channel),
concat_stack)
def lowres(data):
for i in range(len(data)):
data[i] = cv2.resize(data[i], (16, 16))
data[i] = cv2.resize(data[i], (32, 32), interpolation=cv2.INTER_CUBIC)
lowres_arr = io.concatenate_images(data)/255
print(lowres_arr.shape)
np.save("train_data/numeric_data/lowres_data", lowres_arr)
def generator_full_image(directory,
ext,
batch_size,
preprocessing=None,
mode='train'):
""""""
#loading filenames (only and not the image)
fnamesX = sorted(os.listdir(os.path.join(directory, 'X')))
pathnamesX = [
os.path.join(directory, 'X', f) for f in fnamesX
if f.split('.')[-1] in ext
]
pathnamesX = np.array(pathnamesX)
fnamesY = sorted(os.listdir(os.path.join(directory, 'Y')))
pathnamesY = [
os.path.join(directory, 'Y', f) for f in fnamesY
if f.split('.')[-1] in ext
]
pathnamesY = np.array(pathnamesY)
idx = np.arange(pathnamesX.shape[0])
while True:
#shuffling at the start of each epoch..
np.random.shuffle(idx)
for i in xrange(0, pathnamesX.shape[0], batch_size):
#picking a batch..
pathnamesX_batch = pathnamesX[idx[i:i + batch_size]]
pathnamesY_batch = pathnamesY[idx[i:i + batch_size]]
imagesX = io.ImageCollection(pathnamesX_batch)
imagesY = io.ImageCollection(pathnamesY_batch)
#optionally applying preporcessing to each batch
if preprocessing is not None:
imagesX, imagesY = preprocessing(imagesX, imagesY)
imagesX = io.concatenate_images(imagesX)
imagesY = io.concatenate_images(imagesY)
imagesX = img_as_float(imagesX[:, :, :, np.newaxis])
imagesY = img_as_float(imagesY[:, :, :, np.newaxis]).astype(bool)
yield imagesX, imagesY
def save_stacks(vis, gfp, mko, e2crimson, save_path, fov_name):
vis_con = io.concatenate_images(img_as_uint(vis))
tf.imsave(save_path + '/%s_bf_stack.tif' % fov_name, vis_con)
print("BF stack saved")
gfp_con = io.concatenate_images(img_as_uint(gfp))
tf.imsave(save_path + '/%s_gfp_stack.tif' % fov_name, gfp_con)
print("gfp stack saved")
mko_con = io.concatenate_images(img_as_uint(mko))
tf.imsave(save_path + '/%s_mko_stack.tif' % fov_name, mko_con)
print("mko stack saved")
e2crimson_con = io.concatenate_images(img_as_uint(e2crimson))
tf.imsave(save_path + '/%s_e2crimson_stack.tif' % fov_name, e2crimson_con)
print("e2crimson stack saved")
def set_resized_cells_lists(self, cropped_dsred_stacks, cropped_yfp_stacks,
cropped_bf_stacks, cells_dfs):
""" Return three lists - dsred_stacks, yfp_stacks_, and bf_stacks,
wit each element being the resized stack of that cell """
resized_dsred_stacks = []
resized_yfp_stacks = []
resized_bf_stacks = []
# remember that cells_dfs is a list of dfs, one for each
# unique cell in the master_cells_dfs
for cell_index in range(0, len(cells_dfs)):
print(
"Setting resized stacks for each channel (dsred, yfp, bf) of cell %s"
% cell_index)
# use the cropped_dsred_stack of current cell to define slice shapes
slice_shapes_array = self.set_slice_shapes_array(
cropped_dsred_stacks[cell_index])
height_max, width_max = self.set_max_dims(slice_shapes_array)
# the slice shapes offsets
slice_height_offsets, slice_width_offsets = self.set_slice_offsets(
cropped_dsred_stacks[cell_index], slice_shapes_array,
height_max, width_max)
resized_dsred_stack = self.set_resized_stack(
cropped_dsred_stacks[cell_index], slice_height_offsets,
slice_width_offsets)
resized_yfp_stack = self.set_resized_stack(
cropped_yfp_stacks[cell_index], slice_height_offsets,
slice_width_offsets)
resized_bf_stack = self.set_resized_stack(
cropped_bf_stacks[cell_index], slice_height_offsets,
slice_width_offsets)
resized_dsred_stack = io.concatenate_images(resized_dsred_stack)
resized_yfp_stack = io.concatenate_images(resized_yfp_stack)
resized_bf_stack = io.concatenate_images(resized_bf_stack)
resized_dsred_stacks.append(resized_dsred_stack)
resized_yfp_stacks.append(resized_yfp_stack)
resized_bf_stacks.append(resized_bf_stack)
return resized_dsred_stacks, resized_yfp_stacks, resized_bf_stacks
def read_video(filepath, input_size, output_size):
vid = imageio.get_reader(filepath, "ffmpeg")
video_len = vid.get_length()
counter, totalnumber = 1, video_len
y_ = []
x_ = []
for i in range(0, video_len - 1):
if counter % 100 == 0:
print("Importing frame %s of %s (%s%%)" %
(counter, totalnumber, round(counter / totalnumber * 100)))
y_frame = resize(vid.get_data(i), output_size, mode="constant")
y_frame = rgb2gray(y_frame)
x_frame = resize(y_frame, input_size, mode="constant")
y_.append(y_frame)
x_.append(x_frame)
counter += 1
return concatenate_images(x_), concatenate_images(y_)
def noise(data, sigma):
for i in range(len(data)):
np.random.seed()
ga_num = np.random.normal(scale=sigma, size=data[i].shape)
data[i] = data[i] + ga_num
noise_arr = io.concatenate_images(data)/255
print(noise_arr.shape)
np.save("train_data/numeric_data/noise_data", noise_arr)
def load_imgs(str):
x_train = io.ImageCollection(str, load_func=convert_gray)
height = x_train[0].shape[0]
width = x_train[0].shape[1]
x_train = io.concatenate_images(x_train)
x_train = x_train.astype('float32') / 255.
x_train = np.reshape(x_train, (-1, height, width, 1))
return x_train
def blur(data, with_resize = False):
for i in range(len(data)):
data[i] = cv2.GaussianBlur(data[i], ksize=(0,0), sigmaX=0.8, sigmaY=0.8)
if with_resize == True:
data[i] = cv2.resize(data[i], (16,16))
data[i] = cv2.resize(data[i], (32,32), interpolation = cv2.INTER_CUBIC)
blur_arr = io.concatenate_images(data)/255
print(blur_arr.shape)
np.save("train_data/numeric_data/blur_data", blur_arr)
def concatenate_collections_test():
data_path = PROJECT_PATH + '/res/92*.jpg'
coll = io.ImageCollection(data_path)
# 连接的图片数量
print(len(coll))
# 连接前的图片尺寸，所有的都一样
print(coll[0].shape)
mat=io.concatenate_images(coll)
# 连接后的数组尺寸
print(mat.shape)
def load_test_data(test_dir,sample):
print "loading test data!"
cur_dir=test_dir+"images/"
onlyfiles = [f for f in listdir(cur_dir) if isfile(join(cur_dir, f))]
onlyfiles=[cur_dir+f for f in onlyfiles]
numfiles=int(round(len(onlyfiles)*sample))
onlyfiles=onlyfiles[0:numfiles]
images=concatenate_images(ImageCollection(onlyfiles,load_func=imreadconvert))
print "loaded test data:"
print str(images.shape)
return images
def registration(img_stack):
"""input: ndarray of images
ouput: ndarray of registered images
"""
reg_stack = reg_iter(img_stack)
if not np.array_equal(img_stack[0], reg_stack[0]):
print("destination image has been changed in registered stack.")
# src and dst images should be different
assert not np.array_equal(img_stack[0], reg_stack[1])
return io.concatenate_images(reg_stack)
def load_test_data(test_dir, sample):
print "loading test data!"
cur_dir = test_dir + "images/"
onlyfiles = [f for f in listdir(cur_dir) if isfile(join(cur_dir, f))]
onlyfiles = [cur_dir + f for f in onlyfiles]
numfiles = int(round(len(onlyfiles) * sample))
onlyfiles = onlyfiles[0:numfiles]
images = concatenate_images(
ImageCollection(onlyfiles, load_func=imreadconvert))
print "loaded test data:"
print str(images.shape)
return images
def main(img_stack):
"""input: ndarray of images
ouput: ndarray of registered images
"""
# assert len(img_stack) == 3 # Check that images are greyscale.
reg_stack = reg_iter(img_stack)
if not np.array_equal(img_stack[0], reg_stack[0]):
assert False, "destination image has been changed in registered stack."
# src and dst images should be different
assert not np.array_equal(img_stack[0], reg_stack[1])
return io.concatenate_images(reg_stack)
def load_img_stack(path):
"""Load multiple single-channel image files matching the given path expression into a concatenated array
Note that this is intended for use with single channel images; if more channels are present use skimage.io
functions directly.
Args:
path: Path expression compatible with ```skimage.io.imread_collection```
Returns:
3 dimensional numpy array with axes [z, x, y] where z is z-coordinate of images and
x, y are pixel locations
"""
img = io.imread_collection(path)
return io.concatenate_images(img)
def load_data_from_folder(dir):
ic = io.ImageCollection(dir + "*.jpg", load_func=imread_convert)
data = io.concatenate_images(ic)
labels = np.array(ic.files)
for i, f in enumerate(labels):
print(f)
m = f.find("_")
lbl = f[len(dir):m]
# m = re.search("_", f)
#lbl = f[len(dir):m.start()]
# if lbl not in people:
# lbl = "Rando"
# labels[i] = lbl
labels[i] = people.get(lbl, 4)
return (data, labels)
def load_sorted_data(self, d):
all_data = []
all_labels = []
for sub_d in os.listdir(os.getcwd() + '/' + d):
if sub_d == '.DS_Store': continue
current_path = os.getcwd() + '/' + d + '/' + sub_d + '/'
# read all images into an image collection
ic = io.ImageCollection(current_path + "*.bmp",
load_func=self.imread_convert)
# create one large array of image data
data = io.concatenate_images(ic)
labels = np.array([sub_d] * len(data))
all_data.extend(data)
all_labels.extend(labels)
return (np.array(all_data), np.array(all_labels))
def load_images():
"""
Return COIL-20 images
output:
ndarray of shape (1440, 16384)
(1440 flattened 128x128 points)
"""
COIL20_SUFFIX = 'coil-20-proc'
__, DATA_DIR = load_data_utils.get_env_vars(go_up=True)
COIL20_DIR = DATA_DIR + '/' + COIL20_SUFFIX
image_list = imread_collection(COIL20_DIR + '/*.png')
points = concatenate_images(image_list)
imshape = points.shape
reshaped_points = points.reshape(imshape[0], imshape[1] * imshape[2])
return reshaped_points
def load_validation_data(valid_dir,label_dict,sample):
print "loading validation data!"
image_to_id=open(valid_dir+"val_annotations.txt",'r').read().split('\n')
while '' in image_to_id:
image_to_id.remove('')
image_names=[]
image_labels=[]
for entry in image_to_id:
tokens=entry.split('\t')
image_names.append(valid_dir+'images/'+tokens[0])
image_labels.append(label_dict[tokens[1]])
num_entries=int(round(len(image_names)*sample))
image_names=image_names[0:num_entries]
images=concatenate_images(ImageCollection(image_names,load_func=imreadconvert))
print "image val shape:"+str(images.shape)
print "loaded validation data:"
image_labels=np.asarray(image_labels[0:num_entries])
image_labels=np.reshape(image_labels,(len(image_labels),))
image_labels=one_hot_encode(image_labels)
return images, image_labels
def load_ifood(im_dir, label_dir, dim):
"""load images and labels
---------
im_dir: location of the images
label_dir: location of the labels csv file
dim: dimension of the resized labels
"""
# TODO: joblib to parallelize
# https://scikit-image.org/docs/dev/user_guide/tutorial_parallelization.html
images = ImageCollection(im_dir)
images_resized = [resize(image, dim) for image in images]
num_images = len(images_resized)
image_arr = np.reshape(concatenate_images(images_resized), (num_images, -1))
labels = pd.read_csv(label_dir)
labels = labels.sort_values(by=['img_name'])
labels = labels.head(num_images)
y = labels["label"].to_numpy()
unique_labels = labels["label"].unique()
return image_arr, y, unique_labels
def save_cell_stacks():
threshold_channel = input("Choose channel to use for thresholding: ")
cs = Cell_Stack(threshold_channel=threshold_channel)
try:
cell_indices = cs.master_cells_df.sub_coord
except:
cell_indices = cs.master_cells_df.cell_index
abs_i = 0
for cell_index in cell_indices:
print('Cell index = ', cell_index)
# Run all the cropping and processing methods of Cell_Stack on the cell
# with cell_index
cell_rois_dfs = cs.set_cell_crop_roi_dfs(cs.master_cells_df)
cell_cropped_channels_dict = cs.set_cell_cropped_stacks_dict(
cs.master_cells_df, cell_rois_dfs, abs_i)
cs.add_cell_otsu_thresholded_stack()
resized_channels_dict = cs.set_resized_cell_cropped_channels_dict(
cs.cell_cropped_channels_dict)
# Save each stack (Note: compartment_dir is the dir holding all
# xy FOVs for that flow compartment and therefore isolated genetic
# + environmental etc. condition)
# Need to use absolute i so the right compartment_dir etc. is
# selected when there are multiple compartments in a master_index
# and thus cell_index restarts
compartment_dir = cs.get_compartment_dir(cs.master_cells_df, abs_i)
expt_date = int(cs.master_cells_df.date[abs_i])
expt_type = cs.master_cells_df.expt_type[abs_i]
xy = str(int(cs.master_cells_df.xy[abs_i]))
for channel_name, stack in resized_channels_dict.items():
filename = f'{expt_date}_{expt_type}_xy{str(xy).zfill(2)}_cell{str(cell_index).zfill(3)}_{channel_name}_stack.tif'
save_path = f'{compartment_dir}//{filename}'
try:
imsave(save_path, concatenate_images(stack))
except:
print(
f"Could not save stacks for cell {cell_index}, img dims must agree"
)
abs_i += 1
def load_train_data(train_dir,label_dict,sample):
print "loading training data!"
nsamples=int(round(sample*500))#500 images for each of 200 labels
file_names=[]
labels=[]
for label in label_dict:
#print str(label)
cur_dir=train_dir+label+"/images"
onlyfiles = [f for f in listdir(cur_dir) if isfile(join(cur_dir, f))][0:nsamples]
onlyfiles=[cur_dir+'/'+f for f in onlyfiles]
file_names=file_names+onlyfiles
cur_labels=nsamples*[label_dict[label]]
labels=labels+cur_labels
X_train=concatenate_images(ImageCollection(file_names,load_func=imreadconvert))
print "loaded training data"
print str(X_train.shape)
Y_train=np.asarray(labels)
Y_train=np.reshape(Y_train,(len(Y_train),))
Y_train=one_hot_encode(Y_train)
print str(Y_train.shape)
return X_train,Y_train
def __getitem__(self, idx):
dir_to_fetch_from = os.path.join(self.root_dir, os.listdir(self.root_dir)[idx])
images = []
file_list = []
for file in os.listdir(dir_to_fetch_from):
file_list.append(os.path.join(dir_to_fetch_from, file))
images = io.concatenate_images(io.imread_collection(file_list))
file_io = []
for img in images:
file_io.append(transforms.ToTensor()(img).unsqueeze(0))
images = torch.cat(file_io, 0)
if not args.cuda:
images = Variable(images)
else:
images = Variable(images.cuda())
_, mu, _, _ = vae(images)
diff = torch.abs(mu[:self.num_per_sample]-mu[self.num_per_sample:]).data
sample = {'diff': diff.mean(0), 'label': self.labels[idx]}
return sample
def convert_ct():
# Create output folder
if not os.path.isdir(OUTPUT_FOLDER):
os.mkdir(OUTPUT_FOLDER)
for folder_name in os.listdir(DATA_FOLDER):
output_folder = f'{OUTPUT_FOLDER}/{folder_name}'
input_folder = f'{DATA_FOLDER}/{folder_name}/{TYPE}'
if not os.path.isdir(output_folder):
os.mkdir(output_folder)
for phase in ['InPhase', 'OutPhase']:
dicom_dir = input_folder + '/DICOM_anon/' + phase
dicom2nifti.convert_directory(dicom_dir, output_folder, compression=False)
for file in os.listdir(output_folder):
if file.endswith(".nii") and not file.startswith("InPhase"):
os.rename(output_folder + '/' + file ,output_folder + '/' + phase + '.nii')
data = list()
for phase in ['InPhase', 'OutPhase']:
data.append(nib.load(output_folder + '/' + phase + '.nii').get_fdata().astype(np.float32))
# Combine the phases as channels
data = np.stack(data, axis=-1)
nii_data = nib.Nifti1Image(data, affine=np.eye(4))
nib.save(nii_data, output_folder + '/Combined.nii')
# Combine collection of 2D images
images = input_folder + '/Ground/*.png'
label = concatenate_images(imread_collection(images))
label = np.moveaxis(label, 0, -1) # put depth last
label = np.rot90(label, k=3) # 270 degree rotation
# Save image 3D array as nii
nii_label = nib.Nifti1Image(label, affine=np.eye(4))
nii_label.to_filename(output_folder + '/ground.nii')
