def get_data(self):
filenames = self.get_filesindirectory(self.get_prefix(), self.get_extension())
try:
from libtiff import TIFF
im = TIFF.open(filenames[0])
v0 = im.read_image()
x = NXfield(range(v0.shape[1]), dtype=np.uint16, name="x")
y = NXfield(range(v0.shape[0]), dtype=np.uint16, name="y")
z = NXfield(range(1, len(filenames) + 1), dtype=np.uint16, name="z")
v = NXfield(np.zeros(shape=(len(filenames), v0.shape[0], v0.shape[1]), dtype=v0.dtype), name="v")
v[0] = v0
for i in range(1, len(filenames)):
im = TIFF.open(filenames[i])
v[i] = im.read_image()
except ImportError:
im = Image.open(filenames[0])
dtype = np.dtype(np.uint16)
if im.mode == "I;32" or im.mode == "I":
dtype = np.dtype(np.uint32)
x = NXfield(range(im.size[0]), dtype=np.uint16, name="x")
y = NXfield(range(im.size[1]), dtype=np.uint16, name="y")
z = NXfield(range(1, len(filenames) + 1), dtype=np.uint16, name="z")
v = NXfield(np.zeros(shape=(len(filenames), im.size[1], im.size[0]), dtype=dtype), name="v")
v[0] = np.array(im.getdata(), dtype=dtype).reshape(im.size[1], im.size[0])
for i in range(1, len(filenames)):
im = Image.open(filenames[i])
v[i] = np.array(im.getdata(), dtype=dtype).reshape(im.size[1], im.size[0])
return NXentry(NXdata(v, (z, y, x)))
def convert_tiff_to_mat(Working_Directory, img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, img_size_crop_y2,\
median_filter_threshold, text_file):
#Get names of all tiff files in the directory
onlyfiles = [ f for f in os.listdir(Working_Directory)\
if (os.path.isfile(os.path.join(Working_Directory, f)) and f.find('.tif')>0 and f.find('T=')>=0)]
Working_Directory2 = Working_Directory.replace('/Thresholded_OB','')
for lst in xrange(1,np.size(onlyfiles, axis=0)+1):
tif1 = TIFF.open(Working_Directory+'T='+str(lst)+'.tif', mode='r') #Open multitiff
tif2 = TIFF.open(Working_Directory2+'T='+str(lst)+'.tif', mode='r') #Open non thresholded image for template
#Initialize data matrix based on number of planes in the multitiff
if lst==1:
count_z = 0
for image in tif1.iter_images():
count_z = count_z + 1
data_filtered = np.zeros((img_size_x-(img_size_crop_x1+img_size_crop_x2), img_size_y-(img_size_crop_y1+img_size_crop_y2), count_z,np.size(onlyfiles,0)), dtype=np.uint8)
data = np.zeros((img_size_x-(img_size_crop_x1+img_size_crop_x2), img_size_y-(img_size_crop_y1+img_size_crop_y2), count_z,np.size(onlyfiles,0)), dtype=np.uint8)
data_filtered, count_z = get_tif_images_filtered(data_filtered, lst, onlyfiles, text_file, tif1, \
img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, \
img_size_crop_y2,median_filter_threshold)
data = get_tif_images_raw(data, lst, onlyfiles, text_file, tif2, \
img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, \
img_size_crop_y2)
return data, data_filtered, count_z
def downsample_tif(filename_in, filename_out): input_tiff = TIFF.open(filename_in, mode = "r") output_tiff = TIFF.open(filename_out, mode = "w") counter = 0 for img in input_tiff.iter_images(): img = img[::2, ::2] output_tiff.write_image(img) counter += 1 if counter /100 == counter / 100.: print counter
def convert_tiff_to_mat(Working_Directory, img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, img_size_crop_y2,\
median_filter_threshold, text_file):
#Get names of all tiff files in the directory
onlyfiles = [ f for f in os.listdir(Working_Directory)\
if (os.path.isfile(os.path.join(Working_Directory, f)) and f.find('.tif')>0 and f.find('T=')>=0)]
Working_Directory2 = Working_Directory
average_bg_intensity = np.zeros((np.size(onlyfiles, axis=0),1))
for lst in xrange(1,np.size(onlyfiles, axis=0)+1):
tif1 = TIFF.open(Working_Directory+'T='+str(lst)+'.tif', mode='r') #Open multitiff
tif2 = TIFF.open(Working_Directory2+'T='+str(lst)+'.tif', mode='r') #Open non thresholded image for template
#Initialize data matrix based on number of planes in the multitiff
if lst==1:
count_z = 0
for image in tif1.iter_images():
count_z = count_z + 1
data_filtered_raw = np.zeros((img_size_x-(img_size_crop_x1+img_size_crop_x2), img_size_y-(img_size_crop_y1+img_size_crop_y2), count_z,np.size(onlyfiles,0)), dtype=np.uint8)
data_filtered_bgsub = np.zeros((img_size_x-(img_size_crop_x1+img_size_crop_x2), img_size_y-(img_size_crop_y1+img_size_crop_y2), count_z,np.size(onlyfiles,0)), dtype=np.uint8)
data = np.zeros((img_size_x-(img_size_crop_x1+img_size_crop_x2), img_size_y-(img_size_crop_y1+img_size_crop_y2), count_z,np.size(onlyfiles,0)), dtype=np.uint8)
data_filtered_raw, data_filtered_bgsub, count_z, average_bg_intensity[lst-1] = get_tif_images_filtered(data_filtered_raw, data_filtered_bgsub, lst, onlyfiles, text_file, tif1, \
img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, \
img_size_crop_y2,median_filter_threshold)
data = get_tif_images_raw(data, lst, onlyfiles, text_file, tif2, \
img_size_x, img_size_y, img_size_crop_x1,img_size_crop_x2, img_size_crop_y1, \
img_size_crop_y2)
average_bg_intensity = np.array(average_bg_intensity)
print np.shape(average_bg_intensity)
plt.plot(average_bg_intensity)
plt.show()
### If intensities are correlated to the Bg intensity, make them zero
for ii in xrange(0,np.size(data_filtered_raw,0)):
for jj in xrange(0,np.size(data_filtered_raw,1)):
for zz in xrange(0, np.size(data_filtered_raw,2)):
A = copy(data_filtered_raw[ii,jj,zz,:])
corr_A = np.corrcoef(A,np.squeeze(average_bg_intensity))
if corr_A[0,1] > 0.6 and sum(A)!=0:
A[:] = 0
data_filtered_bgsub[ii,jj,zz,:] = A
data[ii,jj,zz,:] = A
return data, data_filtered_bgsub, count_z, average_bg_intensity
def _write_single_image(full_uri, image_array):
"""
internally used method to write single tiff image
"""
tiff = TIFF.open(full_uri, mode='w')
tiff.write_image(image_array.astype('uint16'), write_rgb=False)
tiff.close()
def load_stack(self):
filename, _ = QtGui.QFileDialog.getOpenFileName(self, 'Open file', self.last_dir, "TIFF (*.tif *.tiff);; All files (*.*)")
if not len(filename):
return
self.init_data()
self.last_dir = os.path.dirname(filename)
self.image_location = filename
tif = TIFF.open(self.image_location)
pic = tif.read_image()
misc.imsave("temp.jpg", pic)
# TODO: wait message
#msg_box = QtGui.QMessageBox.information(self, "CellECT New Workspace", "Loading a large stack may take time. Press OK to continue.", defaultB = QtGui.QMessageBox.NoButton )
self.load_metadata_from_tif_info()
#pic = QtGui.QImage(filename)
#self.label_preview_img.setPixmap(QtGui.QPixmap.fromImage(pic))
self.label_preview_img.setPixmap("temp.jpg")
def create_plane(self,roi,sizeX,sizeY,sizeC,description):
tif_image = TIFF.open(self.filename, 'w')
im_dtype = np.dtype('uint8')
image_data = np.zeros((sizeC,sizeY,sizeX),dtype=im_dtype)
print 'num channels=',sizeC
for c in range(sizeC):
if c == 0:
tif_image.set_description(description)
channel = self.channels[c]
imarray = self.mkplane(roi,c)
print 'imarray shape:',imarray.shape
print("Writing channel: ", c+1)
if self.rotation == 0:
plane = imarray[0,:,:]
if self.rotation == 1:
plane = np.rot90(imarray[0,:,:],1)
elif self.rotation == 2:
plane = np.rot90(imarray[0,:,:],3)
image_data[c,:,:] = plane
# tif_image = TIFFimage(image_data,description=description)
# tif_image.write_file(self.filename,compression='lzw')
# del tif_image
tif_image.write_image(image_data, compression=self.compression.encode('ascii','ignore'))
tif_image.close()
def test_write_lzw():
for itype in [uint8, uint16, uint32, uint64,
int8, int16, int32, int64,
float32, float64,
complex64, complex128]:
#image = array([[1,2,3], [4,5,6]], itype)
image = array([range(10000)], itype)
#image = array([[0]*14000], itype)
fn = mktemp('.tif')
tif = TIFFimage(image)
tif.write_file(fn, compression='lzw')
del tif
#os.system('wc %s; echo %s' % (fn, image.nbytes))
tif = TIFF.open(fn,'r')
image2 = tif.read_image()
tif.close()
#os.remove(fn)
atexit.register(os.remove, fn)
for i in range(image.size):
if image.flat[i] != image2.flat[i]:
print `i, image.flat[i-5:i+5].view(dtype=uint8),image2.flat[i-5:i+5].view(dtype=uint8)`
break
assert image.dtype==image2.dtype
assert (image==image2).all()
def test_write_read():
for compression in [None, 'lzw']:
for itype in [uint8, uint16, uint32, uint64,
int8, int16, int32, int64,
float32, float64,
complex64, complex128]:
image = array([[1,2,3], [4,5,6]], itype)
fn = mktemp('.tif')
if 0:
tif = TIFF.open(fn,'w')
tif.write_image(image, compression=compression)
tif.close()
else:
tif = TIFFimage(image)
tif.write_file(fn, compression=compression)
del tif
tif = TIFFfile(fn)
data, names = tif.get_samples()
assert names==['sample0'],`names`
assert len(data)==1, `len(data)`
assert image.dtype==data[0].dtype, `image.dtype, data[0].dtype`
assert (image==data[0]).all()
#os.remove(fn)
atexit.register(os.remove, fn)
def __init__(self, filename, process_func=None, dtype=None,
as_grey=False):
self._filename = filename
self._tiff = TIFF.open(filename)
self._count = 1
while not self._tiff.LastDirectory():
self._count += 1
self._tiff.ReadDirectory()
# reset to 0
self._tiff.SetDirectory(0)
tmp = self._tiff.read_image()
if dtype is None:
self._dtype = tmp.dtype
else:
self._dtype = dtype
self._im_sz = tmp.shape
self._byte_swap = bool(self._tiff.IsByteSwapped())
self._validate_process_func(process_func)
self._as_grey(as_grey, process_func)
def image_to_tiff(src_path, des_path):
from libtiff import TIFF
image = Image.open(src_path)
image = image.convert('L')
tiff_out = TIFF.open(des_path, 'w')
tiff_out.write_image(image, compression=None, write_rgb=True)
tiff_out.close()
def __init__(self, filename):
self.filename = filename
self.tiff = TIFF.open(filename)
self._count = self._count_frames() # used once by TiffStack
self._shape = self.tiff.read_image().shape # used once by TiffStack
self.end = False
self.generator = self.tiff.iter_images()
def get_list_of_stimulus_name(Working_Directory):
## To find num z planes in each trial directory
num_z_planes = []
Name_stimulus = list()
Stimulus_Directories = [f for f in os.listdir(Working_Directory) if os.path.isdir(os.path.join(Working_Directory, f)) and f.find('Figures')<0]
for ii in xrange(0, size(Stimulus_Directories, axis = 0)):
Trial_Directories = [f for f in os.listdir(os.path.join(Working_Directory, Stimulus_Directories[ii]))\
if os.path.isdir(os.path.join(Working_Directory, Stimulus_Directories[ii], f)) and f.find('Figures')<0] #Get only directories
temp_num_z_planes = zeros((size(Trial_Directories)), dtype=int)
for jj in xrange(0, size(Trial_Directories, axis = 0)):
Image_Directory = os.path.join(Working_Directory, Stimulus_Directories[ii], Trial_Directories[jj], 'C=1')+filesep
tif = TIFF.open(Image_Directory +'T=1.tif', mode='r') #Open multitiff
count = 1
for image in tif.iter_images():
temp_num_z_planes[jj] = count
count = count+1
num_z_planes.append(temp_num_z_planes)
for ii in xrange(0, size(Stimulus_Directories, axis = 0)):
Trial_Directories = [f for f in os.listdir(os.path.join(Working_Directory, Stimulus_Directories[ii]))\
if os.path.isdir(os.path.join(Working_Directory, Stimulus_Directories[ii], f)) and f.find('Figures')<0] #Get only directories
for jj in xrange(0, size(Trial_Directories, axis = 0)):
for kk in xrange(0, num_z_planes[ii][jj]):
name_for_saving_figures = Stimulus_Directories[ii] + ' ' + Trial_Directories[jj] + ' Z=' + str(kk+1)
Name_stimulus.append(name_for_saving_figures)
return Name_stimulus
def save_images_to_disk():
print('Disk-saving thread active...')
n = 0
frameTimeOutputFile = open(planOutputPath+'frameTimes.txt','w')
frameTimeOutputFile.write('frameCount\t n\t frameCond\t frameT\t interval\n')
currdict = im_queue.get()
while currdict is not None:
frameTimeOutputFile.write('%i\t %i\t %i\t %s\t %s\n' % (int(currdict['frame']),n,int(currdict['cond']),currdict['time'],currdict['interval']))
if save_as_tiff:
fname = '%s/frame%i.tiff' % (dataOutputPath,int(currdict['frame']))
tiff = TIFF.open(fname, mode='w')
tiff.write_image(currdict['im'])
tiff.close()
elif save_as_npz:
np.savez_compressed('%s/test%d.npz' % (output_path, n), currdict['im'])
else:
fname = '%s/frame%i.tiff' % (dataOutputPath,int(currdict['frame']),)
with open(fname, 'wb') as f:
pkl.dump(currdict, f, protocol=pkl.HIGHEST_PROTOCOL)
# print 'DONE SAVING FRAME: ', currdict['frame'], n #fdict
n += 1
currdict = im_queue.get()
disk_writer_alive = False
#frameTimeOutputFile.close()
print('Disk-saving thread inactive...')
def save_images_to_disk():
print ("Disk-saving thread active...")
n = 0
frameTimeOutputFile = open(planOutputPath + "frameTimes.txt", "w")
frameTimeOutputFile.write("frameCount\t n\t frameCond\t frameT\t interval\n")
currdict = im_queue.get()
while currdict is not None:
frameTimeOutputFile.write(
"%i\t %i\t %i\t %s\t %s\n"
% (int(currdict["frame"]), n, int(currdict["cond"]), currdict["time"], currdict["interval"])
)
if save_as_tiff:
fname = "%s/frame%i.tiff" % (dataOutputPath, int(currdict["frame"]))
tiff = TIFF.open(fname, mode="w")
tiff.write_image(currdict["im"])
tiff.close()
elif save_as_npz:
np.savez_compressed("%s/test%d.npz" % (output_path, n), currdict["im"])
else:
fname = "%s/frame%i.tiff" % (dataOutputPath, int(currdict["frame"]))
with open(fname, "wb") as f:
pkl.dump(currdict, f, protocol=pkl.HIGHEST_PROTOCOL)
# print 'DONE SAVING FRAME: ', currdict['frame'], n #fdict
n += 1
currdict = im_queue.get()
disk_writer_alive = False
# frameTimeOutputFile.close()
print ("Disk-saving thread inactive...")
def load_info_from_tif(self, filename): tif = TIFF.open(filename) buf = StringIO. StringIO(tif.info()) img = tif.read_image() for image in tif.iter_images(): self.num_pages += 1 print self.num_pages self.numx = img.shape[0] self.numy = img.shape[1] line = buf.readline() self.memch = 0 while line: self.get_meta_from_line(line) line = buf.readline() if self.numt ==0: self.numt = 1 if self.numz ==0: self.numz =1 if self.numch == 0: self.numch = 1
def img_data_write(self, im_data, save_path):
list_r = []
list_g = []
list_b = []
for frame_data in im_data:
list_r.append(frame_data[:, :, 0])
list_g.append(frame_data[:, :, 1])
list_b.append(frame_data[:, :, 2])
tiff = TIFF.open(save_path + "r.tiff", "w")
tiff.write_image(list_r)
tiff.close()
tiff = TIFF.open(save_path + "g.tiff", "w")
tiff.write_image(list_g)
tiff.close()
tiff = TIFF.open(save_path + "b.tiff", "w")
tiff.write_image(list_b)
tiff.close()
def convert(imagesDir): if(os.path.isfile(imagesDir + "/labels.tif")): os.remove(imagesDir + "/labels.tif") if(os.path.isfile(imagesDir + "/predictions.tif")): os.remove(imagesDir + "/predictions.tif") trainNames = sorted(os.listdir(imagesDir), key=natural_key) labels = TIFF.open(imagesDir + "/labels.tif", "w") predictions = TIFF.open(imagesDir + "/predictions.tif", "w") for name in trainNames: image = misc.imread(imagesDir + "/" + name + '/labels.png').astype(np.float32) image = image / 255 labels.write_image(image) image = misc.imread(imagesDir + "/" + name + '/predictions.png').astype(np.float32) image = image / 255 predictions.write_image(image)
def read_2D_image(filename):
"""Read 2D image from filename and return it as a numpy array"""
f_tiff = TIFF.open(filename)
im_array = f_tiff.read_image()
return im_array
def test_write_read():
for itype in [uint8, uint16, uint32, uint64,
int8, int16, int32, int64,
float32, float64,
complex64, complex128]:
image = array([[1, 2, 3], [4, 5, 6]], itype)
fn = mktemp('.tif')
tif = TIFF.open(fn, 'w')
tif.write_image(image)
tif.close()
tif = TIFF.open(fn, 'r')
image2 = tif.read_image()
tif.close()
os.remove(fn)
assert image.dtype == image2.dtype
assert (image == image2).all()
def generate_augmented_image(self):
"""Generate augmented image with x,y,z points."""
out_fname = os.path.join(self.segment_me_dir, self.get_filename())
tif = TIFF.open(out_fname, 'w')
for z, (cell_wall_im, measurement_im) in enumerate(zip(self.cell_wall_images,
self.measurement_images)):
point_im = self.get_selected_points_image(z)
aug_im_rgb = np.array([cell_wall_im, measurement_im, point_im])
tif.write_image(aug_im_rgb, write_rgb=True)
def test_fft(self, dimension, expected):
input_name = data_path('sinogram-00005.tif')
output_name = self.tmp_path('r-00000.tif')
reader = self.get_task('reader', path=input_name)
writer = self.get_task('writer', filename=self.tmp_path('r-%05i.tif'))
fft = self.get_task('fft', dimensions=dimension)
ifft = self.get_task('ifft', dimensions=dimension)
self.graph.connect_nodes(reader, fft)
self.graph.connect_nodes(fft, ifft)
self.graph.connect_nodes(ifft, writer)
self.sched.run(self.graph)
ref_img = TIFF.open(input_name, mode='r').read_image()
res_img = TIFF.open(output_name, mode='r').read_image()
diff = np.sum(np.abs(ref_img - res_img))
self.assertLess(diff, expected)
def generate_answer_image(self):
"""Generate image augmented with the answer."""
out_fname = os.path.join(self.answer_dir, self.get_filename())
tif = TIFF.open(out_fname, 'w')
for z, (cell_wall_im, measurement_im) in enumerate(zip(self.cell_wall_images,
self.measurement_images)):
seg_im = self.get_segmentation_outline_image(z)
aug_im_rgb = np.array([cell_wall_im, measurement_im, seg_im])
tif.write_image(aug_im_rgb, write_rgb=True)
def test_slicing():
shape = (16, 16)
image = random.randint(255, size=shape)
for i in range(shape[0]):
for j in range(shape[1]):
image1 = image[:i+1,:j+1]
fn = mktemp('.tif')
tif = TIFF.open(fn,'w')
tif.write_image(image1)
tif.close()
tif = TIFF.open(fn,'r')
image2 = tif.read_image()
tif.close()
assert (image1==image2).all(),`i,j`
os.remove(fn)
def save_multipage_TIFF(input_filenames, output_filename):
"""Read in a list of input filenames and write out as a multi-page TIFF"""
raise NotImplemented("Need to update for multiple planes")
from libtiff import TIFF
f = TIFF.open(input_filenames[0], 'r')
first_img = f.read_image()
f.close()
output_array = np.empty(
[first_img.shape[0], first_img.shape[1], len(input_filenames)],
dtype=first_img.dtype)
for idx, filename in enumerate(input_filenames):
f = TIFF.open(filename, 'r')
output_array[:, :, idx] = f.read_image()
f.close()
f = TIFF.open(output_filename, 'w')
f.write_image(output_array)
f.close()
def _iter_pages(self):
if libtiff_available:
tiff = TIFF.open(self._path, 'r')
for frame in tiff.iter_images():
yield frame.astype(float)
else:
for frame in self.stack.pages:
yield frame.asarray(colormapped=False)
if libtiff_available:
tiff.close()
def readtiff(path):
frames = []
tif = TIFF.open(path, mode='r')
try:
for cc, tframe in enumerate(tif.iter_images()):
frames.append(tframe)
except EOFError:
pass
return frames
def plot_colormaps_each(maps, Working_Directory, name_for_saving_figures, pp, matched_pixels, unique_clrs):
Trial_Directories = [f for f in os.listdir(os.path.join(Working_Directory)) if os.path.isdir(os.path.join(Working_Directory, f)) and f.find('Figures')<0 and f.find('DataFrames')<0] #Get only directories
## To find num z planes in each trial directory
num_z_planes = np.zeros((np.size(Trial_Directories)), dtype=np.int)
for jj in xrange(0, np.size(Trial_Directories, axis = 0)):
Image_Directory = os.path.join(Working_Directory, Trial_Directories[jj],'C=1')+filesep
tif = TIFF.open(Image_Directory +'T=1.tif', mode='r') #Open multitiff
count = 1
for image in tif.iter_images():
num_z_planes[jj] = count
count = count+1
count = 0
count_trial1 = 0
for ii in xrange(0, np.size(Trial_Directories, axis = 0)):
count_subplot = 1
for jj in xrange(0, num_z_planes[ii]):
name_for_saving_figures1 = name_for_saving_figures + ' ' + Trial_Directories[ii] + ' Z=' + str(jj+1)
with sns.axes_style("darkgrid"):
fig2 = plt.subplot(2,2,count_subplot)
plt.imshow(maps[:,:,count,:])
plt.title(name_for_saving_figures1)
plt.axis('off')
count = count+1
count_subplot = count_subplot + 1
# If there are more than 6 panel, save and start new figure
if count_subplot == 5:
fig2 = plt.gcf()
pp.savefig(fig2)
plt.close()
count_subplot = 1
#Plot boxplots for each trial
if count_subplot <= 4:
with sns.axes_style("darkgrid"):
fig2 = plt.subplot(2,2,count_subplot)
fig2 = plot_boxplot(fig2, matched_pixels[:,count_trial1:count_trial1+num_z_planes[ii]], unique_clrs)
# plt.tight_layout()
fig2 = plt.gcf()
pp.savefig(fig2)
plt.close()
count_trial1 = count_trial1 + num_z_planes[ii]
else:
with sns.axes_style("darkgrid"):
fig3 = plt.figure()
fig3 = plot_boxplot(fig3, matched_pixels[:,count_trial1:count_trial1+num_z_planes[ii]], unique_clrs)
# plt.tight_layout()
fig3 = plt.gcf()
pp.savefig(fig3)
plt.close()
count_trial1 = count_trial1 + num_z_planes[ii]
def writeData(data, outputFilename):
"""
Writes data to a tiff, hdf5, or npy file.
Parameters
----------
data : 3D numpy array
The data to be written. Must have 3 dimensions, i.e. data.ndim == 3
outputFilename : string
The absolute or relative location of the particular file to be read
in. outputFilename must end in one of the following extensions
['.tif', '.tiff', '.hdf5', '.h5', '.npy'].
Notes
-----
- Data to be saved must be a 3D array.
"""
assert data.ndim==3, "Can only write out 3D hdf5, tiff, and numpy files"
filename = outputFilename.rstrip('/')
basePath, fName = os.path.split(filename)
name, ext = os.path.splitext(fName)
if basePath and not os.path.exists(basePath):
raise IOError, "Directory does not exist: %s" % (basePath)
if ext.lower() in ['.npy']:
try:
np.save(filename, np.array(data,dtype=np.float32))
except IOError:
raise IOError, "Error writing npy data to: \"%s\"" % filename
elif ext.lower() in ['.h5', '.hdf5']:
from h5py import File
try:
h5File = File(filename, "w")
except IOError:
raise IOError, "Error creating writable hdf5 file at: \"%s\"" % filename
shp = data.shape
comp="gzip"
compOpts=1
dset = h5File.create_dataset("/raw", shp, np.float32, data, chunks=shp, compression=comp, compression_opts=compOpts)
elif ext.lower() in ['.tif', '.tiff']:
from libtiff import TIFF
try:
tiff = TIFF.open(filename, 'w')
tiff.write_image(np.array(data,dtype=np.float32))
except IOError:
raise IOError, "Error writing tif file at: \"%s\"" % filename
tiff.close()
else:
assert False, "Can only write out 3D hdf5, tiff, and numpy files"
def write_libtiff(file_name, data):
"""Write a TIFF file using pylibtiff. Return the written file name."""
from libtiff import TIFF
tiff_file = TIFF.open(file_name, "w")
try:
tiff_file.write_image(data)
finally:
tiff_file.close()
return file_name
def read_tif(fn, div2=False, oldmethod=False, offset2=False):
"""Reads a 3D TIFF file and returns a 3D numpy matrix
from a path
Inputs :
- fn (string): the path of the 3D TIFF file
- div2 (bool): if `True`, only one every to planes in $z$ will be loaded
Returns :
- a 3D numpy matrix
"""
if offset2:
kp = 1
else:
kp = 0
if not oldmethod:
ti = TIFFfile(fn)
ti = ti.get_samples()[0][0].swapaxes(0, 2).swapaxes(0, 1)
if div2:
I = []
for i in range(ti.shape[2]):
j = ti[:, :, i]
if i % 2 == kp:
I.append(j)
ti = np.zeros((I[0].shape[0], I[0].shape[1], len(I)))
for (i, j) in enumerate(I):
ti[:, :, i] = j
return ti
else: # Kept for compatibility. Fails to load some 16 bits images.
im = TIFF.open(fn)
I = []
for (j, i) in enumerate(im.iter_images()):
if div2 and (j + 1) % 2 == 0:
pass
else:
I.append(i)
ret = np.zeros((I[0].shape[0], I[1].shape[1], len(I)))
for (i, ii) in enumerate(I):
ret[:, :, i] = ii
return ret
def read_single_file(self, filepath):
if not os.path.isfile(filepath):
raise Exception('no such file ' + filepath)
str_extension = filepath.split('.')[-1]
if str_extension == 'tif': #2d image or 3d stack of images
input_file = tif.open(self.filepath, mode='r')
volume = list(input_file.iter_images())
if len(volume) == 1:
file_data = input_file.read_image()
else: #we have a volume
file_data = np.zeros(
[volume[0].shape[0], volume[0].shape[1],
len(volume)],
dtype='float32')
for index, image in enumerate(volume):
file_data[:, :, index] = image
input_file.close()
elif str_extension in ('bmp', 'jpg', 'png'):
file_data = misc.imread(filepath)
elif str_extension == 'pkl':
filehandler = open(filepath, 'rb')
file_data = cPickle.load(filehandler)
filehandler.close()
elif str_extension == 'npz':
member = self.filemember
if not member:
member = 'arr_0'
file_data = np.load(self.filepath)[member]
elif str_extension == 'mat':
member = self.filemember
if not member:
member = 'arr_0'
file_data = loadmat(self.filepath)[member]
elif str_extension == 'json':
raise ValueError('json not coded yet')
elif str_extension == 'mnc':
file_data = nib.load(self.filepath).get_data()
else:
raise Exception('unsupported format: ' + str_extension)
return file_data
def load_img_file(file_path):
file_type = file_path.split('.')[-1]
img = None
# try libtiff import or use png instead if import fails
TIFF, file_type = try_tiff_import(file_type)
#tbd: tiff handling
if file_type == 'tiff':
obj = TIFF.open(file_path, 'r')
img = obj.read_image()
obj.close()
elif any(file_type in ext for ext in ('bmp', 'png', 'jpeg', 'jpg')):
img = np.asarray(Image.open(file_path))
elif not any(file_type in ext
for ext in ('bmp', 'png', 'tiff', 'jpeg', 'jpg')):
raise TypeError('Filetype %s not recognized' % file_type)
return img
def read_multi_page_tif_libtiff(fn):
"""Read a multi-page tif file into a numpy array.
Parameters
----------
fn : string
The filename of the image file being read.
Returns
-------
ar : numpy ndarray
The image stack in array format.
Notes
-----
Currently, only grayscale images are supported.
"""
pages = []
tif = TIFF.open(fn)
for img in tif.iter_images():
pages.append(img[..., newaxis])
return concatenate(pages, axis=-1)
def get_list_of_stimulus_name(Working_Directory):
## To find num z planes in each trial directory
num_z_planes = []
Name_stimulus = list()
Stimulus_Directories = [
f for f in os.listdir(Working_Directory)
if os.path.isdir(os.path.join(Working_Directory, f))
and f.find('Figures') < 0
]
for ii in xrange(0, size(Stimulus_Directories, axis=0)):
Trial_Directories = [f for f in os.listdir(os.path.join(Working_Directory, Stimulus_Directories[ii]))\
if os.path.isdir(os.path.join(Working_Directory, Stimulus_Directories[ii], f)) and f.find('Figures')<0] #Get only directories
temp_num_z_planes = zeros((size(Trial_Directories)), dtype=int)
for jj in xrange(0, size(Trial_Directories, axis=0)):
Image_Directory = os.path.join(
Working_Directory, Stimulus_Directories[ii],
Trial_Directories[jj], 'C=1') + filesep
tif = TIFF.open(Image_Directory + 'T=1.tif',
mode='r') #Open multitiff
count = 1
for image in tif.iter_images():
temp_num_z_planes[jj] = count
count = count + 1
num_z_planes.append(temp_num_z_planes)
for ii in xrange(0, size(Stimulus_Directories, axis=0)):
Trial_Directories = [f for f in os.listdir(os.path.join(Working_Directory, Stimulus_Directories[ii]))\
if os.path.isdir(os.path.join(Working_Directory, Stimulus_Directories[ii], f)) and f.find('Figures')<0] #Get only directories
for jj in xrange(0, size(Trial_Directories, axis=0)):
for kk in xrange(0, num_z_planes[ii][jj]):
name_for_saving_figures = Stimulus_Directories[
ii] + ' ' + Trial_Directories[jj] + ' Z=' + str(kk + 1)
Name_stimulus.append(name_for_saving_figures)
return Name_stimulus
def test_simple_slicing():
for planar_config in [1, 2]:
for compression in [None, 'lzw']:
for itype in [
uint8, uint16, uint32, uint64, int8, int16, int32, int64,
float32, float64, complex64, complex128
]:
image = random.randint(0, 100, size=(10, 6, 7)).astype(itype)
fn = mktemp('.tif')
if 0:
if planar_config == 2:
continue
tif = TIFF.open(fn, 'w')
tif.write_image(image, compression=compression)
tif.close()
else:
tif = TIFFimage(image)
tif.write_file(fn,
compression=compression,
planar_config=planar_config)
del tif
tif = TIFFfile(fn)
arr = tif.get_tiff_array()
data = arr[:]
assert len(data) == len(image), repr(len(data))
assert image.dtype == data.dtype, repr(
(image.dtype, data[0].dtype))
assert (image == data).all()
assert arr.shape == image.shape
_indices = [0, slice(None), slice(0, 2), slice(0, 5, 2)]
for _i0 in _indices[:1]:
for i1 in _indices:
for i2 in _indices:
sl = (_i0, i1, i2)
assert (arr[sl] == image[sl]).all(), repr(sl)
atexit.register(os.remove, fn)
def _save_datasets_as_mitiff(self, datasets, image_description,
gen_filename, **kwargs):
"""Put all together and save as a tiff file.
Include the special tags making it a mitiff file.
"""
from libtiff import TIFF
tif = TIFF.open(gen_filename, mode='wb')
tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8'))
cns = self.translate_channel_name.get(kwargs['sensor'], {})
if isinstance(datasets, list):
LOG.debug("Saving datasets as list")
for _cn in self.channel_order[kwargs['sensor']]:
for dataset in datasets:
if dataset.attrs['name'] == _cn:
# Need to possible translate channels names from satpy to mitiff
cn = cns.get(dataset.attrs['name'],
dataset.attrs['name'])
data = self._calibrate_data(
dataset, dataset.attrs['calibration'],
self.mitiff_config[kwargs['sensor']][cn]
['min-val'], self.mitiff_config[
kwargs['sensor']][cn]['max-val'])
tif.write_image(data.astype(np.uint8),
compression='deflate')
break
elif 'dataset' in datasets.attrs['name']:
self._save_single_dataset(datasets, cns, tif, kwargs)
elif self.palette:
LOG.debug("Saving dataset as palette.")
self._save_as_palette(tif, datasets, **kwargs)
else:
LOG.debug("Saving datasets as enhanced image")
self._save_as_enhanced(tif, datasets, **kwargs)
del tif
def image_array_to_tiff(image_dir, file_name, image_type, image_num):
"""
图像序列保存成tiff文件
:params image_dir：图像序列所在文件夹
:params file_name：要保存的tiff文件名
:params image_type:图像序列的类型
:params image_num:要保存的图像数目
:return
"""
from libtiff import TIFF
out_tiff = TIFF.open(file_name, mode='w')
# 这里假定图像名按序号排列
for i in range(0, image_num):
image_name = image_dir + str(i) + image_type
image_array = Image.open(image_name)
# 缩放成统一尺寸
img = image_array.resize((480, 480), Image.ANTIALIAS)
out_tiff.write_image(img, compression=None, write_rgb=True)
out_tiff.close()
return
def test(model, batch_size, model_type, save):
# evaluate
model.eval()
torch_device = torch.device('cuda')
# Loss and optimizer 定义损失函数， 使用的是最小平方误差函数
criterion = nn.MSELoss()
criterion = criterion.to(torch_device)
eval_loss_dict = []
if model_type == 1:
test_loader = get_data.get_test_loader(batch_size)
elif model_type == 2:
test_loader = get_data.get_gauss50_test_loader(batch_size)
else:
test_loader = get_data.get_mixed_test_loader(batch_size)
batch_tqdm = tqdm(enumerate(test_loader, 0), total=test_loader.__len__(), ncols=100)
for i, data in batch_tqdm:
inputs, labels = data
labels = get_data.get_size_labels(1, labels)
inputs = inputs.cuda()
labels = labels.cuda()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss = loss.cuda()
eval_loss_dict.append(loss.item())
if save:
for j in range(10):
save_imgname = 'G:/evocnn/image_set/output/' + str(i * 10 + j) + '.tif'
img = TIFF.open(save_imgname, 'w')
# 要保存之前先要改成CPU
output_img = outputs[j].cpu().detach().numpy()
img.write_image(output_img, write_rgb=True)
mean_test_loss = np.mean(eval_loss_dict)
std_test_loss = np.std(eval_loss_dict)
print("valid mean:{},std:{}".format(mean_test_loss, std_test_loss))
def load_images(self) -> Generator[np.ndarray, None, None]:
"""
Loads TIFF images from paths as numpy arrays.
"""
for path_ in self._paths:
if not os.path.exists(path=path_):
raise ImageNotFoundError(path_)
try:
tiff = TIFF.open(filename=path_, mode='r')
except IOError as e:
raise TiffLoadError() from e
for im in tiff.iter_images():
arr = np.array(im, np.uint8)
if len(arr.shape) > 2:
logging.warning(
'Image {} has not two dimensional greyscale data, '
'actually has shape of {} - filtering first dimension.'
.format(path_, arr.shape))
yield arr[:, :, 0]
yield arr
def process(tifffile, plot=True):
print('[INFO] Processing %s' % tifffile)
tf = TIFF.open(tifffile)
frames = tf.iter_images()
datafile = "%s_data.dat" % tifffile
datalines = []
for fi, frame in enumerate(frames):
# print( frame.shape )
binline = frame[0, :]
txtline = (''.join(([chr(x) for x in binline]))).rstrip()
print(txtline)
data = txtline.split(',')
if len(data) > 1:
datalines.append(txtline)
else:
print('x Frame %d has no arduino data' % fi)
with open(datafile, "w") as f:
f.write("\n".join(datalines))
print("[INFO] Wrote all data to %s" % datafile)
if plot:
plotFile(datafile)
def setup_mask_upload(ns):
"""
Uploads a mask over the uploaded image
"""
#check meta to make the mask
xml = ns.session.fetchxml(ns.image_uri, meta='')
size_x = int(xml.xpath('tag[@name="image_num_x"]/@value')[0])
size_y = int(xml.xpath('tag[@name="image_num_y"]/@value')[0])
mask = np.zeros([size_y, size_x])
mask[size_y/4:(3*size_y)/4,size_x/4:(3*size_x)/4] = 1 #create mask
#save mask
maskname = 'mask.tif'
tif = TIFF.open(os.path.join(ns.store_local_location, maskname), mode='w')
tif.write_image(mask)
content = upload_image_resource(ns.session, os.path.join(ns.store_local_location, maskname), u'%s/%s'%(TEST_PATH, maskname))
mask_resource_uri = content.attrib['uri']
mask_uri = '%s/image_service/image/%s'%(ns.root, content.attrib['resource_uniq'])
ns.mask_uri = mask_uri #image_service uri
ns.mask_resource_uri = mask_resource_uri #data_service uri
def readOneTif(path):
tif = TIFF.open(path, mode='r')
img = tif.read_image() # 此时img为一个numpy.array
# print(img.shape)
# count=0;
# imageAll=np.array([0])
# for image in tif.iter_images():
# if count==0:
# imageAll=image;
# else:
# imageAll=np.dstack((imageAll,image))
# # print(image.shape)
# count=count+1
# cv.imshow("main",image)
# cv.waitKey(0)
# cv.destroyAllWindows()
#
# print("z={}".format(count))
# print(imageAll.shape)
# cv.imshow("path",img);
# cv.waitKey(0);
# cv.destroyAllWindows()
return img
def processPic(self, imageName):
print("processPicCalled: " + imageName)
outW = 0
parsed_name = imageName.split('-')
laserNumber = int(parsed_name[1])
timeStamp = float(parsed_name[2])
while (outW < 500):
try:
image_file = TIFF.open(imageName, mode='r')
outW = 501
except:
outW += 1
image_gray = image_file.read_image()
outW = 0
total = long(0)
xrange = range(self.X1, self.X2)
yrange = range(self.Y1, self.Y2)
for x in xrange:
for y in yrange:
numberIn = image_gray[y, x]
total = total + numberIn
mean = float(total / (self.numberOfPixels))
image_file.close()
if laserNumber == 1:
self.quantizedTimeStamp = timeStamp
self.laser1 = mean
self.updateDataFile(self.quantizedTimeStamp, mean, 1, imageName,
"Data/rawData.txt")
if laserNumber == 2:
self.laser2 = mean
self.updateDataFile(self.quantizedTimeStamp, mean, 2, imageName,
"Data/rawData.txt")
if laserNumber == 3:
self.laser3 = mean
self.updateDataFile(self.quantizedTimeStamp, mean, 3, imageName,
"Data/rawData.txt")
self.updateDifferenceData(self.quantizedTimeStamp)
def tif2png(mapper, tiffile):
fullpath = os.path.abspath(tiffile)
dirname = os.path.dirname(fullpath)
name, ext = os.path.splitext(tiffile)
pngfile = os.path.join(dirname, name + '.png')
tif = TIFF.open(tiffile, mode='r')
for image in tif.iter_images():
matrix = image
shape = matrix.shape
img = []
for i in range(shape[0]):
row = []
for j in range(shape[1]):
v = matrix[i, j]
p = mapper.colormap(v)
row = row + p
img.append(row)
f = open(pngfile, 'wb')
w = png.Writer(shape[0], shape[1], transparent=mapper.transparent)
w.write(f, img)
f.close()
def __init__(self, filename, process_func=None, dtype=None, as_grey=False):
self._filename = filename
self._tiff = TIFF.open(filename)
self._count = 1
while not self._tiff.LastDirectory():
self._count += 1
self._tiff.ReadDirectory()
# reset to 0
self._tiff.SetDirectory(0)
tmp = self._tiff.read_image()
if dtype is None:
self._dtype = tmp.dtype
else:
self._dtype = dtype
self._im_sz = tmp.shape
self._byte_swap = bool(self._tiff.IsByteSwapped())
self._validate_process_func(process_func)
self._as_grey(as_grey, process_func)
def save_array_as_dtype(array, dtype, outpath):
"""
Saves an array with a specific datatype.
:param array: numpy array or array like
:param dtype: dtype such as "float32" or "uint16"
:param outpath: filepath to store tiff
:return:
"""
outdir = os.path.dirname(outpath)
if not os.path.exists(outdir):
os.mkdir(outdir)
# do some integer scaling
if dtype == "uint16":
array = scale_array(array, 0, 2**15)
elif dtype == "uint8":
array = scale_array(array, 0, 2**7)
array = array.astype(dtype)
tiff = TIFF.open(outpath, 'w+')
tiff.write_image(array)
tiff.close()
def test_save_dataset_with_calibration(self):
"""Test writer operation with calibration."""
import os
import numpy as np
from libtiff import TIFF
from satpy.writers.mitiff import MITIFFWriter
expected_ir = np.full((100, 200), 255)
expected_vis = np.full((100, 200), 0)
expected = np.stack([
expected_vis, expected_vis, expected_ir, expected_ir, expected_ir,
expected_vis
])
dataset = self._get_test_dataset_calibration()
w = MITIFFWriter(
filename=dataset.attrs['metadata_requirements']['file_pattern'],
base_dir=self.base_dir)
w.save_dataset(dataset)
filename = (
dataset.attrs['metadata_requirements']['file_pattern']).format(
start_time=dataset.attrs['start_time'])
tif = TIFF.open(os.path.join(self.base_dir, filename))
for i, image in enumerate(tif.iter_images()):
np.testing.assert_allclose(image, expected[i], atol=1.e-6, rtol=0)
def save_img_file(img, file_path, type=None):
img = place_dnp(img)
ext = os.path.splitext(file_path)[-1][1:]
if not type:
type = ext if ext == 'png' or ext == 'tiff' else 'tiff' if img.dtype == 'uint16' else 'png'
file_path = file_path + '.' + type if ext != type else file_path
if type == 'tiff':
obj = TIFF.open(file_path, mode='w')
obj.write_image(misc.uint16_norm(img),
compression=None,
write_rgb=True)
obj.close()
elif type == 'png' or type == 'bmp':
Image.fromarray(misc.uint8_norm(img)).save(file_path,
type,
optimize=True)
return True
raiz = "/mnt/c/Users/Verstand/Pictures/ThermalFolder/EDITED"
root, dirs, files = os.walk(raiz).next()
output_folder = os.path.join(root, "converted_20ago_sur_2")
try:
os.mkdir(output_folder)
except:
pass
count = 0
for file in files:
if file.endswith(".csv"):
img = Image.open(
os.path.join(os.path.dirname(root), file.replace(".csv", ".jpg")))
exif_dict = piexif.load(img.info["exif"])
exif_bytes = piexif.dump(exif_dict)
count += 1
my_data = np.genfromtxt(os.path.join(root, file), delimiter=',')
my_data = np.delete(my_data, 640, axis=1)
out_img = os.path.join(output_folder, file.replace(".csv", ".tiff"))
tiff = TIFF.open(out_img, mode="w")
tiff.write_image(my_data)
tiff.close()
# im = Image.fromarray(my_data)
# im.save(os.path.join(output_folder, file.replace(".csv",".tiff")), "TIFF", exif=exif_bytes)
else:
pass
print str(count) + " imagenes convertidas"
def create_data(src):
images = os.listdir(src)
total = int(len(images) / 2)
print(total)
imgs = np.ndarray((total, image_x, image_y, image_z), dtype=np.uint8)
imgs_mask = np.ndarray((total, image_x, image_y, image_z), dtype=np.uint8)
print('Creating training images...')
i = 0
for image_name in images:
if 'mask' in image_name:
continue
image_mask_name = image_name.split('.')[0] + "_mask.tif"
tif = TIFF.open(src + '/' + image_name, mode='r')
tif_mask = TIFF.open(src + '/' + image_mask_name, mode='r')
count = 0
imageAll = np.array([0])
for image in tif.iter_images():
if count == 0:
imageAll = image
else:
imageAll = np.dstack((imageAll, image))
count = count + 1
count_mask = 0
imageAll_mask = np.array([0])
for image_mask in tif_mask.iter_images():
if count_mask == 0:
imageAll_mask = image_mask
else:
imageAll_mask = np.dstack((imageAll_mask, image_mask))
count_mask = count_mask + 1
imgs[i] = imageAll
imgs_mask[i] = imageAll_mask
i = i + 1
print(imgs.shape)
print(imgs_mask.shape)
imgs = imgs.astype('float32')
imgs_mask = imgs_mask.astype('float32')
mean = np.mean(imgs)
std = np.std(imgs)
print('mean = ', mean)
print('std = ', std)
imgs -= mean
imgs /= std
imgs_mask /= 255.
# add one dimension, corresponding to the input channel
imgs = imgs[..., np.newaxis]
imgs_mask = imgs_mask[..., np.newaxis]
np.save('images_train.npy', imgs)
np.save('images_mask_train.npy', imgs_mask)
print('Creating data done!')
def _save_datasets_as_mitiff(self, datasets, image_description,
gen_filename, **kwargs):
"""Put all togehter and save as a tiff file with the special tag
making it a mitiff file.
"""
from libtiff import TIFF
tif = TIFF.open(gen_filename, mode='w')
tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8'))
cns = self.translate_channel_name.get(kwargs['sensor'], {})
if isinstance(datasets, list):
LOG.debug("Saving datasets as list")
for _cn in self.channel_order[kwargs['sensor']]:
for dataset in datasets:
if dataset.attrs['name'] == _cn:
reverse_offset = 0.
reverse_scale = 1.
if dataset.attrs[
'calibration'] == 'brightness_temperature':
reverse_offset = 255.
reverse_scale = -1.
dataset.data += KELVIN_TO_CELSIUS
# Need to possible translate channels names from satpy to mitiff
cn = cns.get(dataset.attrs['name'],
dataset.attrs['name'])
_data = reverse_offset + reverse_scale * ((
(dataset.data - float(self.mitiff_config[
kwargs['sensor']][cn]['min-val'])) /
(float(self.mitiff_config[kwargs['sensor']][cn][
'max-val']) - float(self.mitiff_config[
kwargs['sensor']][cn]['min-val']))) * 255.)
data = _data.clip(0, 255)
tif.write_image(data.astype(np.uint8),
compression='deflate')
break
elif 'dataset' in datasets.attrs['name']:
LOG.debug("Saving %s as a dataset.", datasets.attrs['name'])
for _cn in self.channel_order[kwargs['sensor']]:
for i, band in enumerate(datasets['bands']):
if band == _cn:
chn = datasets.sel(bands=band)
reverse_offset = 0.
reverse_scale = 1.
if chn.attrs['prerequisites'][i][
4] == 'brightness_temperature':
reverse_offset = 255.
reverse_scale = -1.
chn.data += KELVIN_TO_CELSIUS
# Need to possible translate channels names from satpy to mitiff
cn = cns.get(chn.attrs['prerequisites'][i][0],
chn.attrs['prerequisites'][i][0])
_data = reverse_offset + reverse_scale * ((
(chn.data - float(self.mitiff_config[
kwargs['sensor']][cn]['min-val'])) /
(float(self.mitiff_config[kwargs['sensor']][cn][
'max-val']) - float(self.mitiff_config[
kwargs['sensor']][cn]['min-val']))) * 255.)
data = _data.clip(0, 255)
tif.write_image(data.astype(np.uint8),
compression='deflate')
break
else:
LOG.debug("Saving datasets as enhanced image")
img = get_enhanced_image(datasets.squeeze(), enhance=self.enhancer)
for i, band in enumerate(img.data['bands']):
chn = img.data.sel(bands=band)
data = chn.values.clip(0, 1) * 254. + 1
data = data.clip(0, 255)
tif.write_image(data.astype(np.uint8), compression='deflate')
del tif
####################################################
for inputDir, outputDir, header, footer, firstFrame, lastFrame, scale in zip(
inputDirList, outputDirList, headerList, footerList, firstFrameList,
lastFrameList, scaleList):
if (rank == 0):
print inputDir
frameList = range(firstFrame, lastFrame + 1)
procFrameList = numpy.array_split(frameList, size)
for frame in procFrameList[rank]:
inputFile = inputDir + '/' + header + str(frame).zfill(4) + footer
outputFile = outputDir + '/' + header + str(frame).zfill(6) + footer
if ('png' in footer):
gImg = cv2.imread(inputFile, 0)
elif ('tif' in footer):
inTif = TIFF.open(inputFile, mode='r')
gImg = inTif.read_image()
del inTif
[row, col] = gImg.shape
if (scale < 1):
gImg = cv2.resize(gImg, (int(col * scale), int(row * scale)),
interpolation=cv2.INTER_AREA)
else:
gImg = cv2.resize(gImg, (int(col * scale), int(row * scale)),
interpolation=cv2.INTER_CUBIC)
outTif = TIFF.open(outputFile, mode='w')
outTif.write_image(gImg)
del outTif, gImg
comm.Barrier()
####################################################
def main():
out_dir_imgs = os.path.join(OUT_DIR, "imgs")
out_dir_gt = os.path.join(OUT_DIR, "gt")
if not os.path.exists(out_dir_imgs):
os.makedirs(out_dir_imgs)
if not os.path.exists(out_dir_gt):
os.makedirs(out_dir_gt)
indices_with_gt = []
next_index = 0
# so it's a random order
save_names = np.random.permutation(NUM_TRAIN)
num_img = 0
num_gt = 0
for img_path in sorted(glob.glob(SOURCE_IMGS_DIR + "/*.tif")):
print(("on img: %d %s" % (num_img, datetime.now())))
sys.stdout.flush()
num_img += 1
# each block's image and gt (if exists) share same name (from save_names)
handle = os.path.basename(img_path)[:-len(SOURCE_IMGS_SUFFIX)]
tif = TIFF.open(img_path, mode="r")
img = tif.read_image()
assert (img.shape == (6000, 6000, 4)) # uint8 np array, RGBIR
split_and_save_imgs(img,
next_index,
names=save_names,
cut=400,
rescale=0.5,
dir=out_dir_imgs)
gt_path = os.path.join(SOURCE_GT_DIR, handle + SOURCE_GT_SUFFIX)
if os.path.isfile(gt_path):
num_gt += 1
current_indices = list(
range(next_index, next_index + OUT_PER_SOURCE))
indices_with_gt += current_indices
gt_tif = TIFF.open(gt_path, mode="r")
gt = gt_tif.read_image()
assert (gt.shape == (6000, 6000, 3)) # uint8 np array, RGB
split_and_save_gts(gt,
next_index,
names=save_names,
cut=400,
rescale=0.5,
dir=out_dir_gt)
next_index += OUT_PER_SOURCE
# IID:
# unlabelled_train+labelled_train+labelled_test,
# labelled_train+labelled_test, labelled_train+labelled_test
# IID+:
# unlabelled_train+labelled_train, labelled_train, labelled_test
# make train (does not need GT) and test splits
# choose num_test randomly out of indices_with_gt for test, rest is train
test_inds = np.random.choice(indices_with_gt, size=NUM_TEST, replace=False)
unlabelled_train = open(os.path.join(OUT_DIR, "unlabelled_train.txt"),
"w+")
labelled_train = open(os.path.join(OUT_DIR, "labelled_train.txt"), "w+")
labelled_test = open(os.path.join(OUT_DIR, "labelled_test.txt"), "w+")
for i in range(NUM_TRAIN):
if i in test_inds:
file = labelled_test
elif i in indices_with_gt:
file = labelled_train
else:
file = unlabelled_train
file.write("%d\n" % save_names[i])
unlabelled_train.close()
labelled_train.close()
labelled_test.close()
assert (next_index == NUM_TRAIN)
assert (num_img == NUM_SOURCE_IMGS)
assert (num_gt == NUM_SOURCE_GT)
raster1 = rnc.build_raster(dataset_dir + f + '/imgs_1_rect/')
# raster2 = rnc.build_rasterRGB(dataset_dir + f + '/imgs_2_rect/')
raster2 = rnc.build_raster(dataset_dir + f + '/imgs_2_rect/')
raster = np.concatenate((raster1, raster2), axis=2)
padded_raster = rnc.pad(raster, img_size)
shape = (padded_raster.shape[0], padded_raster.shape[1], classes)
padded_shapes.append(shape)
crops = rnc.crop(padded_raster, img_size, stride)
num_crops.append(len(crops))
train_images = train_images + crops
# Read change maps to get the ground truths
train_labels = []
unpadded_shapes = []
for f in folders:
cm = TIFF.open(labels_dir + f + '/cm/' + f + '-cm.tif').read_image()
cm = np.expand_dims(cm, axis=2)
cm -= 1 # the change map has values 1 for unchange and 2 for change ---> scale back to 0 and 1
unpadded_shapes.append(cm.shape)
cm = cm.flatten()
train_labels.append(cm)
# Create inputs and labels for the Neural Network
inputs = np.asarray(train_images)
y_true = np.asarray(train_labels)
print(y_true.shape)
# Load the model
model = load_model(
save_dir + model_name + '.h5',
custom_objects={'weighted_bce_dice_loss': cdm.weighted_bce_dice_loss})
def download_image_data(image_ids_or_dataset_id,
dataset=False,
download_original=False,
channel=None,
z_stack=0,
frame=0,
coord=(0, 0),
width=0,
height=0,
region_spec='rectangle',
skip_failed=False,
download_tar=False,
omero_host='idr.openmicroscopy.org',
omero_secured=False,
config_file=None):
if config_file is None: # IDR connection
omero_username = '******'
omero_password = '******'
else: # other omero instance
with open(config_file) as f:
cfg = json.load(f)
omero_username = cfg['username']
omero_password = cfg['password']
if omero_username == "" or omero_password == "":
omero_username = '******'
omero_password = '******'
if not download_original and region_spec not in ['rectangle', 'center']:
raise ValueError(
'Got unknown value "{0}" as region_spec argument'.format(
region_spec))
with ExitStack() as exit_stack:
conn = exit_stack.enter_context(
BlitzGateway(omero_username,
omero_password,
host=omero_host,
secure=omero_secured))
# exit_stack.callback(conn.connect().close)
if download_tar:
# create an archive file to write images to
archive = exit_stack.enter_context(
tarfile.open('images.tar', mode='w'))
tempdir = exit_stack.enter_context(TemporaryDirectory())
if dataset:
dataset_warning_id = 'Dataset-ID: {0}'.format(
image_ids_or_dataset_id[0])
try:
dataset_id = int(image_ids_or_dataset_id[0])
except ValueError:
image_ids = None
else:
try:
dataset = conn.getObject("Dataset", dataset_id)
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), dataset_warning_id, warn_skip=True)
else:
raise
else:
image_ids = [image.id for image in dataset.listChildren()]
if image_ids is None:
if skip_failed:
warn(
'Unable to find a dataset with this ID in the '
'database.',
dataset_warning_id,
warn_skip=True)
else:
raise ValueError(
'{0}: Unable to find a dataset with this ID in the '
'database. Aborting!'.format(dataset_warning_id))
else:
# basic argument sanity checks and adjustments
prefix = 'image-'
# normalize image ids by stripping off prefix if it exists
image_ids = [
iid[len(prefix):] if iid[:len(prefix)] == prefix else iid
for iid in image_ids_or_dataset_id
]
for image_id in image_ids:
image_warning_id = 'Image-ID: {0}'.format(image_id)
try:
image_id = int(image_id)
except ValueError:
image = None
else:
try:
image = conn.getObject("Image", image_id)
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
if image is None:
if skip_failed:
warn(
'Unable to find an image with this ID in the '
'database.',
image_warning_id,
warn_skip=True)
continue
raise ValueError(
'{0}: Unable to find an image with this ID in the '
'database. Aborting!'.format(image_warning_id))
if not download_original:
try:
# try to extract image properties
# if anything goes wrong here skip the image
# or abort.
image_name = os.path.splitext(image.getName())[0]
image_warning_id = '{0} (ID: {1})'.format(
image_name, image_id)
if region_spec == 'rectangle':
tile = get_clipping_region(image, *coord, width,
height)
elif region_spec == 'center':
tile = get_clipping_region(
image, *_center_to_ul(*coord, width, height))
ori_z, z_stack = z_stack, confine_plane(image, z_stack)
ori_frame, frame = frame, confine_frame(image, frame)
num_channels = image.getSizeC()
if channel is None:
channel_index = 0
else:
channel_index = find_channel_index(image, channel)
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
# region sanity checks and warnings
if tile[2] < width or tile[3] < height:
# The downloaded image region will have smaller dimensions
# than the specified width x height.
warn(
'Downloaded image dimensions ({0} x {1}) will be smaller '
'than the specified width and height ({2} x {3}).'.
format(tile[2], tile[3], width,
height), image_warning_id)
# z-stack sanity checks and warnings
if z_stack != ori_z:
warn(
'Specified image plane ({0}) is out of bounds. Using {1} '
'instead.'.format(ori_z, z_stack), image_warning_id)
# frame sanity checks and warnings
if frame != ori_frame:
warn(
'Specified image frame ({0}) is out of bounds. Using '
'frame {1} instead.'.format(ori_frame, frame),
image_warning_id)
# channel index sanity checks and warnings
if channel is None:
if num_channels > 1:
warn(
'No specific channel selected for multi-channel '
'image. Using first of {0} channels.'.format(
num_channels), image_warning_id)
else:
if channel_index == -1 or channel_index >= num_channels:
if skip_failed:
warn(
str(channel) +
' is not a known channel name for this image.',
image_warning_id,
warn_skip=True)
continue
else:
raise ValueError(
'"{0}" is not a known channel name for image {1}. '
'Aborting!'.format(channel, image_warning_id))
# download and save the region as TIFF
fname = '__'.join([image_name, str(image_id)] +
[str(x) for x in tile])
try:
if fname[-5:] != '.tiff':
fname += '.tiff'
fname = fname.replace(' ', '_')
im_array = get_image_array(image, tile, z_stack,
channel_index, frame)
if download_tar:
fname = os.path.join(tempdir, fname)
try:
tiff = TIFF.open(fname, mode='w')
tiff.write_image(im_array)
finally:
tiff.close()
# move image into tarball
if download_tar:
archive.add(fname, os.path.basename(fname))
os.remove(fname)
except Exception as e:
if skip_failed:
# respect skip_failed on unexpected errors
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
else:
try:
# try to extract image properties
# if anything goes wrong here skip the image
# or abort.
image_name = os.path.splitext(image.getName())[0]
image_warning_id = '{0} (ID: {1})'.format(
image_name, image_id)
original_image_name = image.getFileset().listFiles(
)[0].getName()
fname = image_name + "__" + str(
image_id) + os.path.splitext(original_image_name)[1]
fname = fname.replace(' ', '_')
fname = fname.replace('/', '_')
download_directory = "./"
if download_tar:
download_directory = tempdir
with cli_login("-u", omero_username, "-s", omero_host,
"-w", omero_password) as cli:
cli.invoke([
"download", f"Image:{image_id}", download_directory
])
if cli.rv != 0:
raise Exception("Download failed.")
# This will download to download_directory/original_image_name
os.rename(
os.path.join(download_directory, original_image_name),
os.path.join(download_directory, fname))
# move image into tarball
if download_tar:
archive.add(os.path.join(download_directory, fname),
os.path.basename(fname))
os.remove(os.path.join(download_directory, fname))
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
def process(**kwargs):
tifffile = kwargs['input']
print('[INFO] Processing %s' % tifffile)
tf = TIFF.open(tifffile)
frames = tf.iter_images()
datalines, arduinoData = [], []
trialType = 'NA'
# If classfier is found. Use it.
classifierFile = kwargs.get('classifier', '')
cascade = None
if classifierFile and os.path.isfile(classifierFile):
cascade = cv2.CascadeClassifier(classifierFile)
classifierValues = []
for fi, frame in enumerate(frames):
# print( frame.shape )
binline = frame[0, :]
txtline = (''.join(([chr(x) for x in binline]))).rstrip()
data = txtline.split(',')
if len(data) > 1:
datalines.append(data)
if len(data) > 2:
arduinoData.append(data)
if cascade is not None:
blinkValue = detectEye(frame, cascade, kwargs['plot'])
classifierValues.append(blinkValue)
tvec, blinkVec, velocityVec = [], [], []
for l in datalines:
if len(l) > 5:
if l[-4] == 'CS+':
tone, led = int(l[5]), int(l[6])
if led == 1:
trialType = 'LIGHT NO SOUND'
elif tone == 1:
trialType = 'SOUND NO LIGHT'
try:
tvec.append(parse_timestamp(l[0]))
blinkVec.append(float(l[-1]))
velocityVec.append(float(l[-2]))
except Exception as e:
print('[WARN] Failed to parse data line %s. Ignoring' % l)
print('\t Error was %s' % e)
if cascade is not None:
blinkVec = classifierValues
mean_ = sum(blinkVec) / len(blinkVec)
cspST, cspET = get_status_timeslice(arduinoData, 'CS+')
assert cspST
usST, usET = get_status_timeslice(arduinoData, 'PUFF')
probeTs = get_status_timeslice(arduinoData, 'PROB')
if probeTs[0] is None and probeTs[1] is None:
# Nowhere we found PROB in trial.
isProbe = False
else:
print('[INFO] Trial %s is a PROBE trial' % tifffile)
isProbe = True
# Computer perfornace of this trial.
baseline, signal, hasLearnt = compute_learning_yesno(tvec, blinkVec, cspST)
if hasLearnt:
print('|| Learning in %s' % tifffile)
temp = os.path.join(os.path.dirname(tifffile), '_results')
datadir = kwargs.get('outdir', temp)
if not os.path.isdir(datadir):
os.makedirs(datadir)
if kwargs.get('plot', False):
ax = plt.subplot(211)
if cspET > cspST:
ax.plot([cspST, cspET], [mean_, mean_])
if (usET and usST) and usET > usST:
ax.plot([usST, usET], [mean_, mean_])
ax.plot(tvec, blinkVec, label='Learning? %s' % hasLearnt)
ax.axhline(y=np.mean(baseline) + np.std(baseline))
plt.title('Trial Type=%s' % trialType)
plt.legend(framealpha=0.4)
plt.title(os.path.basename(sys.argv[1]), fontsize=8)
ax2 = plt.subplot(212, sharex=ax)
ax2.plot(tvec, velocityVec, label='Speed')
plt.xlabel('Time')
plt.ylabel('cm/sec (really?)')
outfile = os.path.join(datadir, '%s.png' % os.path.basename(tifffile))
plt.tight_layout(pad=3)
plt.savefig(outfile)
plt.close()
print('Saved to %s' % outfile)
# Write processed data to pickle.
print('[INFO] Trial type %s' % trialType)
res = dict(time=tvec,
blinks=blinkVec,
cs=[cspST, cspET],
us=[usST, usET],
did_learn=hasLearnt,
is_probe=isProbe,
trial_type=trialType)
pickleFile = os.path.join(datadir,
'%s.pickle' % os.path.basename(tifffile))
with open(pickleFile, 'wb') as pF:
pickle.dump(res, pF)
print('[INFO] Wrote pickle %s' % pickleFile)
return res
def main():
usage = """deepflow_simmatrix.py [input_dir]
Expects refframes directory in [input_dir] to be already filled with i-frames
Example:
./deepflow_simmatrix_viz.py ./simmatrix/20160412/
For help:
./deepflow_simmatrix_viz.py -h
Ben Lansdell
01/04/2016
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'path_in', help='input directory with frames already placed in it')
args = parser.parse_args()
#Test code
class Args:
pass
args = Args()
args.path_in = './simmatrix/20160412/'
#Get all files in
iframes = sorted(glob(args.path_in + 'refframes/*.tif'))
nF = len(iframes)
nx_tile = 64
#Load images
images = []
for fn_in in iframes: # do stuff with image
# to open a tiff file for reading:
tif = TIFF.open(fn_in, mode='r')
image = tif.read_image()
image = cv2.resize(image, (nx_tile, nx_tile))
images.append(image)
tif.close()
D = np.zeros((nF, nF))
#Run DeepFlow
for i in range(nF):
im1 = iframes[i]
fn1 = int(os.path.splitext(os.path.basename(im1))[0].split('_')[1])
for j in range(i + 1, nF):
im2 = iframes[j]
fn2 = int(os.path.splitext(os.path.basename(im2))[0].split('_')[1])
print("DeepFlow between frame %d and %d" % (fn1, fn2))
flow_in1 = args.path_in + 'corrmatrix/%04d_%04d.flo' % (fn1, fn2)
flow_in2 = args.path_in + 'corrmatrix/%04d_%04d.flo' % (fn2, fn1)
#Read in flow
flow1 = readFlo(flow_in1)
flow2 = readFlo(flow_in2)
ny, nx = flow1.shape[0:2]
#For each run we compute the average reconstruction error
fwdmeshy, fwdmeshx = [
a.astype(np.float32)
for a in np.meshgrid(np.arange(nx), np.arange(ny))
]
#Perturb mesh grid by forward flow
#Round to integers
fwdx = fwdmeshx + np.ceil(flow1[:, :, 0])
fwdy = fwdmeshy + np.ceil(flow1[:, :, 1])
fwdx = np.maximum(0, np.minimum(nx - 1, fwdx))
fwdy = np.maximum(0, np.minimum(nx - 1, fwdy))
#Look up flow field using this perturbed map
fwdremapx = fwdx + flow2[fwdx.astype(int), fwdy.astype(int), 0]
fwdremapy = fwdy + flow2[fwdx.astype(int), fwdy.astype(int), 1]
fwdremapx -= fwdmeshx
fwdremapy -= fwdmeshy
fwderr = np.sqrt(fwdremapx**2 + fwdremapy**2)
#fwdtracked = fwderr < threshold
D[i, j] = np.mean(fwderr)
D[j, i] = D[i, j]
# Plot distance matrix.
fig1 = pylab.figure(figsize=(8, 8))
axmatrix1 = fig1.add_axes([0.3, 0.1, 0.6, 0.6])
im = axmatrix1.matshow(D,
aspect='auto',
origin='lower',
cmap=pylab.cm.YlGnBu)
fn_out = args.path_in + 'similarity.png'
fig1.savefig(fn_out)
#Once we've loaded this data we view the similarity matrix
fig = pylab.figure(figsize=(8, 8))
ax1 = fig.add_axes([0.09, 0.1, 0.2, 0.6])
Y = sch.linkage(D, method='centroid')
Z1 = sch.dendrogram(Y, orientation='right')
ax1.set_xticks([])
ax1.set_yticks([])
# Compute and plot second dendrogram.
ax2 = fig.add_axes([0.3, 0.71, 0.6, 0.2])
Y = sch.linkage(D, method='single')
Z2 = sch.dendrogram(Y)
ax2.set_xticks([])
ax2.set_yticks([])
# Plot distance matrix.
axmatrix = fig.add_axes([0.3, 0.1, 0.6, 0.6])
idx1 = Z1['leaves']
idx2 = Z2['leaves']
D = D[idx1, :]
D = D[:, idx2]
im = axmatrix.matshow(D,
aspect='auto',
origin='lower',
cmap=pylab.cm.YlGnBu)
axmatrix.set_xticks([])
axmatrix.set_yticks([])
# Plot colorbar.
axcolor = fig.add_axes([0.91, 0.1, 0.02, 0.6])
pylab.colorbar(im, cax=axcolor)
#fig.show()
fn_out = args.path_in + 'dendrogram.png'
fig.savefig(fn_out)
#Make another version of this plot but bigger and with frame snippets
#Load all the iframe images and resize
fn_out_d1 = args.path_in + 'dend_d1_tile.png'
fn_out_d2 = args.path_in + 'dend_d2_tile.png'
im_d1 = images[idx1[0]]
im_d2 = images[idx2[0]]
for idx in range(1, nF):
im_d1 = np.hstack((im_d1, images[idx1[idx]]))
im_d2 = np.hstack((im_d2, images[idx2[idx]]))
cv2.imwrite(fn_out_d1, im_d1)
cv2.imwrite(fn_out_d2, im_d2)
def download_image_data(image_ids,
channel=None,
z_stack=0,
frame=0,
coord=(0, 0),
width=0,
height=0,
region_spec='rectangle',
skip_failed=False,
download_tar=False):
# basic argument sanity checks and adjustments
prefix = 'image-'
# normalize image ids by stripping off prefix if it exists
image_ids = [
iid[len(prefix):] if iid[:len(prefix)] == prefix else iid
for iid in image_ids
]
if region_spec not in ['rectangle', 'center']:
raise ValueError(
'Got unknown value "{0}" as region_spec argument'.format(
region_spec))
# connect to idr
conn = BlitzGateway('public',
'public',
host='idr.openmicroscopy.org',
secure=True)
conn.connect()
if download_tar:
# create an archive file to write images to
archive = tarfile.open('images.tar', mode='w')
try:
for image_id in image_ids:
image_warning_id = 'Image-ID: {0}'.format(image_id)
try:
image_id = int(image_id)
except ValueError:
image = None
else:
try:
image = conn.getObject("Image", image_id)
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
if image is None:
if skip_failed:
warn(
'Unable to find an image with this ID in the '
'database.',
image_warning_id,
warn_skip=True)
continue
raise ValueError(
'{0}: Unable to find an image with this ID in the '
'database. Aborting!'.format(image_warning_id))
try:
# try to extract image properties
# if anything goes wrong here skip the image
# or abort.
image_name = os.path.splitext(image.getName())[0]
image_warning_id = '{0} (ID: {1})'.format(image_name, image_id)
if region_spec == 'rectangle':
tile = get_clipping_region(image, *coord, width, height)
elif region_spec == 'center':
tile = get_clipping_region(
image, *_center_to_ul(*coord, width, height))
ori_z, z_stack = z_stack, confine_plane(image, z_stack)
ori_frame, frame = frame, confine_frame(image, frame)
num_channels = image.getSizeC()
if channel is None:
channel_index = 0
else:
channel_index = find_channel_index(image, channel)
except Exception as e:
# respect skip_failed on unexpected errors
if skip_failed:
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
# region sanity checks and warnings
if tile[2] < width or tile[3] < height:
# The downloaded image region will have smaller dimensions
# than the specified width x height.
warn(
'Downloaded image dimensions ({0} x {1}) will be smaller '
'than the specified width and height ({2} x {3}).'.format(
tile[2], tile[3], width, height), image_warning_id)
# z-stack sanity checks and warnings
if z_stack != ori_z:
warn(
'Specified image plane ({0}) is out of bounds. Using {1} '
'instead.'.format(ori_z, z_stack), image_warning_id)
# frame sanity checks and warnings
if frame != ori_frame:
warn(
'Specified image frame ({0}) is out of bounds. Using '
'frame {1} instead.'.format(ori_frame, frame),
image_warning_id)
# channel index sanity checks and warnings
if channel is None:
if num_channels > 1:
warn(
'No specific channel selected for multi-channel '
'image. Using first of {0} channels.'.format(
num_channels), image_warning_id)
else:
if channel_index == -1 or channel_index >= num_channels:
if skip_failed:
warn(str(channel) +
' is not a known channel name for this image.',
image_warning_id,
warn_skip=True)
continue
else:
raise ValueError(
'"{0}" is not a known channel name for image {1}. '
'Aborting!'.format(channel, image_warning_id))
# download and save the region as TIFF
fname = '__'.join([image_name, str(image_id)] +
[str(x) for x in tile])
try:
if fname[-5:] != '.tiff':
fname += '.tiff'
fname = fname.replace(' ', '_')
im_array = get_image_array(image, tile, z_stack, channel_index,
frame)
# pack images into tarball
if download_tar:
tar_img = Image.fromarray(im_array)
# Use TemporaryFile() for intermediate storage of images.
# TO DO: could this be improved by using
# SpooledTemporaryFile with a suitable max_size?
with TemporaryFile() as buf:
tar_img.save(buf, format='TIFF')
tarinfo = tarfile.TarInfo(name=fname)
buf.seek(0, 2)
tarinfo.size = buf.tell()
buf.seek(0)
archive.addfile(tarinfo=tarinfo, fileobj=buf)
else: # save image as individual file
try:
tiff = TIFF.open(fname, mode='w')
tiff.write_image(im_array)
finally:
tiff.close()
except Exception as e:
if skip_failed:
# respect skip_failed on unexpected errors
warn(str(e), image_warning_id, warn_skip=True)
continue
else:
raise
finally:
# close the archive file,if it is created
if download_tar:
archive.close()
# Close the connection
conn.close()
def _save_datasets_as_mitiff(self, datasets, image_description,
gen_filename, **kwargs):
"""Put all together and save as a tiff file.
Include the special tags making it a mitiff file.
"""
from libtiff import TIFF
tif = TIFF.open(gen_filename, mode='wb')
tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8'))
cns = self.translate_channel_name.get(kwargs['sensor'], {})
if isinstance(datasets, list):
LOG.debug("Saving datasets as list")
for _cn in self.channel_order[kwargs['sensor']]:
for dataset in datasets:
if dataset.attrs['name'] == _cn:
# Need to possible translate channels names from satpy to mitiff
cn = cns.get(dataset.attrs['name'],
dataset.attrs['name'])
data = self._calibrate_data(
dataset, dataset.attrs['calibration'],
self.mitiff_config[kwargs['sensor']][cn]
['min-val'], self.mitiff_config[
kwargs['sensor']][cn]['max-val'])
tif.write_image(data.astype(np.uint8),
compression='deflate')
break
elif 'dataset' in datasets.attrs['name']:
LOG.debug("Saving %s as a dataset.", datasets.attrs['name'])
if len(datasets.dims) == 2 and (all('bands' not in i
for i in datasets.dims)):
# Special case with only one channel ie. no bands
# Need to possible translate channels names from satpy to mitiff
# Note the last index is a tuple index.
cn = cns.get(datasets.attrs['prerequisites'][0]['name'],
datasets.attrs['prerequisites'][0]['name'])
data = self._calibrate_data(
datasets,
datasets.attrs['prerequisites'][0].get('calibration'),
self.mitiff_config[kwargs['sensor']][cn]['min-val'],
self.mitiff_config[kwargs['sensor']][cn]['max-val'])
tif.write_image(data.astype(np.uint8), compression='deflate')
else:
for _cn_i, _cn in enumerate(
self.channel_order[kwargs['sensor']]):
for band in datasets['bands']:
if band == _cn:
chn = datasets.sel(bands=band)
# Need to possible translate channels names from satpy to mitiff
# Note the last index is a tuple index.
cn = cns.get(
chn.attrs['prerequisites'][_cn_i]['name'],
chn.attrs['prerequisites'][_cn_i]['name'])
data = self._calibrate_data(
chn, chn.attrs['prerequisites'][_cn_i].get(
'calibration'), self.mitiff_config[
kwargs['sensor']][cn]['min-val'],
self.mitiff_config[
kwargs['sensor']][cn]['max-val'])
tif.write_image(data.astype(np.uint8),
compression='deflate')
break
elif self.palette:
LOG.debug("Saving dataset as palette.")
self._save_as_palette(tif, datasets, **kwargs)
else:
LOG.debug("Saving datasets as enhanced image")
self._save_as_enhanced(tif, datasets, **kwargs)
del tif
