def generate_extra_data(n=3000, file_name=None, load_file=None, seed=None, rotate=True, emboss=True, elastic=False):
	print "Creating {} new images".format(n)
	if load_file:
		if not load_file.startswith("data/"):
			load_file = "data/" + load_file
		return np.load(load_file +".npy"), np.load(load_file+"_targets.npy")
	if seed:
		np.random.seed(seed)
	X = np.load("data/mnist_train_extra_inputs.npy")
	Y = np.load("data/mnist_train_extra_outputs.npy")
	positions = np.random.random_integers(0, X.shape[0]-1, n)
	transformed = X[positions,:]
	targets = Y[positions]
	final = np.zeros((n,48*48))
	angles = np.random.uniform(0,359, n)
	for i in range(n):
		temp = transformed[i,:].reshape((28,28))
		temp = imresize(temp, (48,48))
		if rotate:
			temp = imrotate(temp, angles[i], reshape=False)
		if emboss:
			temp = imfilter(temp,"emboss")
		if elastic:
			temp = elastic_distortion(temp, size=48)
		final[i,:] = temp.flatten()
	if file_name:
		if not file_name.startswith("data/"):
			file_name = "data/" + file_name
		np.save(file_name, final)
		np.save(file_name + "_targets", targets)
	return final, targets
示例#2
0
def generate_extra_data(X=None,
                        Y=None,
                        n=3000,
                        seed=None,
                        folder=io.DATA_FOLDER + 'train',
                        file_name=None,
                        rotate=True,
                        noise=True):
    print "Creating {} new images".format(n)

    if X is None and Y is None:
        # automatically resizes images to (720, 1028)
        X, _, _ = io.create_test_train_split(samples=-1,
                                             auto_resize=True,
                                             folder=folder)
        X, Y = io.dataset_dict_to_array(X)

    if seed:
        np.random.seed(seed)

    positions = np.random.randint(0, X.shape[0] - 1, n)
    transformed = X[positions, :]
    targets = Y[positions, :]
    final = np.zeros_like(transformed)
    final_resized_190_320 = np.zeros((n, 190, 320, 3))
    final_resized_244_244 = np.zeros_like((n, 244, 244, 3))
    final_resized_48_48 = np.zeros_like((n, 48, 48, 3))

    # slight pertubations in the angle of the image
    angles = np.random.uniform(-20, +20, n)
    for i in range(n):
        temp = transformed[i, :]

        if rotate:
            temp = imrotate(temp, angles[i], reshape=False)
        if noise:
            rand_noise = np.random.randint(0, 255, temp.shape)
            keepers = np.random.binomial(1, 0.95, size=temp.shape)
            temp = temp * keepers + rand_noise * (1 - keepers)

        final[i, :] = temp
        final_resized_190_320[i, :] = imresize(temp, size=(190, 320))
        final_resized_244_244[i, :] = imresize(temp, size=(244, 244))
        final_resized_48_48[i, :] = imresize(temp, size=(48, 48))

        if i % 1000 == 0:
            print "Created {} images.".format(i)

    if file_name:
        if not file_name.startswith("./data/"):
            file_name = folder + file_name
        np.save(file_name + '.npy', final)
        np.save(file_name + '_190_320.npy', final_resized_190_320)
        np.save(file_name + '_244_244.npy', final_resized_244_244)
        np.save(file_name + '_48_48.npy', final_resized_48_48)
        np.save(file_name + "_targets.npy", targets)
    return final, targets
 def do(image, anno=None):
     """
     Does the rotation for image and rectangles for 90 degrees counterclockwise.
     Args:
         image (Image): The target image to rotate.
         anno (Annotation): The annotations to be rotated with the image.
     Returns (tuple):
         Rotated image and annotations for it.
     """
     w = image.shape[1]
     new_image = imrotate(image, 90, reshape=True)
     if anno is not None:
         anno.rects = [al.AnnoRect(r.y1, w - r.x2, r.y2, w - r.x1) for r in anno.rects]
     return new_image, anno
示例#4
0
 def do(image, anno=None):
     """
     Does the rotation for image and rectangles for 90 degrees counterclockwise.
     Args:
         image (Image): The target image to rotate.
         anno (Annotation): The annotations to be rotated with the image.
     Returns (tuple):
         Rotated image and annotations for it.
     """
     w = image.shape[1]
     new_image = imrotate(image, 90, reshape=True)
     if anno is not None:
         anno.rects = [al.AnnoRect(r.y1, w - r.x2, r.y2, w - r.x1) for r in anno.rects]
     return new_image, anno
def perturb_modified_digits(X, Y, n=3000, seed=None, alpha=8, sigma=3):
	print "Modifying {} digits".format(n)
	positions = np.random.random_integers(0, X.shape[0]-1, n)
	transformed = X[positions,:]
	targets = Y[positions]

	final = np.zeros((n,48*48))
	angles = np.random.uniform(0,359, n)
	for i in range(n):
		temp = transformed[i,:].reshape((48,48))
		temp = imrotate(temp, angles[i], reshape=False)
		temp = elastic_distortion(temp, size=48, alpha=alpha, sigma=sigma)
		final[i,:] = temp.flatten()
	return final, targets
示例#6
0
def find_similar(exp_proj):
    tic = time.time()
    grid_size = model_proj_imgs.shape[2]
    likelihood = np.zeros((grid_size, 360))
    psi_step = 3
    psi_angles = np.arange(0, 360, psi_step)
    for i in range(grid_size):
        ref_proj = model_proj_imgs[:, :, i]
        for j in psi_angles:
            rot_ref_proj = imrotate(ref_proj, j, reshape=False, mode='nearest')
            prob = np.mean((exp_proj - rot_ref_proj) ** 2 / (-2 * exp_proj ** 2))
            likelihood[i, j] = prob
    idx = np.argmax(likelihood)
    row_idx, col_idx = np.unravel_index(idx, (grid_size, 360))
    toc = time.time() - tic
    if toc > 3:
        print('\r{0} forloops cost {1:.4f} seconds.'.format(
            grid_size * 360, toc), end='')
    return row_idx, col_idx
示例#7
0
def find_similar(exp_proj):
    tic = time.time()
    grid_size = model_proj_imgs.shape[2]
    likelihood = np.zeros((grid_size, 360))
    psi_step = 3
    psi_angles = np.arange(0, 360, psi_step)
    for i in range(grid_size):
        ref_proj = model_proj_imgs[:, :, i]
        for j in psi_angles:
            rot_ref_proj = imrotate(ref_proj, j, reshape=False, mode='nearest')
            prob = np.mean((exp_proj - rot_ref_proj)**2 / (-2 * exp_proj**2))
            likelihood[i, j] = prob
    idx = np.argmax(likelihood)
    row_idx, col_idx = np.unravel_index(idx, (grid_size, 360))
    toc = time.time() - tic
    if toc > 3:
        print('\r{0} forloops cost {1:.4f} seconds.'.format(
            grid_size * 360, toc),
              end='')
    return row_idx, col_idx
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast):
	"""
	Leginondb will be queried to get the 'best' defocus estimate on a per-image basis.
	Confident defoci will be gathered and unconfident defoci will be interpolated.
	Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package.
	A plot of the defocus values will is made.
	A CTF plot using the mean defocus is made.
	"""
	try:
		apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...')
		os.chdir(rundir)
		raw_path=rundir+'/raw/'
		ctfdir='%s/ctf_correction/' % rundir
		os.system("mkdir %s" % ctfdir)
		defocus_file_full=ctfdir+seriesname+'_defocus.txt'
		tilt_file_full=ctfdir+seriesname+'_tilts.txt'
		image_list_full=ctfdir+seriesname+'_images.txt'
		uncorrected_stack=ctfdir+'stack_uncorrected.mrc'
		corrected_stack=ctfdir+'stack_corrected.mrc'
		out_full=ctfdir+'out'
		log_file_full=ctfdir+'ctf_correction.log'
		
		project='ap'+projectid
		sinedon.setConfig('appiondata', db=project)
		sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
		tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
		tiltdata = apTomo.getImageList([tiltseriesdata])
		
		frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
		tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
		cs = tiltdata[0]['scope']['tem']['cs']*1000
		voltage = int(tiltdata[0]['scope']['high tension']/1000)
		if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected
			sys.exit()
		
		#Get tilt azimuth
		cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename)
		proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
		(tilt_azimuth, err) = proc.communicate()
		tilt_azimuth=float(tilt_azimuth)
		
		estimated_defocus=[]
		for image in range(len(ordered_imagelist)):
			imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False)
			try:
				if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation
					estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2)
				else:  #Poorly estimated. Guess its value later
					estimated_defocus.append(999999999)
			except:  #No data. Guess its value later
				estimated_defocus.append(999999999)
		
		#Find mean and stdev to prune out confident defocus values that are way off
		defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus)
		avg=np.array(defocus_stats_list).mean()
		stdev=np.array(defocus_stats_list).std()
		
		good_tilts=[]
		good_defocus_list=[]
		for tilt, defocus in zip(tilts, estimated_defocus):
			if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
				good_defocus_list.append(defocus)
				good_tilts.append(tilt)
		
		#Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways.
		x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000)
		s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1)
		y=s(x)
		
		#Make defocus list with good values and interpolations for bad values
		finished_defocus_list=[]
		for tilt, defocus in zip(tilts, estimated_defocus):
			if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
				finished_defocus_list.append(int(round(defocus)))
			else:  #Interpolate
				finished_defocus_list.append(int(round(y[int(round(tilt))])))
		
		new_avg=np.array(finished_defocus_list).mean()
		new_stdev=np.array(finished_defocus_list).std()
		
		#Write defocus file, tilt file, and image list file for ctfphaseflip and newstack
		f = open(defocus_file_full,'w')
		f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0]))
		for i in range(1,len(tilts)):
			f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i]))
		f.close()
		
		f = open(tilt_file_full,'w')
		for tilt in tilts:
			f.write("%.2f\n" % tilt)
		f.close()
		
		mrc_list=[]
		presetname=tiltdata[0]['preset']['name']
		for image in ordered_mrc_files:
			mrcname=presetname+image.split(presetname)[-1]
			mrc_list.append(raw_path+'/'+mrcname)
		f = open(image_list_full,'w')
		f.write("%d\n" % len(tilts))
		for filename in mrc_list:
			f.write(filename+'\n')
			f.write("%d\n" % 0)
		f.close()
		
		#Rotate and pad images so that they are treated properly by ctfphaseflip.
		apDisplay.printMsg("Preparing images for IMOD...")
		for filename in mrc_list:
			image=mrc.read(filename)
			dimx=len(image[0])
			dimy=len(image)
			#First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image
			image=np.rot90(image, k=-1)
			#Rotate image and write
			image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic
			mrc.write(image, filename)
		
		f = open(log_file_full,'w')
		#Make stack for correction,phase flip, extract images, replace images
		cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full)
		f.write("%s\n\n" % cmd1)
		print cmd1
		subprocess.check_call([cmd1], shell=True)
		
		cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full)
		f.write("\n\n%s\n\n" % cmd2)
		print cmd2
		subprocess.check_call([cmd2], shell=True)
		
		cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full)
		f.write("\n\n%s\n\n" % cmd3)
		print cmd3
		subprocess.check_call([cmd3], shell=True)
		f.write("\n\n")
		
		apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images")
		new_images=glob.glob(ctfdir+'out*mrc')
		new_images.sort()
		
		#Unrotate and unpad images
		for filename in new_images:
			image=mrc.read(filename)
			image=imrotate(image, tilt_azimuth, order=1)
			image=np.rot90(image, k=1)
			big_dimx=len(image[0])
			big_dimy=len(image)
			cropx1=int((big_dimx-dimx)/2)
			cropx2=int(dimx+(big_dimx-dimx)/2)
			cropy1=int((big_dimy-dimy)/2)
			cropy2=int(dimy+(big_dimy-dimy)/2)
			image=image[cropy1:cropy2,cropx1:cropx2]
			mrc.write(image, filename)
		
		for i in range(len(new_images)):
			cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i])
			f.write("%s\n" % cmd4)
			os.system(cmd4)
		
		#Make plots
		apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full)
		apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs)
		
		cleanup="rm %s %s" % (uncorrected_stack, corrected_stack)
		os.system(cleanup)
		output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber)
		output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000)
		f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close()
		apDisplay.printMsg(output1)
		apDisplay.printMsg(output2)
		apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber)
		
	except subprocess.CalledProcessError:
		apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.")
	
	except SystemExit:
		apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber)

	except:
		apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")