def generate_extra_data(n=3000, file_name=None, load_file=None, seed=None, rotate=True, emboss=True, elastic=False):
print "Creating {} new images".format(n)
if load_file:
if not load_file.startswith("data/"):
load_file = "data/" + load_file
return np.load(load_file +".npy"), np.load(load_file+"_targets.npy")
if seed:
np.random.seed(seed)
X = np.load("data/mnist_train_extra_inputs.npy")
Y = np.load("data/mnist_train_extra_outputs.npy")
positions = np.random.random_integers(0, X.shape[0]-1, n)
transformed = X[positions,:]
targets = Y[positions]
final = np.zeros((n,48*48))
angles = np.random.uniform(0,359, n)
for i in range(n):
temp = transformed[i,:].reshape((28,28))
temp = imresize(temp, (48,48))
if rotate:
temp = imrotate(temp, angles[i], reshape=False)
if emboss:
temp = imfilter(temp,"emboss")
if elastic:
temp = elastic_distortion(temp, size=48)
final[i,:] = temp.flatten()
if file_name:
if not file_name.startswith("data/"):
file_name = "data/" + file_name
np.save(file_name, final)
np.save(file_name + "_targets", targets)
return final, targets
def generate_extra_data(X=None,
Y=None,
n=3000,
seed=None,
folder=io.DATA_FOLDER + 'train',
file_name=None,
rotate=True,
noise=True):
print "Creating {} new images".format(n)
if X is None and Y is None:
# automatically resizes images to (720, 1028)
X, _, _ = io.create_test_train_split(samples=-1,
auto_resize=True,
folder=folder)
X, Y = io.dataset_dict_to_array(X)
if seed:
np.random.seed(seed)
positions = np.random.randint(0, X.shape[0] - 1, n)
transformed = X[positions, :]
targets = Y[positions, :]
final = np.zeros_like(transformed)
final_resized_190_320 = np.zeros((n, 190, 320, 3))
final_resized_244_244 = np.zeros_like((n, 244, 244, 3))
final_resized_48_48 = np.zeros_like((n, 48, 48, 3))
# slight pertubations in the angle of the image
angles = np.random.uniform(-20, +20, n)
for i in range(n):
temp = transformed[i, :]
if rotate:
temp = imrotate(temp, angles[i], reshape=False)
if noise:
rand_noise = np.random.randint(0, 255, temp.shape)
keepers = np.random.binomial(1, 0.95, size=temp.shape)
temp = temp * keepers + rand_noise * (1 - keepers)
final[i, :] = temp
final_resized_190_320[i, :] = imresize(temp, size=(190, 320))
final_resized_244_244[i, :] = imresize(temp, size=(244, 244))
final_resized_48_48[i, :] = imresize(temp, size=(48, 48))
if i % 1000 == 0:
print "Created {} images.".format(i)
if file_name:
if not file_name.startswith("./data/"):
file_name = folder + file_name
np.save(file_name + '.npy', final)
np.save(file_name + '_190_320.npy', final_resized_190_320)
np.save(file_name + '_244_244.npy', final_resized_244_244)
np.save(file_name + '_48_48.npy', final_resized_48_48)
np.save(file_name + "_targets.npy", targets)
return final, targets
def do(image, anno=None):
"""
Does the rotation for image and rectangles for 90 degrees counterclockwise.
Args:
image (Image): The target image to rotate.
anno (Annotation): The annotations to be rotated with the image.
Returns (tuple):
Rotated image and annotations for it.
"""
w = image.shape[1]
new_image = imrotate(image, 90, reshape=True)
if anno is not None:
anno.rects = [al.AnnoRect(r.y1, w - r.x2, r.y2, w - r.x1) for r in anno.rects]
return new_image, anno
def do(image, anno=None):
"""
Does the rotation for image and rectangles for 90 degrees counterclockwise.
Args:
image (Image): The target image to rotate.
anno (Annotation): The annotations to be rotated with the image.
Returns (tuple):
Rotated image and annotations for it.
"""
w = image.shape[1]
new_image = imrotate(image, 90, reshape=True)
if anno is not None:
anno.rects = [al.AnnoRect(r.y1, w - r.x2, r.y2, w - r.x1) for r in anno.rects]
return new_image, anno
def perturb_modified_digits(X, Y, n=3000, seed=None, alpha=8, sigma=3):
print "Modifying {} digits".format(n)
positions = np.random.random_integers(0, X.shape[0]-1, n)
transformed = X[positions,:]
targets = Y[positions]
final = np.zeros((n,48*48))
angles = np.random.uniform(0,359, n)
for i in range(n):
temp = transformed[i,:].reshape((48,48))
temp = imrotate(temp, angles[i], reshape=False)
temp = elastic_distortion(temp, size=48, alpha=alpha, sigma=sigma)
final[i,:] = temp.flatten()
return final, targets
def find_similar(exp_proj):
tic = time.time()
grid_size = model_proj_imgs.shape[2]
likelihood = np.zeros((grid_size, 360))
psi_step = 3
psi_angles = np.arange(0, 360, psi_step)
for i in range(grid_size):
ref_proj = model_proj_imgs[:, :, i]
for j in psi_angles:
rot_ref_proj = imrotate(ref_proj, j, reshape=False, mode='nearest')
prob = np.mean((exp_proj - rot_ref_proj) ** 2 / (-2 * exp_proj ** 2))
likelihood[i, j] = prob
idx = np.argmax(likelihood)
row_idx, col_idx = np.unravel_index(idx, (grid_size, 360))
toc = time.time() - tic
if toc > 3:
print('\r{0} forloops cost {1:.4f} seconds.'.format(
grid_size * 360, toc), end='')
return row_idx, col_idx
def find_similar(exp_proj):
tic = time.time()
grid_size = model_proj_imgs.shape[2]
likelihood = np.zeros((grid_size, 360))
psi_step = 3
psi_angles = np.arange(0, 360, psi_step)
for i in range(grid_size):
ref_proj = model_proj_imgs[:, :, i]
for j in psi_angles:
rot_ref_proj = imrotate(ref_proj, j, reshape=False, mode='nearest')
prob = np.mean((exp_proj - rot_ref_proj)**2 / (-2 * exp_proj**2))
likelihood[i, j] = prob
idx = np.argmax(likelihood)
row_idx, col_idx = np.unravel_index(idx, (grid_size, 360))
toc = time.time() - tic
if toc > 3:
print('\r{0} forloops cost {1:.4f} seconds.'.format(
grid_size * 360, toc),
end='')
return row_idx, col_idx
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast):
"""
Leginondb will be queried to get the 'best' defocus estimate on a per-image basis.
Confident defoci will be gathered and unconfident defoci will be interpolated.
Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package.
A plot of the defocus values will is made.
A CTF plot using the mean defocus is made.
"""
try:
apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...')
os.chdir(rundir)
raw_path=rundir+'/raw/'
ctfdir='%s/ctf_correction/' % rundir
os.system("mkdir %s" % ctfdir)
defocus_file_full=ctfdir+seriesname+'_defocus.txt'
tilt_file_full=ctfdir+seriesname+'_tilts.txt'
image_list_full=ctfdir+seriesname+'_images.txt'
uncorrected_stack=ctfdir+'stack_uncorrected.mrc'
corrected_stack=ctfdir+'stack_corrected.mrc'
out_full=ctfdir+'out'
log_file_full=ctfdir+'ctf_correction.log'
project='ap'+projectid
sinedon.setConfig('appiondata', db=project)
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
cs = tiltdata[0]['scope']['tem']['cs']*1000
voltage = int(tiltdata[0]['scope']['high tension']/1000)
if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected
sys.exit()
#Get tilt azimuth
cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename)
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(tilt_azimuth, err) = proc.communicate()
tilt_azimuth=float(tilt_azimuth)
estimated_defocus=[]
for image in range(len(ordered_imagelist)):
imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False)
try:
if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation
estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2)
else: #Poorly estimated. Guess its value later
estimated_defocus.append(999999999)
except: #No data. Guess its value later
estimated_defocus.append(999999999)
#Find mean and stdev to prune out confident defocus values that are way off
defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus)
avg=np.array(defocus_stats_list).mean()
stdev=np.array(defocus_stats_list).std()
good_tilts=[]
good_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
good_defocus_list.append(defocus)
good_tilts.append(tilt)
#Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways.
x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000)
s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1)
y=s(x)
#Make defocus list with good values and interpolations for bad values
finished_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
finished_defocus_list.append(int(round(defocus)))
else: #Interpolate
finished_defocus_list.append(int(round(y[int(round(tilt))])))
new_avg=np.array(finished_defocus_list).mean()
new_stdev=np.array(finished_defocus_list).std()
#Write defocus file, tilt file, and image list file for ctfphaseflip and newstack
f = open(defocus_file_full,'w')
f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0]))
for i in range(1,len(tilts)):
f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i]))
f.close()
f = open(tilt_file_full,'w')
for tilt in tilts:
f.write("%.2f\n" % tilt)
f.close()
mrc_list=[]
presetname=tiltdata[0]['preset']['name']
for image in ordered_mrc_files:
mrcname=presetname+image.split(presetname)[-1]
mrc_list.append(raw_path+'/'+mrcname)
f = open(image_list_full,'w')
f.write("%d\n" % len(tilts))
for filename in mrc_list:
f.write(filename+'\n')
f.write("%d\n" % 0)
f.close()
#Rotate and pad images so that they are treated properly by ctfphaseflip.
apDisplay.printMsg("Preparing images for IMOD...")
for filename in mrc_list:
image=mrc.read(filename)
dimx=len(image[0])
dimy=len(image)
#First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image
image=np.rot90(image, k=-1)
#Rotate image and write
image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic
mrc.write(image, filename)
f = open(log_file_full,'w')
#Make stack for correction,phase flip, extract images, replace images
cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full)
f.write("%s\n\n" % cmd1)
print cmd1
subprocess.check_call([cmd1], shell=True)
cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full)
f.write("\n\n%s\n\n" % cmd2)
print cmd2
subprocess.check_call([cmd2], shell=True)
cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full)
f.write("\n\n%s\n\n" % cmd3)
print cmd3
subprocess.check_call([cmd3], shell=True)
f.write("\n\n")
apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images")
new_images=glob.glob(ctfdir+'out*mrc')
new_images.sort()
#Unrotate and unpad images
for filename in new_images:
image=mrc.read(filename)
image=imrotate(image, tilt_azimuth, order=1)
image=np.rot90(image, k=1)
big_dimx=len(image[0])
big_dimy=len(image)
cropx1=int((big_dimx-dimx)/2)
cropx2=int(dimx+(big_dimx-dimx)/2)
cropy1=int((big_dimy-dimy)/2)
cropy2=int(dimy+(big_dimy-dimy)/2)
image=image[cropy1:cropy2,cropx1:cropx2]
mrc.write(image, filename)
for i in range(len(new_images)):
cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i])
f.write("%s\n" % cmd4)
os.system(cmd4)
#Make plots
apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full)
apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs)
cleanup="rm %s %s" % (uncorrected_stack, corrected_stack)
os.system(cleanup)
output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber)
output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000)
f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close()
apDisplay.printMsg(output1)
apDisplay.printMsg(output2)
apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber)
except subprocess.CalledProcessError:
apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.")
except SystemExit:
apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber)
except:
apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")