Example #1
0
def main():

	inputs = sys.argv[1:-1]
	output = sys.argv[-1]

	idx = 0
	m = 1
	M = len(inputs)
	for stack in inputs:

		N = spx.EMUtil.get_image_count(stack)

		for n in np.arange(N):

			img = spx.EMData()
			img.read_image(stack, n)

			# Write image to the particle stack:
			if idx == 0:

				# If this is the first image, we initiate as a normal .mrcs stack.
				img.write_image(output, idx)

			else:

				# Subsequent images are directly appended to the file:
				with open(output, 'ab') as mrcf:
					spx.EMNumPy.em2numpy(img).tofile(mrcf)

			idx += 1

			print 'Appended image %d/%d from stack %d/%d. Total images appended: %d.' % (n+1, N, m, M, idx)

		m += 1

	# Maybe it is better to update the NZ in header with the MRCheaderUpdate.py script?

	header = ioMRC.readMRCHeader( output )
	header['dimensions'][0] = idx
	# Now we write the header back:
	with open( output, 'rb+' ) as mrcf: 
		ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )

	print 'Done!'
Example #2
0
def main():

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <.mrc(s) file>

	Given an existing .mrc(s) file, will evaluate its size and number of images/slices (NZ) and UPDATE IN-PLACE (CAUTION!) corresponding header information.

	Reference for MRC format description:
	http://bio3d.colorado.edu/imod/doc/mrc_format.txt

	This script is based on Robb McLeod's ioMRC.py available at:
	https://bitbucket.org/emmrcz/python-mrcz

	NOTE: Only vanilla MRC(S) files are supported (i.e. compressed MRC formats not supported)

	"""

	parser = OptionParser(usage)
	
	parser.add_option( "--stats", action="store_true", help="Will update the statistics of the file (minval, maxval, avgval, std). May take some time depending on file size.", default=False )
	parser.add_option( "--endian", metavar="le", type="string", default='le', help="MRC file endianness (default is little-endian)" )

	(options, args) = parser.parse_args()

	command = ' '.join(sys.argv)

	if options.endian == 'le':
		endchar = '<'
	else:
		endchar = '>'

	fpath = args[0] # The file to be updated
	hsize = 1024 # Standard MRC header is 1024 bytes long

	S = os.path.getsize( fpath ) # Get the current size of the file in bytes
	# print S-hsize
	header = ioMRC.readMRCHeader( fpath, slices=0 )[0] # Read in the current header of the file
	if 'meta' not in header.keys():
		header['meta'] = None
	
	sizeof = np.dtype(header['dtype']).itemsize # Get the size in bytes of each value in the current MRC format
	# print sizeof

	with open(fpath, 'rb+') as f:

		NX = header['dimensions'][2]
		NY = header['dimensions'][1]
		NZ = header['dimensions'][0]
		print('Current dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZ))
		NZnew = ( S - hsize ) / ( NX*NY*sizeof ) # This will yield the actual number of Z slices
		print('New dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZnew))
		header['dimensions'][0] = NZnew

		if options.stats:
		# Let's update the statistics of the stack/volume:
			header['meanImage'] = 0.0

			f.seek(1024)
			for i in np.arange(NZnew):

				# We will read one image or slice at a time in order to save memory:
				image = np.fromfile( f, dtype=header['dtype'], count=NX*NY )

				minImage = np.min(image)
				maxImage = np.max(image)
				meanImage = np.mean(image)

				if minImage < header['minImage']:
					header['minImage'] = minImage

				if maxImage > header['maxImage']:
					header['maxImage'] = maxImage

				header['meanImage'] += meanImage

			header['meanImage'] /= NZnew


		# Now we write the header back:
		ioMRC.writeMRCHeader( f, header, slices=0, endchar=endchar )
Example #3
0
def main():

	# Get arguments:
	folders = sys.argv[1]
	merge_dirfile = sys.argv[2]
	png_path = sys.argv[3]
	stack_path = sys.argv[4]
	stack_rootname = sys.argv[5]
	box_size = int(sys.argv[6])
	phaori_shift = np.array([float(sys.argv[7].split(',')[0]), float(sys.argv[7].split(',')[1])]) # How many degrees to offset the phase origins in order to get a protein in the center of the particle for a zero-tilt image
	apix = float(sys.argv[8]) # pixel size in Angstroems
	microscope_voltage = float(sys.argv[9]) # microscope voltage in kV
	microscope_cs = float(sys.argv[10]) # microscope spherical aberration in mm
	ampcontrast = 1-float(sys.argv[11])**2  # amplitude contrast of CTF (obtained here from phase contrast)
	magnification = float(sys.argv[12])
	sigcc = float(sys.argv[13])
	if sys.argv[14] == 'y':
		invert_micrograph = True
	else:
		invert_micrograph = False
	if sys.argv[15] == 'y':
		normalize_box = True
	else:
		normalize_box = False
	if sys.argv[16] == 'y':
		calculate_defocus_tilted = True
	else:
		calculate_defocus_tilted = False
	if sys.argv[17] == 'y':
		save_phase_flipped = True
	else:
		save_phase_flipped = False
	if sys.argv[18] == 'y':
		save_wiener_filtered = True
	else:
		save_wiener_filtered = False
	sigma = float(sys.argv[19]) # Sigma for normalization of the windowed images (if normalize_box == True)
	if sys.argv[20] == 'Defocus/Lattice':
		tiltgeom = ''
	elif sys.argv[20] == 'Defocus':
		tiltgeom = 'DEFOCUS_'
	elif sys.argv[20] == 'Lattice':
		tiltgeom = 'LATTICE_'
	elif sys.argv[20] == 'Merge':
		tiltgeom = 'MERGE_'
	if sys.argv[21] == 'Micrograph':
		ctfcor = True
		stack_rootname = stack_rootname + '_ctfcor'
	else:
		ctfcor = False
	if sys.argv[22] == 'y':
		save_pick_fig = True
	else:
		save_pick_fig = False
	if sys.argv[23] == 'y':
		use_masked_image = True
	else:
		use_masked_image = False
	n_threads = int(sys.argv[24])
	if n_threads < 1:
		n_threads = 1
	this_thread = int(sys.argv[25])
	# End arguments

	f = open(merge_dirfile,'r')
	img_dirs = f.readlines()
	f.close()

	# Constant parameters to be written on the .par file of this dataset:
	shx = 0.0
	shy = 0.0
	occ = 100.0
	logp = 0
	sig = 0.5 # This has nothing to do with the normalization SIGMA!
	score = 0.0
	chg = 0.0

	idx = 0
	# box_fail = 0
	phaori_err = 0
	prog = 0.0

	N = len(img_dirs)

	batch_size = round(float(N)/n_threads)
	first_img = int((this_thread-1) * batch_size)

	if this_thread < n_threads:

		last_img = int(first_img + batch_size)

	else:

		last_img = N

	img_dirs = img_dirs[first_img:last_img]

	n = first_img + 1

	print '\nJob %d/%d picking particles from micrographs %d to %d...\n' % (this_thread, n_threads, n, last_img)

	f = open(stack_path+stack_rootname+'_1_r1-%.4d.par' % this_thread, 'w+')

	for d in img_dirs:

		d = d.strip()
		imname = d.split('/')[-1]

		try:

			# Read in all relevant image parameters:
			params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')

		except:

			print '\nProblem with image %s!\n' % d
			continue

		if ctfcor:

			try:

				mrc = glob.glob(folders+d+'/image_ctfcor.mrc')[0]
				img = spx.get_image(mrc) # Read image

				bf = open(folders+d+'/image_ctfcor.box', 'w+')

			except:

				print '\nCTF-corrected micrograph not found for image %s!\n' % d
				continue

		else:

			if use_masked_image:

				# First we look for the masked, zero-padded, normalized micrograph:
				try:

					# # There might be some funny numbers appended to the file name so we have to look for the shortest one to get the right file:
					# mrclist = glob.glob(folders+d+'/m'+imname+'*.mrc')
					# lenlist = []
					# for m in mrclist:
					# 	lenlist.append(len(m))
					# shortest_idx = np.argsort(lenlist)[0]
					# # print mrclist[shortest_idx]
					# mrc = mrclist[shortest_idx]
					# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
					# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
					# # bf = open(folders+d+'/m'+imname+'.box', 'w+')

					mrc = folders + d + '/' + params['imagename'] + '.mrc'
					img = spx.get_image(mrc) # Read image

					bf = open(folders + d + '/' + params['imagename'] + '.box', 'w+')
					
				except:

					# If it doesn't exist, we try the unmasked, zero-padded, normalized micrograph:
					try:

						# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
						# lenlist = []
						# for m in mrclist:
						# 	lenlist.append(len(m))
						# shortest_idx = np.argsort(lenlist)[0]
						# # print mrclist[shortest_idx]
						# mrc = mrclist[shortest_idx]
						# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
						# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
						# # bf = open(folders+d+'/m'+imname+'.box', 'w+')

						mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
						img = spx.get_image(mrc) # Read image

						bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')

					except:

							# If neither exist we skip this image

							print '::\nProblem with image %s!\n' % d
							continue

			else:

					# If the user requires, we go directly to the unmasked, zero-padded, normalized micrograph:
					try:

						# mrclist = glob.glob(folders+d+'/'+imname+'*.mrc')
						# lenlist = []
						# for m in mrclist:
						# 	lenlist.append(len(m))
						# shortest_idx = np.argsort(lenlist)[0]
						# # print mrclist[shortest_idx]
						# mrc = mrclist[shortest_idx]
						# bf = open(os.path.splitext(mrc)[0]+'.box', 'w+')
						# # mrc = sorted(glob.glob(folders+d+'/m'+imname+'*.mrc'))[0]
						# # bf = open(folders+d+'/m'+imname+'.box', 'w+')

						mrc = folders + d + '/' + params['nonmaskimagename'] + '.mrc'
						img = spx.get_image(mrc) # Read image

						bf = open(folders + d + '/' + params['nonmaskimagename'] + '.box', 'w+')

					except:

							# If neither exist we skip this image

							print '::\nProblem with image %s!' % d
							print '::'
							continue

		print '::\nNow boxing unit cells of micrograph %d/%d.\n' % (n, N)
		print mrc

		try:


			if invert_micrograph:

				img = -1.0 * img

			imgarr = spx.EMNumPy.em2numpy(img)

			profile = glob.glob(folders+d+'/*profile.dat')[0]
			dat = ReadProfileDat(profile)
			ccmean = np.mean(dat[:,4])
			ccstd = np.std(dat[:,4])
			cc_thr = ccmean + sigcc * ccstd

			print ':\nImage average value:%.2f' % img['mean']
			print ':Image standard deviation:%.2f' % img['sigma']
			print ':'
			print ':CC scores average value:%.2f' % ccmean
			print ':CC scores standard deviation:%.2f' % ccstd
			print ':Only particles with CC score above %.2f will be picked.\n' % cc_thr

			# # Get several values related to defocus, astigmatism and tilting:
			# params = Read2dxCfgFile(folders+d+'/2dx_image.cfg')

			w = img.get_xsize()
			# Get the unit-cell vectors:
			if sum(params['PHAORI']) == 0:

				PhaOriX = 0
				PhaOriY = 0
				a = [0,0]
				b = [0,0]
			    
			else:
			    
				
				a,b = LatticeReciprocal2Real(params['u'], params['v'], w)

				# Convert from Numpy-array to list:
				a = [a[0], a[1]]
				b = [b[0], b[1]]

			x,y = CalculatePickingPositions(dat, a, b, w, params['PHAORI'], phaori_shift, params[tiltgeom+'TLTANG'])

			# Plot the picking profile:

			if save_pick_fig:

				meanimg = np.mean(imgarr)
				stdimg = np.std(imgarr)
				climimg = [meanimg - 2 * stdimg, meanimg + 2 * stdimg]

				fig1 = plt.figure()
				plt.imshow(imgarr, cmap=cm.gray, vmin=climimg[0], vmax=climimg[1])

				Axes1 = fig1.gca()

			# The following values are constant within each crystal:
			phi = 90.0 - params[tiltgeom+'TAXA']
			theta = params[tiltgeom+'TLTANG']
			psi = 270.0 - params[tiltgeom+'TLTAXIS']
			ang = params['AST_ANGLE']

			m = 0
			for i in np.arange(dat.shape[0]):

				try:

					# Adjust the picking coordinates for the .box file and picking plots:
					xbox = x[i] + w/2 - box_size/2
					ybox = y[i] + w/2 - box_size/2

					# if dat[i,4] < cc_thr or np.isnan(x[i]) or np.isnan(y[i]):
					if dat[i,4] < cc_thr:

						if save_pick_fig:
							# Write red patch on image to be saved as .png describing the picking positions:
							# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
							Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))

					else:

						box = spx.Util.window(img,int(box_size),int(box_size),1,int(round(x[i])),int(round(y[i])))

						# Normalize box to zero mean and constant pre-defined sigma:
						if normalize_box:

							box = NormalizeStack([box], sigma)[0]

						if calculate_defocus_tilted and not ctfcor:

							RLDEF1,RLDEF2 = CalculateDefocusTilted(x[i], y[i], apix, params[tiltgeom+'TLTAXIS'], params[tiltgeom+'TLTANG'], params['DEFOCUS1'], params['DEFOCUS2'])

						else:

							RLDEF1 = params['DEFOCUS1']
							RLDEF2 = params['DEFOCUS2']

						# Write .par file with the parameters for each particle in the dataset:
						print >>f, '      %d' % (idx+1),'  %.2f' % psi,'  %.2f' % theta,'    %.2f' % phi,'     %.2f' % shx,'      %.2f' % shy,'   %d' % magnification,'     %d' % n,'  %.2f' % RLDEF1,'  %.2f' % RLDEF2,'  %.2f' % ang,'  %.2f' % occ,'        %d' % logp,'     %.4f' % sig,'   %.2f' % score,'   %.2f' % chg

						# Write the picking information to the .box file:
						print >>bf, '%d' % xbox, '\t%d' % ybox, '\t%d' % box_size, '\t%d' % box_size

						# Write image to the particle stack:
						if idx == 0:
							# If this is the first image, we initiate as a normal .mrcs stack.
							box.write_image(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, idx)

						else:
							# Subsequent images are directly appended to the file:
							with open(stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'ab') as mrcf:
								spx.EMNumPy.em2numpy(box).tofile(mrcf)

						if (save_phase_flipped or save_wiener_filtered) and not ctfcor:

							# Convert CTF parameters to SPARX convention:
							defocus = (RLDEF1+RLDEF2)/2
							ast = RLDEF1-RLDEF2
							if params['AST_ANGLE'] < 0.0:

								astang = 360.0 + params['AST_ANGLE']

							else:

								astang = params['AST_ANGLE']

							# Generate SPARX CTF object:
							p = [defocus * 1e-4, microscope_cs, microscope_voltage, apix, 0, ampcontrast * 100, ast * 1e-4, astang]
							
							spx.set_ctf(box, p)

							ctf = spx.generate_ctf(p)

						# Phase-flip the image:
						if save_phase_flipped and not ctfcor:

							boxctfcor = spx.filt_ctf(box, ctf,binary=1)
							if normalize_box:

								boxctfcor = NormalizeStack([boxctfcor], sigma)[0]

							# Write image to the particle stack:
							if idx == 0:
								# If this is the first image, we initiate as a normal .mrcs stack.
								boxctfcor.write_image( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, idx )

							else:
								# Subsequent images are directly appended to the file:
								with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
									spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )

						# Wiener-filter the image:
						if save_wiener_filtered and not ctfcor:

							boxctfcor = spx.filt_ctf(box, ctf,binary=0)
							if normalize_box:

								boxctfcor = NormalizeStack([boxctfcor], sigma)[0]
								
							# Write image to the particle stack:
							if idx == 0:
								# If this is the first image, we initiate as a normal .mrcs stack.
								boxctfcor.write_image( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, idx )

							else:
								# Subsequent images are directly appended to the file:
								with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'ab' ) as mrcf:
									spx.EMNumPy.em2numpy(boxctfcor).tofile( mrcf )

						if save_pick_fig:
							# Write green patch on image to be saved as .png describing the picking positions:
							# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='lime', facecolor='none', linewidth=0.2, radius=20))
							Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='lime', facecolor='none', linewidth=0.2))

						m += 1
						idx += 1

				except RuntimeError:

					if save_pick_fig:
						# Write red patch on image to be saved as .png describing the picking positions:
						# Axes1.add_patch(patches.Circle((dat[i,2], dat[i,3]), edgecolor='red', facecolor='none', linewidth=0.2, radius=20))
						Axes1.add_patch(patches.Rectangle(xy=(xbox, ybox), width=box_size, height=box_size, edgecolor='red', facecolor='none', linewidth=0.2))

					print 'Failed to box CC peak (%d,%d) at position (%d,%d) in micrograph %d/%d!' % (dat[i,0], dat[i,1], int(round(x[i])), int(round(y[i])), n, N)

			print '\nBoxed %d/%d CC peaks from micrograph %d/%d.\n' % (m, i+1, n, N)

			# Update the counts in the MRC headers:
			# First the normal particle stack:
			header = ioMRC.readMRCHeader( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread )
			header['dimensions'][0] = idx
			# Now we write the header back:
			with open( stack_path+stack_rootname+'-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf: 
				ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )
			# Then the phase-flipped:
			if save_phase_flipped and not ctfcor:
				
				header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread )
				header['dimensions'][0] = idx
				# Now we write the header back:
				with open( stack_path+stack_rootname+'_phase-flipped-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf: 
					ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )

			# Then the Wiener-filtered:
			if save_wiener_filtered and not ctfcor:
				
				header = ioMRC.readMRCHeader( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread )
				header['dimensions'][0] = idx
				# Now we write the header back:
				with open( stack_path+stack_rootname+'_wiener-filtered-%.4d.mrcs' % this_thread, 'rb+' ) as mrcf: 
					ioMRC.writeMRCHeader( mrcf, header, endchar = '<' )

			# print '<<@progress: %d>>' % round(n*100.0/N)
			# Report progress to the GUI:
			prog += 75.0/N
			if prog >= 1.0:
				print '<<@progress: +%d>>' % round(prog)
				prog -= np.floor(prog)

			if save_pick_fig:

				fig1.savefig(png_path+'mic_%.3d_' % n+imname+'_picking.png', dpi=300)
				plt.close(fig1)

			n += 1

		except RuntimeError:

			# print '::PROBLEM WITH MICROGRAPH %d/%d!!! Maybe it was not found?' % (n, N)
			# print '::'

			# print mrc
			print '\nPROBLEM WITH MICROGRAPH:'
			print '%s' % mrc
			print 'Maybe it was not found?'

		except ValueError:

			# print '::PROBLEM WITH CC PROFILE FOR IMAGE %d/%d!!!' % (n, N)
			# print '::'

			# print mrc

			print '\nPROBLEM WITH CC PROFILE FOR IMAGE:'
			print '%s' % mrc

		bf.close()

		# n += 1


	# print '::Total boxed unit cells:%d' % idx
	# print '::Failed to box %d unit cells.' % box_fail

	# print '<<@progress: +%d>>' % round(prog/0.01)

	print '\nJob %d/%d finished picking particles.\n' % (this_thread, n_threads)

	f.close()
Example #4
0
def main():

	progname = os.path.basename(sys.argv[0])
	usage = progname + """ [options] <.mrc(s) file>

	Given an existing .mrc(s) file, will evaluate its size and number of images/slices (NZ) and UPDATE IN-PLACE (CAUTION!) corresponding header information.

	Reference for MRC format description:
	http://bio3d.colorado.edu/imod/doc/mrc_format.txt

	This script is based on Robb McLeod's ioMRC.py available at:
	https://bitbucket.org/emmrcz/python-mrcz

	NOTE: Only vanilla MRC(S) files are supported (i.e. compressed MRC formats not supported)

	"""

	parser = OptionParser(usage)
	
	parser.add_option( "--stats", action="store_true", help="Will update the statistics of the file (minval, maxval, avgval, std). May take some time depending on file size.", default=False )
	parser.add_option( "--endian", metavar="le", type="string", default='le', help="MRC file endianness (default is little-endian)" )

	(options, args) = parser.parse_args()

	command = ' '.join(sys.argv)

	if options.endian == 'le':
		endchar = '<'
	else:
		endchar = '>'

	fpath = args[0] # The file to be updated
	hsize = 1024 # Standard MRC header is 1024 bytes long

	S = os.path.getsize( fpath ) # Get the current size of the file in bytes
	# print S-hsize
	header = ioMRC.readMRCHeader( fpath ) # Read in the current header of the file
	
	sizeof = np.dtype(header['dtype']).itemsize # Get the size in bytes of each value in the current MRC format
	# print sizeof

	with open(fpath, 'rb+') as f:

		NX = header['dimensions'][2]
		NY = header['dimensions'][1]
		NZ = header['dimensions'][0]
		print 'Current dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZ)
		NZnew = ( S - hsize ) / ( NX*NY*sizeof ) # This will yield the actual number of Z slices
		print 'New dimensions of MRC file: NX=%d,NY=%d,NZ=%d' % (NX,NY,NZnew)
		header['dimensions'][0] = NZnew

		if options.stats:
		# Let's update the statistics of the stack/volume:
			header['meanImage'] = 0.0

			f.seek(1024)
			for i in np.arange(NZnew):

				# We will read one image or slice at a time in order to save memory:
				image = np.fromfile( f, dtype=header['dtype'], count=NX*NY )

				minImage = np.min(image)
				maxImage = np.max(image)
				meanImage = np.mean(image)

				if minImage < header['minImage']:
					header['minImage'] = minImage

				if maxImage > header['maxImage']:
					header['maxImage'] = maxImage

				header['meanImage'] += meanImage

			header['meanImage'] /= NZnew


		# Now we write the header back:
		ioMRC.writeMRCHeader( f, header, endchar )