def scipion_split_particle_stacks(inputStar, inputStack, output, filename_prefix, deleteStack):
""" Read a STAR file with particles and write as individual images.
If a stack of images is given, use these instead of the images from the STAR file.
Also write a new STAR file pointing to these images.
This function requires that the script is run within Scipion Python environment. """
import pyworkflow.utils as pwutils
from pyworkflow.em import ImageHandler
ih = ImageHandler()
md = MetaData(inputStar)
md.addLabels('rlnOriginalName')
# Initialize progress bar
progressbar = ProgressBar(width=60, total=len(md))
for i, particle in enumerate(md, start=1):
outputImageName = '%s/%s_%06d.mrc' % (output, filename_prefix, i)
if inputStack:
ih.convert((i, inputStack), outputImageName )
particle.rlnOriginalName = '%s/%06d@%s' %(output, i, inputStack)
else:
ih.convert(particle.rlnImageName, outputImageName)
particle.rlnOriginalName = particle.rlnImageName
particle.rlnImageName = outputImageName
progressbar.notify()
print("\n")
md.write("%s/%s.star" % (output, filename_prefix))
if inputStack and deleteStack:
pwutils.cleanPath(inputStack)
def main():
parser = argparse.ArgumentParser(description="make individual micrographs from a stack")
add = parser.add_argument # shortcut
add('--files', default='',
help='list of files to split on individual microgrpahs')
add('--ext', default='mrc',
help='define extension of individual microgrpahs')
args = parser.parse_args()
def getNumOfElements(fileN):
_, _, z, n = imgh.getDimensions(fileN)
if z > 1:
return z
else:
return n
filePaths = glob.glob(args.files)
filePaths.sort()
imgh = ImageHandler()
for fileN in filePaths:
n = getNumOfElements(fileN)
for i in range(1, n+1):
outputFn = "%s_%03d.%s" % (os.path.splitext(fileN)[0], i, args.ext)
imgh.convert((i, fileN), outputFn)
def _sendFrames(cls, delay=0, port=5000):
# start with a delay so the protocol has already launched the
# socket server when we send the first file
time.sleep(10)
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
paths = ""
print("Writing frame stack for movie %d..." % (i + 1))
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc' %
(i + 1, j))
print("%d : %s" % (j, outputFramePath))
ih.convert((j, f), outputFramePath)
paths += os.path.abspath(outputFramePath) + '\n'
time.sleep(delay)
try:
print("Sending movie stack %d" % (i + 1))
clientSocket.sendall(paths)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % paths)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
clientSocket.connect((host, port))
# Everything sent, lets wait until server has read all files
serverConnected = True
lastFile = paths.strip().split('\n')[-1]
print('Waiting for server socket to finish...')
print('Last file sent: %s' % lastFile)
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if lastFile in reply:
serverConnected = False
time.sleep(delay)
else:
serverConnected = False
print('Finished! Closing client socket')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
def _sendFrames(cls, delay=0, port=5000):
# start with a delay so the protocol has already launched the
# socket server when we send the first file
time.sleep(10)
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
paths = ""
print("Writing frame stack for movie %d..." % (i+1))
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc'
% (i + 1, j))
print("%d : %s" % (j, outputFramePath))
ih.convert((j, f), outputFramePath)
paths += os.path.abspath(outputFramePath)+'\n'
time.sleep(delay)
try:
print("Sending movie stack %d" %(i+1))
clientSocket.sendall(paths)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % paths)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientSocket.connect((host, port))
# Everything sent, lets wait until server has read all files
serverConnected = True
lastFile = paths.strip().split('\n')[-1]
print('Waiting for server socket to finish...')
print('Last file sent: %s' % lastFile)
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if lastFile in reply:
serverConnected = False
time.sleep(delay)
else:
serverConnected = False
print('Finished! Closing client socket')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
class Main():
def getNumOfElements(self, fileN):
_, _, z, n = self.imgh.getDimensions(fileN)
if z > 1:
return z
else:
return n
def getOutputLoc(self, filePrefix, i):
if self.n == 1:
index = 1
fn = "%s_%03d.%s" % (filePrefix, i, self.args.ext)
else:
index = self.outIndex
fn = "%s_%03d.%s" % (filePrefix, self.groupCount, self.args.ext)
self.outIndex += 1
if self.outIndex > self.n:
self.outIndex = 1
self.groupCount += 1
return (index, fn)
def __init__(self):
parser = argparse.ArgumentParser(description="Extract images from a stack and write them into "
"smaller stacks or even individual images.")
add = parser.add_argument # shortcut
add('--files', default='',
help='Pattern to match input files.')
add('--ext', default='mrc',
help='Define extension of output files.')
add('-n', default=1,
help='Group output images in stacks of this size.')
self.args = parser.parse_args()
self.n = int(self.args.n)
filePaths = glob.glob(self.args.files)
filePaths.sort()
self.imgh = ImageHandler()
for fileN in filePaths:
self.outIndex = 1
self.groupCount = 1
n = self.getNumOfElements(fileN)
if fileN.endswith('.mrc'):
fileN += ':mrcs'
filePrefix = os.path.splitext(fileN)[0]
for i in range(1, 2):
outputLoc = self.getOutputLoc(filePrefix, i)
self.imgh.convert((i, fileN), outputLoc)
class Main():
def getNumOfElements(self, fileN):
_, _, z, n = self.imgh.getDimensions(fileN)
if z > 1:
return z
else:
return n
def getOutputLoc(self, filePrefix, i):
if self.n == 1:
index = 1
fn = "%s_%03d.%s" % (filePrefix, i, self.args.ext)
else:
index = self.outIndex
fn = "%s_%03d.%s" % (filePrefix, self.groupCount, self.args.ext)
self.outIndex += 1
if self.outIndex > self.n:
self.outIndex = 1
self.groupCount += 1
return (index, fn)
def __init__(self):
parser = argparse.ArgumentParser(description="Extract images from a stack and write them into "
"smaller stacks or even individual images.")
add = parser.add_argument # shortcut
add('--files', default='',
help='Pattern to match input files.')
add('--ext', default='mrc',
help='Define extension of output files.')
add('-n', default=1,
help='Group output images in stacks of this size.')
self.args = parser.parse_args()
self.n = int(self.args.n)
filePaths = glob.glob(self.args.files)
filePaths.sort()
self.imgh = ImageHandler()
for fileN in filePaths:
self.outIndex = 1
self.groupCount = 1
n = self.getNumOfElements(fileN)
if fileN.endswith('.mrc'):
fileN += ':mrcs'
filePrefix = os.path.splitext(fileN)[0]
for i in range(1, n+1):
outputLoc = self.getOutputLoc(filePrefix, i)
self.imgh.convert((i, fileN), outputLoc)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName, 'w')
shift = ("0 " * numberOfFrames + "\n") * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName,'w')
shift = ("0 " * numberOfFrames + "\n" ) * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n"%movieName)
logFile = self._getLogFile(movieId)
def _createFrames(cls, delay=0):
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc' %
(i + 1, j))
ih.convert((j, f), outputFramePath)
time.sleep(delay)
def _createFrames(cls, delay=0):
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc'
% (i+1, j))
ih.convert((j, f), outputFramePath)
time.sleep(delay)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def __init__(self):
parser = argparse.ArgumentParser(
description="Create movie stacks from the individual "
"frame files.")
add = parser.add_argument # shortcut
add('--files', default='',
help='Pattern to match input frame files.')
add('-n',
help='Number of frames per movie.')
add('--suffix',
help='Provide suffix added to create movie file. '
'e.g. _frames.mrcs')
add('--delete_frames', action='store_true',
help='Provide this option if you want to delete individual frame '
'files after the movie stack is created. ')
args = parser.parse_args()
n = int(args.n)
frameRegex = re.compile("(?P<prefix>.+[^\d]+)(?P<frameid>\d+)")
# Group all frames for each movie
# Key of the dictionary will be the common prefix and the value
# will be a list with all frames in that movie
frameDict = {}
filePaths = glob.glob(args.files)
filePaths.sort()
for fileName in filePaths:
fnNoExt = pwutils.removeExt(fileName)
match = frameRegex.match(fnNoExt)
if match is None:
raise Exception("Incorrect match of frame files pattern!")
d = match.groupdict()
prefix = d['prefix']
frameid = int(d['frameid'])
if prefix not in frameDict:
frameDict[prefix] = []
frameDict[prefix].append((frameid, fileName))
suffix = args.suffix
ih = ImageHandler()
for k, v in frameDict.iteritems():
if len(v) != n:
raise Exception("Incorrect number of frames!")
movieFn = k + suffix
movieOut = movieFn
if movieOut.endswith("mrc"):
movieOut += ":mrcs"
print "Writing movie stack: ", movieFn
for i, frame in enumerate(sorted(v, key=lambda x: x[0])):
frameFn = frame[1] # Frame name stored previously
ih.convert(frameFn, (i+1, movieFn))
if args.delete_frames:
pwutils.cleanPath(frameFn)
def testRisosome(self):
print("Importing 2D averages (subset of 4)")
ih = ImageHandler()
classesFn = self.ds.getFile('import/classify2d/extra/'
'relion_it015_classes.mrcs')
outputName = 'input_averages.mrcs'
inputTmp = os.path.abspath(self.proj.getTmpPath())
outputFn = self.proj.getTmpPath(outputName)
for i, index in enumerate([5, 16, 17, 18, 24]):
ih.convert((index, classesFn), (i + 1, outputFn))
protAvgs = self.newProtocol(ProtImportAverages,
objLabel='avgs - 5',
filesPath=inputTmp,
filesPattern=outputName,
samplingRate=7.08)
self.launchProtocol(protAvgs)
# First, import a set of micrographs
print("Importing a set of micrographs...")
protImport = self.newProtocol(ProtImportMicrographs,
filesPath=os.path.abspath(
self.proj.getTmpPath()),
filesPattern="*%s" % self.ext,
samplingRateMode=1,
magnification=79096,
scannedPixelSize=56,
voltage=300,
sphericalAberration=2.0,
dataStreaming=True,
fileTimeout=10,
timeout=60)
protImport.setObjLabel('import 20 mics (streaming)')
self.proj.launchProtocol(protImport, wait=False)
self._waitOutput(protImport, 'outputMicrographs')
# Now estimate CTF on the micrographs with ctffind
print("Performing CTFfind...")
protCTF = self.newProtocol(ProtCTFFind,
useCtffind4=True,
lowRes=0.02,
highRes=0.45,
minDefocus=1.2,
maxDefocus=3,
runMode=1,
numberOfMpi=1,
numberOfThreads=1)
protCTF.inputMicrographs.set(protImport.outputMicrographs)
protCTF.setObjLabel('CTF ctffind')
self.proj.launchProtocol(protCTF, wait=False)
self._waitOutput(protCTF, 'outputCTF')
self._waitUntilMinSize(protCTF.outputCTF)
# Select some good averages from the iterations mrcs a
protPick = self.newProtocol(ProtRelion2Autopick,
objLabel='autopick refs',
runType=0,
micrographsNumber=3,
referencesType=0,
refsHaveInvertedContrast=True,
particleDiameter=380)
protPick.inputMicrographs.set(protImport.outputMicrographs)
protPick.ctfRelations.set(protCTF.outputCTF)
protPick.inputReferences.set(protAvgs.outputAverages)
self.launchProtocol(protPick)
protPick.runType.set(1)
self.launchProtocol(protPick)
def processMovieStep(self, movieDict, hasAlignment):
movie = Movie()
movie.setAcquisition(Acquisition())
if hasAlignment:
movie.setAlignment(MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
if (self.isContinued() and os.path.exists(self._getMovieDone(movie))):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(self._getMovieDone(movie))
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.tif'):
#FIXME: It seems that we have some flip problem with compressed
# tif files, we need to check that
newMovieName = movieName.replace('.tif', '.mrc')
# we assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tif2mrc',
'%s %s' % (movieName, newMovieName),
cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
if (self.CONVERT_TO_MRC
and not (newMovieName.endswith("mrc")
or newMovieName.endswith("mrcs"))):
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
newMovieName = pwutils.replaceExt(newMovieName,
self.CONVERT_TO_MRC)
outputMovieFn = os.path.join(movieFolder, newMovieName)
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
ImageHandler().convertStack(inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
self.info('Clean movie data DISABLED. '
'Movie folder will remain in disk!!!')
else:
self.info("Erasing.....movieFolder: %s" % movieFolder)
os.system('rm -rf %s' % movieFolder)
# cleanPath(movieFolder)
# Mark this movie as finished
open(self._getMovieDone(movie), 'w').close()
def processMovieStep(self, movieDict, hasAlignment):
movie = Movie()
movie.setAcquisition(Acquisition())
if hasAlignment:
movie.setAlignment(MovieAlignment())
movie.setAttributesFromDict(movieDict, setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
movieDoneFn = self._getMovieDone(movie)
if (self.isContinued() and os.path.exists(movieDoneFn)):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(movieDoneFn)
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2', '-d -f %s' % movieName, cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin, line.strip())
ih.convert(inputFrame,
(i+1, os.path.join(movieFolder, newMovieName)))
else:
newMovieName = movieName
convertExt = self._getConvertExtension(newMovieName)
correctGain = self._doCorrectGain()
if convertExt or correctGain:
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
if convertExt:
newMovieName = pwutils.replaceExt(newMovieName, convertExt)
else:
newMovieName = '%s_corrected.%s' % os.path.splitext(newMovieName)
outputMovieFn = os.path.join(movieFolder, newMovieName)
# If the protocols wants Scipion to apply the gain, then
# there is no reason to convert, since we can produce the
# output in the format expected by the program. In some cases,
# the alignment programs can directly deal with gain and dark
# correction images, so we don't need to apply it
if self._doCorrectGain():
self.info("Correcting gain and dark '%s' -> '%s'"
% (inputMovieFn, outputMovieFn))
gain, dark = self.getGainAndDark()
self.correctGain(inputMovieFn, outputMovieFn,
gainFn=gain, darkFn=dark)
else:
self.info("Converting movie '%s' -> '%s'"
% (inputMovieFn, outputMovieFn))
ImageHandler().convertStack(inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if self._doMovieFolderCleanUp():
self._cleanMovieFolder(movieFolder)
# Mark this movie as finished
open(movieDoneFn, 'w').close()
def testRisosome(self):
print "Importing 2D averages (subset of 4)"
ih = ImageHandler()
classesFn = self.ds.getFile('import/classify2d/extra/'
'relion_it015_classes.mrcs')
outputName = 'input_averages.mrcs'
inputTmp = os.path.abspath(self.proj.getTmpPath())
outputFn = self.proj.getTmpPath(outputName)
for i, index in enumerate([5, 16, 17, 18, 24]):
ih.convert((index, classesFn), (i + 1, outputFn))
protAvgs = self.newProtocol(ProtImportAverages,
objLabel='avgs - 5',
filesPath=inputTmp,
filesPattern=outputName,
samplingRate=7.08
)
self.launchProtocol(protAvgs)
# First, import a set of micrographs
print "Importing a set of micrographs..."
protImport = self.newProtocol(ProtImportMicrographs,
filesPath=os.path.abspath(self.proj.getTmpPath()),
filesPattern="*%s" % self.ext,
samplingRateMode=1,
magnification=79096,
scannedPixelSize=56, voltage=300,
sphericalAberration=2.0,
dataStreaming=True,
fileTimeout=10,
timeout=60)
protImport.setObjLabel('import 20 mics (streaming)')
self.proj.launchProtocol(protImport, wait=False)
self._waitOutput(protImport, 'outputMicrographs')
# Now estimate CTF on the micrographs with ctffind
print "Performing CTFfind..."
protCTF = self.newProtocol(ProtCTFFind,
useCtffind4=True,
lowRes=0.02, highRes=0.45,
minDefocus=1.2, maxDefocus=3,
runMode=1,
numberOfMpi=1, numberOfThreads=1)
protCTF.inputMicrographs.set(protImport.outputMicrographs)
protCTF.setObjLabel('CTF ctffind')
self.proj.launchProtocol(protCTF, wait=False)
self._waitOutput(protCTF, 'outputCTF')
self._waitUntilMinSize(protCTF.outputCTF)
# Select some good averages from the iterations mrcs a
protPick = self.newProtocol(ProtRelion2Autopick,
objLabel='autopick refs',
runType=0,
micrographsNumber=3,
referencesType=0,
refsHaveInvertedContrast=True,
particleDiameter=380
)
protPick.inputMicrographs.set(protImport.outputMicrographs)
protPick.ctfRelations.set(protCTF.outputCTF)
protPick.inputReferences.set(protAvgs.outputAverages)
self.launchProtocol(protPick)
protPick.runType.set(1)
self.launchProtocol(protPick)
def __init__(self):
parser = argparse.ArgumentParser(
description="Create movie stacks from the individual "
"frame files.")
add = parser.add_argument # shortcut
add('--files', default='', help='Pattern to match input frame files.')
add('-n', help='Number of frames per movie.')
add('--suffix',
help='Provide suffix added to create movie file. '
'e.g. _frames.mrcs')
add('--delete_frames',
action='store_true',
help='Provide this option if you want to delete individual frame '
'files after the movie stack is created. ')
args = parser.parse_args()
n = int(args.n)
frameRegex = re.compile("(?P<prefix>.+[^\d]+)(?P<frameid>\d+)")
# Group all frames for each movie
# Key of the dictionary will be the common prefix and the value
# will be a list with all frames in that movie
frameDict = {}
filePaths = glob.glob(args.files)
filePaths.sort()
for fileName in filePaths:
fnNoExt = pwutils.removeExt(fileName)
match = frameRegex.match(fnNoExt)
if match is None:
raise Exception("Incorrect match of frame files pattern!")
d = match.groupdict()
prefix = d['prefix']
frameid = int(d['frameid'])
if prefix not in frameDict:
frameDict[prefix] = []
frameDict[prefix].append((frameid, fileName))
suffix = args.suffix
ih = ImageHandler()
for k, v in frameDict.iteritems():
if len(v) != n:
raise Exception("Incorrect number of frames!")
movieFn = k + suffix
movieOut = movieFn
if movieOut.endswith("mrc"):
movieOut += ":mrcs"
print "Writing movie stack: ", movieFn
for i, frame in enumerate(sorted(v, key=lambda x: x[0])):
frameFn = frame[1] # Frame name stored previously
ih.convert(frameFn, (i + 1, movieFn))
if args.delete_frames:
pwutils.cleanPath(frameFn)