def _sendFrames(cls, delay=0, port=5000):
# start with a delay so the protocol has already launched the
# socket server when we send the first file
time.sleep(10)
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
paths = ""
print("Writing frame stack for movie %d..." % (i + 1))
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc' %
(i + 1, j))
print("%d : %s" % (j, outputFramePath))
ih.convert((j, f), outputFramePath)
paths += os.path.abspath(outputFramePath) + '\n'
time.sleep(delay)
try:
print("Sending movie stack %d" % (i + 1))
clientSocket.sendall(paths)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % paths)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
clientSocket.connect((host, port))
# Everything sent, lets wait until server has read all files
serverConnected = True
lastFile = paths.strip().split('\n')[-1]
print('Waiting for server socket to finish...')
print('Last file sent: %s' % lastFile)
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if lastFile in reply:
serverConnected = False
time.sleep(delay)
else:
serverConnected = False
print('Finished! Closing client socket')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
def _sendFrames(cls, delay=0, port=5000):
# start with a delay so the protocol has already launched the
# socket server when we send the first file
time.sleep(10)
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
paths = ""
print("Writing frame stack for movie %d..." % (i+1))
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc'
% (i + 1, j))
print("%d : %s" % (j, outputFramePath))
ih.convert((j, f), outputFramePath)
paths += os.path.abspath(outputFramePath)+'\n'
time.sleep(delay)
try:
print("Sending movie stack %d" %(i+1))
clientSocket.sendall(paths)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % paths)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientSocket.connect((host, port))
# Everything sent, lets wait until server has read all files
serverConnected = True
lastFile = paths.strip().split('\n')[-1]
print('Waiting for server socket to finish...')
print('Last file sent: %s' % lastFile)
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if lastFile in reply:
serverConnected = False
time.sleep(delay)
else:
serverConnected = False
print('Finished! Closing client socket')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName, 'w')
shift = ("0 " * numberOfFrames + "\n") * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName,'w')
shift = ("0 " * numberOfFrames + "\n" ) * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n"%movieName)
logFile = self._getLogFile(movieId)
def _createFrames(cls, delay=0):
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc' %
(i + 1, j))
ih.convert((j, f), outputFramePath)
time.sleep(delay)
def _createFrames(cls, delay=0):
# Create a test folder path
pattern = cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
ih = ImageHandler()
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, _, _, nFrames = ih.getDimensions(f)
for j in range(1, nFrames + 1):
outputFramePath = cls.proj.getTmpPath('movie%06d_%03d.mrc'
% (i+1, j))
ih.convert((j, f), outputFramePath)
time.sleep(delay)
def _validate(self):
from pyworkflow.em.convert import ImageHandler
ci = self.getImportClass()
if ci is None:
errors = ProtImportMicBase._validate(self)
for micFn, _ in self.iterFiles():
imgh = ImageHandler()
if imgh.isImageFile(micFn):
_, _, z, n = imgh.getDimensions(micFn)
if n > 1 or z > 1:
errors.append("The protocol not support micrographs stored in stacks. "
"If you want to obtain your micrographs individually, "
"you can run the following command:\n"
"scipion run scipion_directory/scripts/split_stacks.py --files *your files* --ext *extension*")
# JMRT: only check the first image, for large dataset
# even reading the header can take a while
break
return errors
else:
return ci.validateMicrographs()
def importVolumesStep(self, pattern, samplingRate):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
vol = Volume()
vol.setSamplingRate(self.samplingRate.get())
copyOrLink = self.getCopyOrLink()
imgh = ImageHandler()
volSet = self._createSetOfVolumes()
volSet.setSamplingRate(self.samplingRate.get())
for fileName, fileId in self.iterFiles():
dst = self._getExtraPath(basename(fileName))
copyOrLink(fileName, dst)
x, y, z, n = imgh.getDimensions(dst)
# First case considers when reading mrc without volume flag
# Second one considers single volumes (not in stack)
if (z == 1 and n != 1) or (z != 1 and n == 1):
vol.setObjId(fileId)
if dst.endswith('.mrc'):
dst += ':mrc'
vol.setLocation(dst)
volSet.append(vol)
else:
for index in range(1, n + 1):
vol.cleanObjId()
vol.setLocation(index, dst)
volSet.append(vol)
if volSet.getSize() > 1:
self._defineOutputs(outputVolumes=volSet)
else:
self._defineOutputs(outputVolume=vol)
def importVolumesStep(self, pattern, samplingRate):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
vol = Volume()
vol.setSamplingRate(self.samplingRate.get())
copyOrLink = self.getCopyOrLink()
imgh = ImageHandler()
volSet = self._createSetOfVolumes()
volSet.setSamplingRate(self.samplingRate.get())
for fileName, fileId in self.iterFiles():
dst = self._getExtraPath(basename(fileName))
copyOrLink(fileName, dst)
x, y, z, n = imgh.getDimensions(dst)
# First case considers when reading mrc without volume flag
# Second one considers single volumes (not in stack)
if (z == 1 and n != 1) or (z !=1 and n == 1):
vol.setObjId(fileId)
if dst.endswith('.mrc'):
dst += ':mrc'
vol.setLocation(dst)
volSet.append(vol)
else:
for index in range(1, n+1):
vol.cleanObjId()
vol.setLocation(index, dst)
volSet.append(vol)
if volSet.getSize() > 1:
self._defineOutputs(outputVolumes=volSet)
else:
self._defineOutputs(outputVolume=vol)
def _validate(self):
from pyworkflow.em.convert import ImageHandler
ci = self.getImportClass()
if ci is None:
errors = ProtImportMicBase._validate(self)
for micFn, _ in self.iterFiles():
imgh = ImageHandler()
if imgh.isImageFile(micFn):
_, _, z, n = imgh.getDimensions(micFn)
if n > 1 or z > 1:
errors.append("The protocol not support micrographs "
"stored in stacks. If you want to "
"obtain your micrographs individually, "
"you can run the following command:\n"
"scipion run scipion_directory/scripts/"
"split_stacks.py --files *your files* "
"--ext *extension*")
# JMRT: only check the first image, for large dataset
# even reading the header can take a while
break
return errors
else:
return ci.validateMicrographs()
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def importVolumesStep(self, pattern, samplingRate, setOrigCoord=False):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
vol = Volume()
vol.setSamplingRate(samplingRate)
imgh = ImageHandler()
volSet = self._createSetOfVolumes()
volSet.setSamplingRate(samplingRate)
for fileName, fileId in self.iterFiles():
x, y, z, n = imgh.getDimensions(fileName)
if fileName.endswith('.mrc') or fileName.endswith('.map'):
fileName += ':mrc'
if (z == 1 and n != 1):
zDim = n
n = 1
else:
zDim = z
else:
zDim = z
origin = Transform()
if setOrigCoord:
origin.setShiftsTuple(self._getOrigCoord())
else:
origin.setShifts(x / -2. * samplingRate,
y / -2. * samplingRate,
zDim / -2. * samplingRate)
vol.setOrigin(origin) # read origin from form
if self.copyFiles or setOrigCoord:
newFileName = abspath(self._getVolumeFileName(fileName, "mrc"))
Ccp4Header.fixFile(fileName, newFileName, origin.getShifts(),
samplingRate, Ccp4Header.ORIGIN)
else:
newFileName = abspath(self._getVolumeFileName(fileName))
if fileName.endswith(':mrc'):
fileName = fileName[:-4]
createAbsLink(fileName, newFileName)
# Make newFileName relative
# https://github.com/I2PC/scipion/issues/1935
newFileName = relpath(newFileName)
if n == 1:
vol.cleanObjId()
vol.setFileName(newFileName)
volSet.append(vol)
else:
for index in range(1, n + 1):
vol.cleanObjId()
vol.setLocation(index, newFileName)
volSet.append(vol)
if volSet.getSize() > 1:
self._defineOutputs(outputVolumes=volSet)
else:
self._defineOutputs(outputVolume=vol)
