def __convertCorrectionImage(self, correctionImage):
""" Will convert a gain or dark file to a compatible one and return
the final file name. Otherwise, will return same passed parameter"""
# Get final correction image file
finalName = self.getFinalCorrectionImagePath(correctionImage)
# If correctionImage is None, finalName will be None
if finalName is None:
return None
elif not os.path.exists(finalName):
# Conversion never happened...
print('converting %s to %s' % (correctionImage, finalName))
if correctionImage.endswith('.gain'): # treat as tif file
fnBase = basename(correctionImage)
tmpLink = self._getTmpPath(pwutils.replaceExt(fnBase, 'tif'))
pwutils.createAbsLink(correctionImage, tmpLink)
correctionImage = tmpLink
emlib.image.ImageHandler().convert(correctionImage, finalName)
# return final name
return os.path.abspath(finalName)
def fixLinks(self, searchDir):
runs = self.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
f = f.split(':')[0]
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
def _convertGain(self):
""" We need to transform gain file for a possible polishing job. """
rotation = self.gainRot.get()
flip = self.gainFlip.get()
gainFn = self.gainFile.get()
if rotation or flip:
args = "%s %s " % (gainFn, self._getPath(os.path.basename(gainFn)))
if flip:
# flip axis Y - left to right
args += "--process xform.flip:axis=%s " % ("y" if flip == 2 else "x")
if rotation:
args += "--rotate %d " % (rotation * 90)
from pwem import Domain
eman2 = Domain.importFromPlugin('eman2')
pwutils.runJob(self._log, eman2.Plugin.getProgram('e2proc2d.py'), args,
env=eman2.Plugin.getEnviron())
return self._getPath(os.path.basename(gainFn))
else:
outputGain = self._getPath(os.path.basename(gainFn))
pwutils.createAbsLink(gainFn, outputGain)
return outputGain
def decompress(program, args, ext, nExt):
movieFolder = self._getTmpPath()
movieName = basename(movie.getFileName())
movieTmpLink = join(movieFolder, movieName)
pwutils.cleanPath(movieTmpLink)
pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
self.runJob(program, args % movieName, cwd=movieFolder)
dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
def decompress(program, args, ext, nExt):
movieFolder = self._getTmpPath()
movieName = basename(movie.getFileName())
movieTmpLink = join(movieFolder, movieName)
pwutils.cleanPath(movieTmpLink)
pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
self.runJob(program, args % movieName, cwd=movieFolder)
dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
def _createInputLinks(cls):
# Create a test folder path
pattern = cls.ds.getFile('allMics')
files = glob(pattern)
nFiles = len(files)
for i in range(nFiles):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
pwutils.createAbsLink(f, moviePath)
time.sleep(10)
def _createInputLinks(cls):
# Create a test folder path
pattern = cls.ds.getFile('allMics')
files = glob(pattern)
nFiles = len(files)
for i in range(nFiles):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
pwutils.createAbsLink(f, moviePath)
time.sleep(10)
def _createInputLinks(cls):
# Create a test folder path
pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
pwutils.createAbsLink(f, moviePath)
time.sleep(DELAY)
def _createInputLinks(cls):
# Create a test folder path
pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i+1, cls.ext))
pwutils.createAbsLink(f, moviePath)
time.sleep(DELAY)
def testImportFromCTffind4Conflict(self):
micsPath = os.path.abspath(self.proj.getTmpPath('micrographs'))
ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs'))
pwutils.makePath(micsPath)
pwutils.makePath(ctfsPath)
micDict = {'BPV_1386': 'mic1', 'BPV_1387': 'mic10', 'BPV_1388': 'mic2'}
pattern = 'micrographs/%s.mrc'
for k, v in micDict.items():
# Create a micrograph link with a given naming convention
pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k),
self.proj.getTmpPath(pattern % v))
pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k),
self.proj.getTmpPath('ctfs/%s' % v))
protCTF = self.newProtocol(
emprot.ProtImportCTF,
importFrom=emprot.ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF.inputMicrographs.set(self.protImport.outputMicrographs)
protCTF.setObjLabel('import from ctffind3 - empty')
self.launchProtocol(protCTF)
# FIXME: After the execution of the above case, some empty sets
# are created, if there is no matching, there should not be any output
# Let's import now the correct names, but with the problematic matching
protImport2 = self.newProtocol(emprot.ProtImportMicrographs,
filesPath=micsPath,
filesPattern='mic*mrc',
samplingRate=1.237,
voltage=300)
self.launchProtocol(protImport2)
protCTF2 = self.newProtocol(
emprot.ProtImportCTF,
importFrom=emprot.ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF2.inputMicrographs.set(protImport2.outputMicrographs)
protCTF2.setObjLabel('import from ctffind3 - match')
self.launchProtocol(protCTF2)
# The whole set (3 items) should find its corresponding CTF
self.assertEqual(protCTF2.outputCTF.getSize(), 3)
def testImportFromCTffind4Conflict(self):
micsPath = os.path.abspath(self.proj.getTmpPath('micrographs'))
ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs'))
pwutils.makePath(micsPath)
pwutils.makePath(ctfsPath)
micDict = {'BPV_1386': 'mic1',
'BPV_1387': 'mic10',
'BPV_1388': 'mic2'}
pattern = 'micrographs/%s.mrc'
for k, v in micDict.iteritems():
# Create a micrograph link with a given naming convention
pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k),
self.proj.getTmpPath(pattern % v))
pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k),
self.proj.getTmpPath('ctfs/%s' % v))
protCTF = self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF.inputMicrographs.set(self.protImport.outputMicrographs)
protCTF.setObjLabel('import from ctffind3 - empty')
self.launchProtocol(protCTF)
# FIXME: After the execution of the above case, some empty sets
# are created, if there is no matching, there should not be any output
# Let's import now the correct names, but with the problematic matching
protImport2 = self.newProtocol(ProtImportMicrographs,
filesPath=micsPath,
filesPattern='mic*mrc',
samplingRate=1.237, voltage=300)
self.launchProtocol(protImport2)
protCTF2 = self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF2.inputMicrographs.set(protImport2.outputMicrographs)
protCTF2.setObjLabel('import from ctffind3 - match')
self.launchProtocol(protCTF2)
# The whole set (3 items) should find its corresponding CTF
self.assertEqual(protCTF2.outputCTF.getSize(), 3)
def _sendMovies(cls, delay=0, port=5000):
time.sleep(10) # wait so the server socket is launched
# Create a test folder path
pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
print("Connecting to server socket...")
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
print("Creating movie %d link..." % (i + 1))
pwutils.createAbsLink(f, moviePath)
absPath = os.path.abspath(moviePath) + '\n'
try:
clientSocket.sendall(absPath)
time.sleep(delay)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % absPath)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
clientSocket.connect((host, port))
serverConnected = True
print('Last file sent: %s' % absPath)
print('Waiting for server socket to finish reading...')
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if absPath in reply:
serverConnected = False
time.sleep(delay) # wait a bit to check for reply
else:
serverConnected = False
print('Finished! Disconnecting client...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
def importModelStep(self):
""" Create a link to the provided input model path
and register the output to be used later for further training
or picking.
"""
absPath = os.path.abspath(self.modelPath.get())
outputPath = self._getExtraPath(os.path.basename(absPath))
self.info("Creating link:\n" "%s -> %s" % (outputPath, absPath))
self.info(
"NOTE: If you move this project to another computer, the symbolic"
"link to the model will be broken, but you can update the link "
"and get it working again. ")
pwutils.createAbsLink(absPath, outputPath)
self._defineOutputs(outputModel=TopazModel(outputPath))
def _sendMovies(cls, delay=0, port=5000):
time.sleep(10) # wait so the server socket is launched
# Create a test folder path
pattern = PATTERN if PATTERN else cls.ds.getFile('ribo/Falcon*mrcs')
files = glob(pattern)
nFiles = len(files)
nMovies = MOVS
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
print("Connecting to server socket...")
clientSocket.connect((host, port))
for i in range(nMovies):
# Loop over the number of input movies if we want more for testing
f = files[i % nFiles]
_, cls.ext = os.path.splitext(f)
moviePath = cls.proj.getTmpPath('movie%06d%s' % (i + 1, cls.ext))
print("Creating movie %d link..." %(i+1))
pwutils.createAbsLink(f, moviePath)
absPath = os.path.abspath(moviePath)+'\n'
try:
clientSocket.sendall(absPath)
time.sleep(delay)
except socket.error as err:
# Send failed
print('Failed to send file: %s' % absPath)
print(err)
print('Trying to reconnect...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
time.sleep(2)
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientSocket.connect((host, port))
serverConnected = True
print('Last file sent: %s' % absPath)
print('Waiting for server socket to finish reading...')
while serverConnected:
reply = clientSocket.recv(4096)
if reply:
if absPath in reply:
serverConnected = False
time.sleep(delay) # wait a bit to check for reply
else:
serverConnected = False
print('Finished! Disconnecting client...')
clientSocket.shutdown(socket.SHUT_WR)
clientSocket.close()
def assignStep(self):
# Create a Volume object
self.inVol = self.inVolume.get()
self.outVol = emobj.Volume()
self.outVol.copy(self.inVol)
# Set sampling Rate (or copy from input)
if self.setSampling.get():
samplingRate = self.samplingRate.get()
else:
samplingRate = self.inVol.getSamplingRate()
self.outVol.setSamplingRate(samplingRate)
# set Origin
if self.setOrigCoord.get():
origin = emobj.Transform()
origin.setShifts(self.x.get(), self.y.get(), self.z.get())
else:
origin = self.inVol.getOrigin()
self.outVol.setOrigin(origin)
# Files system stuff
fileName = os.path.abspath(self.inVol.getFileName())
fileName = fileName.replace(':mrc', '')
# copy or link
if self.copyFiles:
imgBase = pwutils.replaceBaseExt(fileName, "mrc")
imgDst = self._getExtraPath(imgBase)
Ccp4Header.fixFile(fileName,
imgDst,
origin.getShifts(),
sampling=samplingRate)
else:
imgBase = basename(fileName)
imgDst = self._getExtraPath(imgBase)
pwutils.createAbsLink(fileName, imgDst)
self.outVol.setLocation(imgDst)
# save
self._defineOutputs(outputVolume=self.outVol)
self._defineSourceRelation(self.inVol, self.outVol)
def runGautomatch(cls,
micNameList,
refStack,
workDir,
extraArgs,
env=None,
runJob=None):
""" Run Gautomatch with the given parameters.
If micrographs are not .mrc, they will be converted.
If runJob=None, it will use pwutils.runJob.
"""
args = ''
ih = ImageHandler()
for micName in micNameList:
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workDir,
pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micName), outMic)
else:
ih.convert(micName, outMic)
args += ' %s/*.mrc' % workDir
if refStack is not None:
args += ' -T %s' % refStack
args += ' %s' % extraArgs
environ = env if env is not None else cls.getEnviron()
if runJob is None:
pwutils.runJob(None, cls.getProgram(), args, env=environ)
else:
runJob(cls.getProgram(), args, env=environ)
for micName in micNameList:
outMic = os.path.join(workDir,
pwutils.replaceBaseExt(micName, 'mrc'))
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def convertInputStep(self, micsId, refsId):
""" Match ctf information against the micrographs. """
self.ctfDict = {}
if self.ctfRelations.get() is not None:
for ctf in self.ctfRelations.get():
self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone()
ih = emlib.image.ImageHandler()
for mic in self.getInputMicrographs():
micName = mic.getFileName()
# We convert the input micrographs if they are not .mrc
outMic = os.path.join(self._getTmpPath(),
pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micName), outMic)
else:
ih.convert(micName, outMic, emlib.DT_FLOAT)
if refsId is not None:
writeReferences(self.getInputReferences(),
self._getExtraPath('references.mrc'))
def _doCtfEstimation(self, mic, **kwargs):
""" Run ctffind with required parameters.
:param mic: input mic object
:param kwargs: dict with arguments
"""
if self.usePowerSpectra:
micFn = mic._powerSpectra.getFileName()
powerSpectraPix = self.psSampling
else:
micFn = mic.getFileName()
powerSpectraPix = None
micDir = self._getTmpPath('mic_%06d' % mic.getObjId())
# Create micrograph dir
pwutils.makePath(micDir)
micFnMrc = os.path.join(micDir, pwutils.replaceBaseExt(micFn, 'mrc'))
ih = emlib.image.ImageHandler()
if not pwutils.exists(micFn):
raise FileNotFoundError("Missing input micrograph: %s" % micFn)
if micFn.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micFn), micFnMrc)
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
try:
program, args = self._ctfProgram.getCommand(
micFn=micFnMrc,
powerSpectraPix=powerSpectraPix,
ctffindOut=self._getCtfOutPath(mic),
ctffindPSD=self._getPsdPath(mic),
**kwargs)
self.runJob(program, args)
pwutils.cleanPath(micDir)
except Exception as e:
self.error("ERROR: Ctffind has failed for %s. %s" %
(micFnMrc, self._getErrorFromCtffindTxt(mic, e)))
def testAssignCTF(self):
""" Test the particle extraction after importing another
SetOfMicraphs with a different micName but same ids.
We will use assign-ctf protocol and extract from the
newly imported mics with the assigned CTF.
For the other mics, we will just create symbolic links.
"""
# Create the links with a different micrograph name
micsPath = self.proj.getPath('otherMicrographs')
pwutils.makePath(micsPath)
for i in [6, 7, 8]:
micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
micLink = join(micsPath, basename(micPath).replace('.mrc', '_DW.mrc'))
pwutils.createAbsLink(micPath, micLink)
protImportDW = self.proj.copyProtocol(self.protImport)
protImportDW.setObjLabel('import -mics DW')
protImportDW.filesPath.set(micsPath)
protImportDW.filesPattern.set('*_DW.mrc')
self.launchProtocol(protImportDW)
protAssignCTF = self.newProtocol(ProtCTFAssign)
protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
self.launchProtocol(protAssignCTF)
protExtract = self.newProtocol(XmippProtExtractParticles,
boxSize=183, downsampleType=OTHER,
downFactor=3.0,doFlip=False)
protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
protExtract.ctfRelations.set(self.protCTF.outputCTF)
protExtract.setObjLabel("extract-other (DW mics)")
self.launchProtocol(protExtract)
inputCoords = protExtract.inputCoordinates.get()
outputParts = protExtract.outputParticles
def testAssignCTF(self):
""" Test the particle extraction after importing another
SetOfMicraphs with a different micName but same ids.
We will use assign-ctf protocol and extract from the
newly imported mics with the assigned CTF.
For the other mics, we will just create symbolic links.
"""
# Create the links with a different micrograph name
micsPath = self.proj.getPath('otherMicrographs')
pwutils.makePath(micsPath)
for i in [6, 7, 8]:
micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
micLink = join(micsPath,
basename(micPath).replace('.mrc', '_DW.mrc'))
pwutils.createAbsLink(micPath, micLink)
protImportDW = self.proj.copyProtocol(self.protImport)
protImportDW.setObjLabel('import -mics DW')
protImportDW.filesPath.set(micsPath)
protImportDW.filesPattern.set('*_DW.mrc')
self.launchProtocol(protImportDW)
protAssignCTF = self.newProtocol(ProtCTFAssign)
protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
self.launchProtocol(protAssignCTF)
protExtract = self.newProtocol(XmippProtExtractParticles,
boxSize=183,
downsampleType=OTHER,
downFactor=3.0,
doFlip=False)
protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
protExtract.ctfRelations.set(self.protCTF.outputCTF)
protExtract.setObjLabel("extract-other (DW mics)")
self.launchProtocol(protExtract)
inputCoords = protExtract.inputCoordinates.get()
outputParts = protExtract.outputParticles
def _runPreviousProtocols(self):
protImportTomogram = self.newProtocol(tomo.protocols.ProtImportTomograms,
filesPath=self.tomogram,
samplingRate=5)
self.launchProtocol(protImportTomogram)
coords = pwutils.removeBaseExt(self.coords3D)
coords = protImportTomogram._getExtraPath(coords + '.txt')
pwutils.createAbsLink(self.coords3D_Large, coords)
protImportCoordinates3d = self.newProtocol(tomo.protocols.ProtImportCoordinates3D,
auto=IMPORT_FROM_EMAN,
filesPath=coords,
importTomograms=protImportTomogram.outputTomograms,
filesPattern='', boxSize=32,
samplingRate=5)
self.launchProtocol(protImportCoordinates3d)
self.assertIsNotNone(protImportTomogram.outputTomograms,
"There was a problem with tomogram output")
self.assertIsNotNone(protImportCoordinates3d.outputCoordinates,
"There was a problem with coordinates 3d output")
doInvert = False
doNormalize = False
boxSize = 32
protTomoExtraction = self.newProtocol(EmanProtTomoExtraction,
inputTomograms=protImportTomogram.outputTomograms,
inputCoordinates=protImportCoordinates3d.outputCoordinates,
tomoSource=0,
doInvert=doInvert,
doNormalize=doNormalize,
boxSize=boxSize)
self.launchProtocol(protTomoExtraction)
self.assertIsNotNone(protTomoExtraction.outputSetOfSubtomogram,
"There was a problem with SetOfSubtomogram output")
return protTomoExtraction
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
""" Function to generate thumbnails for the report. Uses data from
self.thumbPaths.
===== Params =====
- firstThumbIndex: index from which we start generating thumbnails
- micScaleFactor: how much to reduce in size the micrographs.
"""
ih = ImageHandler()
numMics = len(self.thumbPaths[MIC_PATH])
for i in range(firstThumbIndex, numMics):
print('Generating images for mic %d' % (i + 1))
# mic thumbnails
dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
if not exists(dstImgPath):
if self.micThumbSymlinks:
pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i],
dstImgPath)
else:
ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
dstImgPath,
scaleFactor=micScaleFactor)
# shift plots
if SHIFT_THUMBS in self.thumbPaths:
dstImgPath = join(self.reportDir,
self.thumbPaths[SHIFT_THUMBS][i])
if not exists(dstImgPath):
pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i],
dstImgPath)
# Psd thumbnails
if self.ctfProtocol is None:
srcImgPath = self.thumbPaths[PSD_PATH][i]
dstImgPath = join(self.reportDir,
self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
if srcImgPath.endswith('psd'):
psdImg1 = ih.read(srcImgPath)
psdImg1.convertPSD()
psdImg1.write(dstImgPath)
ih.computeThumbnail(dstImgPath,
dstImgPath,
scaleFactor=1)
else:
pwutils.createAbsLink(srcImgPath, dstImgPath)
else:
dstImgPath = join(self.reportDir,
self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
dstImgPath,
scaleFactor=1)
return
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
""" Function to generate thumbnails for the report. Uses data from
self.thumbPaths.
===== Params =====
- firstThumbIndex: index from which we start generating thumbnails
- micScaleFactor: how much to reduce in size the micrographs.
"""
ih = ImageHandler()
numMics = len(self.thumbPaths[MIC_PATH])
for i in range(firstThumbIndex, numMics):
print('Generating images for mic %d' % (i+1))
# mic thumbnails
dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
if not exists(dstImgPath):
if self.micThumbSymlinks:
pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i], dstImgPath)
else:
ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
dstImgPath, scaleFactor=micScaleFactor,
flipOnY=True)
# shift plots
if SHIFT_THUMBS in self.thumbPaths:
dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i])
if not exists(dstImgPath):
pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i], dstImgPath)
# Psd thumbnails
# If there ARE thumbnail for the PSD (no ctf protocol and
# moviealignment hasn't computed it
if PSD_THUMBS in self.thumbPaths:
if self.ctfProtocol is None:
srcImgPath = self.thumbPaths[PSD_PATH][i]
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath) and srcImgPath is not None:
if srcImgPath.endswith('psd'):
psdImg1 = ih.read(srcImgPath)
psdImg1.convertPSD()
psdImg1.write(dstImgPath)
ih.computeThumbnail(dstImgPath, dstImgPath,
scaleFactor=1, flipOnY=True)
else:
pwutils.createAbsLink(srcImgPath, dstImgPath)
else:
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
dstImgPath, scaleFactor=1, flipOnY=True)
return
n = len(sys.argv)
if n < 2 or n > 5:
usage("Incorrect number of input parameters")
fromLocation = sys.argv[1]
projName = fromLocation.rsplit('/', 1)[-1] if n < 3 else sys.argv[2]
jsonFile = None if n < 4 else os.path.abspath(sys.argv[3])
location = None if n < 5 else sys.argv[4]
path = os.path.join(os.environ['SCIPION_HOME'], 'pyworkflow', 'gui',
'no-tkinter')
sys.path.insert(1, path)
# Create a new project
manager = Manager()
# if manager.hasProject(projName):
# usage("There is already a project with this name: %s"
# % pwutils.red(projName))
project = manager.createProject(projName, location=location)
# copy sqlite and make links of runs
pwutils.copyFile(os.path.join(fromLocation, PROJECT_DBNAME), project.path)
fromRuns = os.path.join(fromLocation, PROJECT_RUNS)
for r in os.listdir(fromRuns):
print " %s,%s" % (os.path.join(fromRuns,
r), os.path.join(project.runsPath, r))
pwutils.createAbsLink(os.path.join(fromRuns, r),
os.path.join(project.runsPath, r))
if len(sys.argv) == 4:
delay = sys.argv[3]
else:
delay = 30
inputFiles = glob(pwutils.expandPattern(inputPattern))
inputFiles.sort()
print "Input pattern: ", inputPattern
print "Input files: ", inputFiles
print "Delay: ", str(delay), " seconds."
print "Cleaning output directory: ", outputDir
pwutils.cleanPath(outputDir)
pwutils.makePath(outputDir)
aTime = int(delay)
n = 5
t = aTime / n
print "t=%s" % t
for f in inputFiles:
outputPath = os.path.join(outputDir, os.path.basename(f))
print "Linking %s -> %s" % (outputPath, f)
for i in range(n):
open(outputPath, 'w').close()
time.sleep(t)
pwutils.cleanPath(outputPath)
pwutils.createAbsLink(f, outputPath)
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if ':' in f:
f = f.split(':')[0]
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
def processMovieStep(self, movieDict, hasAlignment):
movie = emobj.Movie()
movie.setAcquisition(emobj.Acquisition())
if hasAlignment:
movie.setAlignment(emobj.MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
movieDoneFn = self._getMovieDone(movie)
if self.isContinued() and os.path.exists(movieDoneFn):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(movieDoneFn)
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createAbsLink(os.path.abspath(movieFn),
join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = emlib.image.ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
convertExt = self._getConvertExtension(newMovieName)
correctGain = self._doCorrectGain()
if convertExt or correctGain:
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
if convertExt:
newMovieName = pwutils.replaceExt(newMovieName, convertExt)
else:
newMovieName = '%s_corrected.%s' % os.path.splitext(
newMovieName)
outputMovieFn = os.path.join(movieFolder, newMovieName)
# If the protocols wants Scipion to apply the gain, then
# there is no reason to convert, since we can produce the
# output in the format expected by the program. In some cases,
# the alignment programs can directly deal with gain and dark
# correction images, so we don't need to apply it
if self._doCorrectGain():
self.info("Correcting gain and dark '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
gain, dark = self.getGainAndDark()
self.correctGain(inputMovieFn,
outputMovieFn,
gainFn=gain,
darkFn=dark)
else:
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
emlib.image.ImageHandler().convertStack(
inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if self._doMovieFolderCleanUp():
self._cleanMovieFolder(movieFolder)
# Mark this movie as finished
open(movieDoneFn, 'w').close()
def _link(mic, newName):
pwutils.createAbsLink(os.path.abspath(mic.getFileName()),
os.path.join(micDir, newName))
def importVolumesStep(self, pattern, samplingRate, setOrigCoord=False):
""" Copy images matching the filename pattern
Register other parameters.
"""
volSet = self._createSetOfVolumes()
vol = emobj.Volume()
if self.importFrom == self.IMPORT_FROM_FILES:
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
vol.setSamplingRate(samplingRate)
imgh = emlib.image.ImageHandler()
volSet.setSamplingRate(samplingRate)
for fileName, fileId in self.iterFiles():
x, y, z, n = imgh.getDimensions(fileName)
if fileName.endswith('.mrc') or fileName.endswith('.map'):
fileName += ':mrc'
if z == 1 and n != 1:
zDim = n
n = 1
else:
zDim = z
else:
zDim = z
origin = emobj.Transform()
if setOrigCoord:
origin.setShiftsTuple(self._getOrigCoord())
else:
origin.setShifts(x / -2. * samplingRate,
y / -2. * samplingRate,
zDim / -2. * samplingRate)
vol.setOrigin(origin) # read origin from form
if self.copyFiles or setOrigCoord:
newFileName = abspath(
self._getVolumeFileName(fileName, "mrc"))
emconv.Ccp4Header.fixFile(fileName, newFileName,
origin.getShifts(), samplingRate,
emconv.Ccp4Header.ORIGIN)
if self.setHalfMaps.get():
newFileName1 = abspath(
self._getVolumeFileName(self.half1map.get(),
"mrc"))
emconv.Ccp4Header.fixFile(self.half1map.get(),
newFileName1,
origin.getShifts(),
samplingRate,
emconv.Ccp4Header.ORIGIN)
newFileName2 = abspath(
self._getVolumeFileName(self.half2map.get(),
"mrc"))
emconv.Ccp4Header.fixFile(self.half2map.get(),
newFileName2,
origin.getShifts(),
samplingRate,
emconv.Ccp4Header.ORIGIN)
vol.setHalfMaps(
[relpath(newFileName1),
relpath(newFileName2)])
else:
newFileName = abspath(self._getVolumeFileName(fileName))
if fileName.endswith(':mrc'):
fileName = fileName[:-4]
pwutils.createAbsLink(fileName, newFileName)
if self.setHalfMaps.get():
pwutils.createAbsLink(
self.half1map.get(),
abspath(
self._getVolumeFileName(self.half1map.get())))
pwutils.createAbsLink(
self.half2map.get(),
abspath(
self._getVolumeFileName(self.half2map.get())))
vol.setHalfMaps([
relpath(
self._getVolumeFileName(self.half1map.get())),
relpath(
self._getVolumeFileName(self.half2map.get()))
])
# Make newFileName relative
# https://github.com/I2PC/scipion/issues/1935
newFileName = relpath(newFileName)
if n == 1:
vol.cleanObjId()
vol.setFileName(newFileName)
volSet.append(vol)
else:
for index in range(1, n + 1):
vol.cleanObjId()
vol.setLocation(index, newFileName)
volSet.append(vol)
else: # import from EMDB
self.info("Downloading map with ID = %s" % self.emdbId)
try:
localFileName, sampling, origin = \
fetch_emdb_map(self.emdbId,
self._getExtraPath(),
self._getTmpPath())
except Exception as e:
print(e)
return
# open volume and fill sampling and origin
vol.setSamplingRate(sampling)
vol.setFileName(localFileName)
from pwem.objects.data import Transform
originMat = Transform()
originMat.setShifts(origin[0], origin[1], origin[2])
vol.setOrigin(originMat)
if volSet.getSize() > 1:
self._defineOutputs(outputVolumes=volSet)
else:
self._defineOutputs(outputVolume=vol)
def convertInputStep(self, inputCoordinates, scale, kfold):
""" Converts a set of coordinates to box files and binaries to mrc
if needed. It generates 2 folders 1 for the box files and another for
the mrc files.
"""
micIds = []
coordSet = self.inputCoordinates.get()
setFn = coordSet.getFileName()
self.debug("Loading input db: %s" % setFn)
# Load set of coordinates with a user determined number of coordinates for the training step
enoughMicrographs = False
while True:
coordSet = SetOfCoordinates(filename=setFn)
coordSet._xmippMd = params.String()
coordSet.loadAllProperties()
for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]):
micIds.append(micAgg["_micId"])
if len(micIds) == self.micsForTraining.get():
enoughMicrographs = True
break
if enoughMicrographs:
break
else:
if coordSet.isStreamClosed():
raise Exception("We have a problem!!")
self.info("Not yet there: %s" % len(micIds))
import time
time.sleep(10)
# Create input folder and pre-processed micrographs folder
micDir = self._getFileName(TRAINING)
pw.utils.makePath(micDir)
prepDir = self._getFileName(TRAININGPREPROCESS)
pw.utils.makePath(prepDir)
ih = ImageHandler()
# Get a refreshed set of micrographs
micsFn = self.inputCoordinates.get().getMicrographs().getFileName()
# not updating, refresh problem
coordMics = SetOfMicrographs(filename=micsFn)
coordMics.loadAllProperties()
# Create a 0/1 list to mark micrographs for training/testing
n = len(micIds)
indexes = np.zeros(n, dtype='int8')
testSetImages = int((kfold / float(100)) * n)
# Both the training and the test data set should contain at least one micrograph
if testSetImages < 1:
requiredMinimumPercentage = (1 * 100 / n) + 1
testSetImages = int((requiredMinimumPercentage / float(100)) * n)
elif testSetImages == n:
testSetImages = int(0.99 * n)
indexes[:testSetImages] = 1
np.random.shuffle(indexes)
self.info('indexes: %s' % indexes)
# Write micrographs files
csvMics = [
CsvMicrographList(self._getFileName(TRAININGLIST), 'w'),
CsvMicrographList(self._getFileName(TRAININGTEST), 'w')
]
# Store the micId and indexes in micDict
micDict = {}
for i, micId in zip(indexes, micIds):
mic = coordMics[micId]
micFn = mic.getFileName()
baseFn = pw.utils.removeBaseExt(micFn)
inputFn = self._getFileName(TRAINING_MIC, **{"mic": baseFn})
if micFn.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micFn), inputFn)
else:
ih.convert(micFn, inputFn)
prepMicFn = self._getFileName(TRAININGPRE_MIC, **{"mic": baseFn})
csvMics[i].addMic(micId, prepMicFn)
micDict[micId] = i # store if train or test
for csv in csvMics:
csv.close()
# Write particles files
csvParts = [
CsvCoordinateList(self._getFileName(PARTICLES_TRAIN_TXT), 'w'),
CsvCoordinateList(self._getFileName(PARTICLES_TEST_TXT), 'w')
]
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId in micDict:
x = int(round(float(coord.getX()) / scale))
y = int(round(float(coord.getY()) / scale))
csvParts[micDict[micId]].addCoord(micId, x, y)
for csv in csvParts:
csv.close()
manager = Manager()
if not manager.hasProject(projName):
usage("Unexistent project: %s" % pwutils.red(projName))
if not os.path.exists(searchDir):
usage("Unexistent SEARCH_DIR: %s" % pwutils.red(searchDir))
project = manager.loadProject(projName)
runs = project.getRuns()
for prot in runs:
broken = False
if isinstance(prot, em.ProtImport):
for _, attr in prot.iterOutputEM():
fn = attr.getFiles()
for f in attr.getFiles():
if not os.path.exists(f):
if not broken:
broken = True
print "Found broken links in run: ", pwutils.magenta(prot.getRunName())
print " Missing: ", pwutils.magenta(f)
if os.path.islink(f):
print " -> ", pwutils.red(os.path.realpath(f))
newFile = pwutils.findFile(os.path.basename(f), searchDir, recursive=True)
if newFile:
print " Found file %s, creating link..." % newFile
print pwutils.green(" %s -> %s" % (f, newFile))
pwutils.createAbsLink(newFile, f)
def convertBinaryFiles(imgSet,
outputDir,
extension='mrcs',
forceConvert=False):
""" Convert binary images files to a format read by Relion.
Or create links if there is no need to convert the binary files.
Params:
imgSet: input image set to be converted.
outputDir: where to put the converted file(s)
extension: extension accepted by the program
forceConvert: if True, the files will be converted and no root will be used
Return:
A dictionary with old-file as key and new-file as value
If empty, not conversion was done.
"""
filesDict = {}
ih = ImageHandler()
outputRoot = outputDir if forceConvert else os.path.join(
outputDir, 'input')
# Get the extension without the dot
stackFiles = imgSet.getFiles()
ext = pwutils.getExt(next(iter(stackFiles)))[1:]
rootDir = pwutils.commonPath(list(stackFiles))
def getUniqueFileName(fn, extension):
""" Get an unique file for either link or convert files.
It is possible that the base name overlap if they come
from different runs. (like particles.mrcs after relion preprocess)
"""
newFn = os.path.join(outputRoot, pwutils.replaceBaseExt(fn, extension))
newRoot = pwutils.removeExt(newFn)
values = filesDict.values()
counter = 1
while newFn in values:
counter += 1
newFn = '%s_%05d.%s' % (newRoot, counter, extension)
return newFn
def createBinaryLink(fn):
""" Just create a link named .mrcs to Relion understand
that it is a binary stack file and not a volume.
"""
newFn = getUniqueFileName(fn, extension)
if not os.path.exists(newFn):
pwutils.createLink(fn, newFn)
print(" %s -> %s" % (newFn, fn))
return newFn
def convertStack(fn):
""" Convert from a format that is not read by Relion
to an spider stack.
"""
newFn = getUniqueFileName(fn, 'mrcs')
ih.convertStack(fn, newFn)
print(" %s -> %s" % (newFn, fn))
return newFn
def replaceRoot(fn):
""" Link create to the root folder, so just replace that
in the name, no need to do anything else.
"""
return fn.replace(rootDir, outputRoot)
if forceConvert:
print("convertBinaryFiles: forceConvert = True")
mapFunc = convertStack
elif ext == extension:
print("convertBinaryFiles: creating soft links.")
print(" Root: %s -> %s" % (outputRoot, rootDir))
mapFunc = replaceRoot
pwutils.createAbsLink(os.path.abspath(rootDir), outputRoot)
elif ext == 'mrc' and extension == 'mrcs':
print("convertBinaryFiles: creating soft links (mrcs -> mrc).")
mapFunc = createBinaryLink
elif ext.endswith('hdf'): # assume eman .hdf format
print("convertBinaryFiles: converting stacks. (%s -> %s)" %
(extension, ext))
mapFunc = convertStack
else:
mapFunc = None
if mapFunc is not None:
pwutils.makePath(outputRoot)
for fn in stackFiles:
newFn = mapFunc(fn) # convert or link
filesDict[fn] = newFn # map new filename
return filesDict
def testAssignCTF(self):
""" Test the particle extraction after importing another
SetOfMicrographs with a different micName but same ids.
We will use assign-ctf protocol and extract from the
newly imported mics with the assigned CTF.
For the other mics, we will just create symbolic links.
"""
# Create the links with a different micrograph name
micsPath = self.proj.getPath('otherMicrographs')
pwutils.makePath(micsPath)
for i in [6, 7, 8]:
micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
micLink = join(micsPath,
basename(micPath).replace('.mrc', '_DW.mrc'))
pwutils.createAbsLink(micPath, micLink)
protImportDW = self.proj.copyProtocol(self.protImport)
protImportDW.setObjLabel('import -mics DW')
protImportDW.filesPath.set(os.path.abspath(micsPath))
protImportDW.filesPattern.set('*_DW.mrc')
self.launchProtocol(protImportDW)
protAssignCTF = self.newProtocol(ProtCTFAssign)
protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
self.launchProtocol(protAssignCTF)
downFactor = 3.0
protExtract = self.newProtocol(XmippProtExtractParticles,
boxSize=183,
downsampleType=OTHER,
doDownsample=True,
downFactor=downFactor,
doInvert=False,
doFlip=False)
protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
protExtract.ctfRelations.set(self.protCTF.outputCTF)
protExtract.setObjLabel("extract-other (DW mics)")
self.launchProtocol(protExtract)
inputCoords = protExtract.inputCoordinates.get()
outputParts = protExtract.outputParticles
samplingCoords = inputCoords.getMicrographs().getSamplingRate()
samplingFinal = protImportDW.outputMicrographs.getSamplingRate(
) * downFactor
samplingMics = protExtract.inputMicrographs.get().getSamplingRate()
factor = samplingFinal / samplingCoords
self.assertIsNotNone(outputParts,
"There was a problem generating the output.")
def compare(objId, delta=1.0):
cx, cy = inputCoords[objId].getPosition()
px, py = outputParts[objId].getCoordinate().getPosition()
micNameCoord = inputCoords[objId].getMicName()
micNamePart = outputParts[objId].getCoordinate().getMicName()
self.assertAlmostEquals(cx / factor, px, delta=delta)
self.assertAlmostEquals(cy / factor, py, delta=delta)
self.assertEqual(
micNameCoord, micNamePart,
"The micName should be %s and its %s" %
(micNameCoord, micNamePart))
compare(45)
compare(229)
self.assertAlmostEqual(outputParts.getSamplingRate() / samplingMics,
downFactor, 1)
self._checkSamplingConsistency(outputParts)
usage("Incorrect number of input parameters")
pathToProj = os.path.abspath(sys.argv[1])
now = datetime.now()
tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(os.environ['SCIPION_USER_DATA'],
'tmp', tempSpace)
projectsDir = os.path.join(customUserData, 'projects')
pwutils.makePath(projectsDir)
print "Loading projects from:\n", projectsDir
projName = os.path.basename(pathToProj)
pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))
# Create a new project
manager = Manager(SCIPION_USER_DATA=customUserData)
proj = manager.loadProject(projName)
projPath = manager.getProjectPath(projName)
class EditorProjectWindow(ProjectWindow):
def close(self, e=None):
try:
print "Deleting temporary folder: ", customUserData
pwutils.cleanPath(customUserData)
except Exception, ex:
print "Error: ", ex
ProjectWindow.close(self, e)
def create_movies_project(request):
if request.is_ajax():
import os
from pyworkflow.object import Pointer
from pyworkflow.em.protocol import ProtImportMovies
from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment
# Create a new project
projectName = request.GET.get('projectName')
# Filename to use as test data
testDataKey = request.GET.get('testData')
manager = getServiceManager('movies')
writeCustomMenu(manager.protocols)
project = manager.createProject(projectName,
runsView=1,
hostsConf=manager.hosts,
protocolsConf=manager.protocols)
project.getSettings().setLifeTime(336) # 14 days * 24 hours
project.saveSettings()
# copyFile(customMenu, project.getPath('.config', 'protocols.conf'))
# Create symbolic link for uploads
projectPath = manager.getProjectPath(projectName)
dest = os.path.join(projectPath, 'Uploads')
os.rmdir(dest) #in movies uploads is created as a link
# @todo: this path to uploads dir should be configurable outside the code...
source = "/services/scipion/data/uploads/" + projectName
pwutils.path.makePath(source)
pwutils.createLink(source, dest)
# 1. Import movies
if testDataKey:
attr = getAttrTestFile(testDataKey)
path_test = attr['path']
for f in os.listdir(path_test):
# Create a symbolic link for each file
file_path = os.path.join(path_test, f)
source_file = os.path.join(source, f)
pwutils.createAbsLink(file_path, source_file)
label_import = "import movies (" + testDataKey + ")"
protImport = project.newProtocol(ProtImportMovies,
objLabel=label_import)
protImport.filesPath.set(attr["filesPath"])
protImport.voltage.set(attr['voltage'])
protImport.sphericalAberration.set(attr['sphericalAberration'])
protImport.amplitudeContrast.set(attr['amplitudeContrast'])
protImport.magnification.set(attr['magnification'])
protImport.samplingRate.set(attr['samplingRate'])
project.launchProtocol(protImport, wait=True)
else:
protImport = project.newProtocol(ProtImportMovies,
objLabel='import movies')
project.saveProtocol(protImport)
# 2. Movie Alignment
protMovAlign = project.newProtocol(ProtMovieAlignment)
protMovAlign.setObjLabel('xmipp - movie alignment')
protMovAlign.inputMovies.set(protImport)
protMovAlign.inputMovies.setExtended('outputMovies')
project.saveProtocol(protMovAlign)
return HttpResponse(mimetype='application/javascript')
if n < 2 or n > 3:
usage("Incorrect number of input parameters")
pathToProj = os.path.abspath(sys.argv[1])
now = datetime.now()
tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(pw.Config.SCIPION_USER_DATA, 'tmp', tempSpace)
projectsDir = os.path.join(customUserData, 'projects')
pwutils.makePath(projectsDir)
print("Loading projects from:\n", projectsDir)
projName = os.path.basename(pathToProj)
pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))
# Create a new project
manager = Manager(workspace=customUserData)
proj = manager.loadProject(projName)
projPath = manager.getProjectPath(projName)
class EditorProjectWindow(ProjectWindow):
def close(self, e=None):
try:
print("Deleting temporary folder: ", customUserData)
pwutils.cleanPath(customUserData)
except Exception as ex:
print("Error: ", ex)
def create_movies_project(request):
if request.is_ajax():
import os
from pyworkflow.object import Pointer
from pyworkflow.em.protocol import ProtImportMovies
from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment
# Create a new project
projectName = request.GET.get('projectName')
# Filename to use as test data
testDataKey = request.GET.get('testData')
manager = getServiceManager('movies')
writeCustomMenu(manager.protocols)
project = manager.createProject(projectName, runsView=1,
hostsConf=manager.hosts,
protocolsConf=manager.protocols
)
project.getSettings().setLifeTime(14)
project.saveSettings()
# copyFile(customMenu, project.getPath('.config', 'protocols.conf'))
# Create symbolic link for uploads
projectPath = manager.getProjectPath(projectName)
dest = os.path.join(projectPath,'Uploads')
# @todo: this path to uploads dir should be configurable outside the code...
source = "/services/scipion/data/uploads/"+ projectName
pwutils.path.makePath(source)
pwutils.createLink(source, dest)
# 1. Import movies
if testDataKey :
attr = getAttrTestFile(testDataKey)
path_test = attr['path']
for f in os.listdir(path_test):
# Create a symbolic link for each file
file_path = os.path.join(path_test, f)
source_file = os.path.join(source, f)
pwutils.createAbsLink(file_path, source_file)
label_import = "import movies ("+ testDataKey +")"
protImport = project.newProtocol(ProtImportMovies, objLabel=label_import)
protImport.filesPath.set(attr["filesPath"])
protImport.voltage.set(attr['voltage'])
protImport.sphericalAberration.set(attr['sphericalAberration'])
protImport.amplitudeContrast.set(attr['amplitudeContrast'])
protImport.magnification.set(attr['magnification'])
protImport.samplingRate.set(attr['samplingRate'])
project.launchProtocol(protImport, wait=True)
else:
protImport = project.newProtocol(ProtImportMovies, objLabel='import movies')
project.saveProtocol(protImport)
# 2. Movie Alignment
protMovAlign = project.newProtocol(ProtMovieAlignment)
protMovAlign.setObjLabel('xmipp - movie alignment')
protMovAlign.inputMovies.set(protImport)
protMovAlign.inputMovies.setExtended('outputMovies')
project.saveProtocol(protMovAlign)
return HttpResponse(mimetype='application/javascript')