def testOneRemoteHostToLocal(self):
tempFolder = "oneRemoteHostToLocal"
filePaths = {}
checkPathList = []
remoteSourceFilePathList2 = ft.getRemoteFolderFiles(
self.remoteHostName2, self.remoteUserName2, self.remotePassword2,
self.remoteSourceFolder2)
for remoteSourceFilePath in remoteSourceFilePathList2:
remoteSourceFileName = basename(remoteSourceFilePath)
targetFilePath = join(self.localTargetFolder, tempFolder, "test2",
remoteSourceFileName)
filePaths[remoteSourceFilePath] = targetFilePath
checkPathList.append(targetFilePath)
self.fileTransfer.transferFilesFrom(filePaths,
self.remoteHostName2,
self.remoteUserName2,
self.remotePassword2,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)
passTest = len(
self.fileTransfer.checkFiles(checkPathList,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)) == 0
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
def testLocalToOneRemoteHost(self):
tempFolder = "localToOneRemote"
filePaths = {}
checkPathList = []
sourceFilesPathList = getFiles(self.localSourceFolder)
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath = join(self.remoteTargetFolder1, tempFolder,
sourceFileName)
filePaths[sourceFilePath] = targetFilePath
checkPathList.append(self.hostsRefs[0] + ":" + targetFilePath)
self.fileTransfer.transferFilesTo(filePaths,
self.remoteHostName1,
self.remoteUserName1,
self.remotePassword1,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)
passTest = len(
self.fileTransfer.checkFiles(checkPathList,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)) == 0
ft.removeRemoteFolder(self.remoteHostName1, self.remoteUserName1,
self.remotePassword1,
join(self.remoteTargetFolder1, tempFolder))
self.assertTrue(passTest)
def testLocalToLocal(self):
tempFolder = "localToLocal"
filePaths = {}
sourceFilesPathList = getFiles(self.localSourceFolder)
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath = join(self.localTargetFolder, tempFolder,
sourceFileName)
targetFilePathList = []
targetFilePathList.append(targetFilePath)
filePaths[sourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(
self.fileTransfer.checkFiles(checkPathList,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
def testSeveralRemoteHostsToLocal(self):
tempFolder = "severalRemoteToLocal"
filePaths = {}
remoteSourceFilePathList1 = ft.getRemoteFolderFiles(
self.remoteHostName1, self.remoteUserName1, self.remotePassword1,
self.remoteSourceFolder1)
remoteSourceFilePathList2 = ft.getRemoteFolderFiles(
self.remoteHostName2, self.remoteUserName2, self.remotePassword2,
self.remoteSourceFolder2)
for remoteSourceFilePath in remoteSourceFilePathList1:
targetFilePathList = []
targetFilePathList.append(
join(self.localTargetFolder, tempFolder, "test1",
basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[0] + ":" +
remoteSourceFilePath] = targetFilePathList
for remoteSourceFilePath in remoteSourceFilePathList2:
targetFilePathList = []
targetFilePathList.append(
join(self.localTargetFolder, tempFolder, "test2",
basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[1] + ":" +
remoteSourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(
self.fileTransfer.checkFiles(checkPathList,
self.hostPasswords,
gatewayHosts=self.gatewayHosts,
numberTrials=self.numberTrials,
forceOperation=self.forceOperation,
operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
def testLocalToOneRemoteHost(self):
tempFolder = "localToOneRemote"
filePaths = {}
checkPathList = []
sourceFilesPathList = getFiles(self.localSourceFolder)
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath = join(self.remoteTargetFolder1, tempFolder, sourceFileName)
filePaths[sourceFilePath] = targetFilePath
checkPathList.append(self.hostsRefs[0] + ":" + targetFilePath)
self.fileTransfer.transferFilesTo(filePaths, self.remoteHostName1, self.remoteUserName1, self.remotePassword1, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
ft.removeRemoteFolder(self.remoteHostName1, self.remoteUserName1, self.remotePassword1, join(self.remoteTargetFolder1, tempFolder))
self.assertTrue(passTest)
def testOneRemoteHostToLocal(self):
tempFolder = "oneRemoteHostToLocal"
filePaths = {}
checkPathList = []
remoteSourceFilePathList2 = ft.getRemoteFolderFiles(self.remoteHostName2, self.remoteUserName2, self.remotePassword2, self.remoteSourceFolder2)
for remoteSourceFilePath in remoteSourceFilePathList2:
remoteSourceFileName = basename(remoteSourceFilePath)
targetFilePath = join(self.localTargetFolder, tempFolder, "test2", remoteSourceFileName)
filePaths[remoteSourceFilePath] = targetFilePath
checkPathList.append(targetFilePath)
self.fileTransfer.transferFilesFrom(filePaths, self.remoteHostName2, self.remoteUserName2, self.remotePassword2, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
def testLocalToLocal(self):
tempFolder = "localToLocal"
filePaths = {}
sourceFilesPathList = getFiles(self.localSourceFolder)
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath = join(self.localTargetFolder, tempFolder, sourceFileName)
targetFilePathList = []
targetFilePathList.append(targetFilePath)
filePaths[sourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Armstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Angstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def testLocalToSeveralRemoteHosts(self):
tempFolder = "localToSeveralRemote"
filePaths = {}
sourceFilesPathList = getFiles(self.localSourceFolder)
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath1 = join(self.remoteTargetFolder1, tempFolder, sourceFileName)
targetFilePath2 = join(self.remoteTargetFolder2, tempFolder, sourceFileName)
targetFilePathList = []
targetFilePathList.append(self.hostsRefs[0] + ":" + targetFilePath1)
targetFilePathList.append(self.hostsRefs[1] + ":" + targetFilePath2)
filePaths[sourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
ft.removeRemoteFolder(self.remoteHostName1, self.remoteUserName1, self.remotePassword1, join(self.remoteTargetFolder1, tempFolder))
ft.removeRemoteFolder(self.remoteHostName2, self.remoteUserName2, self.remotePassword2, join(self.remoteTargetFolder2, tempFolder))
self.assertTrue(passTest)
def testAll(self):
tempFolder = "allTransfer"
filePaths = {}
sourceFilesPathList = getFiles(self.localSourceFolder)
remoteSourceFilePathList1 = ft.getRemoteFolderFiles(self.remoteHostName1, self.remoteUserName1, self.remotePassword1, self.remoteSourceFolder1)
remoteSourceFilePathList2 = ft.getRemoteFolderFiles(self.remoteHostName2, self.remoteUserName2, self.remotePassword2, self.remoteSourceFolder2)
localUserName = getLocalUserName()
localHostname = getLocalHostName()
for sourceFilePath in sourceFilesPathList:
sourceFileName = basename(sourceFilePath)
targetFilePath1 = join(self.remoteTargetFolder1, tempFolder, sourceFileName)
targetFilePath2 = join(self.remoteTargetFolder2, tempFolder, sourceFileName)
targetFilePath3 = join (self.localTargetFolder, tempFolder, sourceFileName)
targetFilePathList = []
targetFilePathList.append(self.hostsRefs[0] + ":" + targetFilePath1)
targetFilePathList.append(self.hostsRefs[1] + ":" + targetFilePath2)
targetFilePathList.append(localUserName + "@" + localHostname + ":" + targetFilePath3)
filePaths[sourceFilePath] = targetFilePathList
for remoteSourceFilePath in remoteSourceFilePathList1:
targetFilePathList = []
targetFilePathList.append(join(self.localTargetFolder, tempFolder, "test1", basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[0] + ":" + remoteSourceFilePath] = targetFilePathList
for remoteSourceFilePath in remoteSourceFilePathList2:
targetFilePathList = []
targetFilePathList.append(join(self.localTargetFolder, tempFolder, "test2", basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[1] + ":" + remoteSourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
cleanPath(join(self.localTargetFolder, tempFolder))
ft.removeRemoteFolder(self.remoteHostName1, self.remoteUserName1, self.remotePassword1, join(self.remoteTargetFolder1, tempFolder))
ft.removeRemoteFolder(self.remoteHostName2, self.remoteUserName2, self.remotePassword2, join(self.remoteTargetFolder2, tempFolder))
self.assertTrue(passTest)
def testSeveralRemoteHostsToLocal(self):
tempFolder = "severalRemoteToLocal"
filePaths = {}
remoteSourceFilePathList1 = ft.getRemoteFolderFiles(self.remoteHostName1, self.remoteUserName1, self.remotePassword1, self.remoteSourceFolder1)
remoteSourceFilePathList2 = ft.getRemoteFolderFiles(self.remoteHostName2, self.remoteUserName2, self.remotePassword2, self.remoteSourceFolder2)
for remoteSourceFilePath in remoteSourceFilePathList1:
targetFilePathList = []
targetFilePathList.append(join(self.localTargetFolder, tempFolder, "test1", basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[0] + ":" + remoteSourceFilePath] = targetFilePathList
for remoteSourceFilePath in remoteSourceFilePathList2:
targetFilePathList = []
targetFilePathList.append(join(self.localTargetFolder, tempFolder, "test2", basename(remoteSourceFilePath)))
filePaths[self.hostsRefs[1] + ":" + remoteSourceFilePath] = targetFilePathList
self.fileTransfer.transferFiles(filePaths, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
checkPathList = ft.getFilePathList(filePaths)
passTest = len(self.fileTransfer.checkFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)) == 0
# self.fileTransfer.deleteFiles(checkPathList, self.hostPasswords, gatewayHosts=self.gatewayHosts, numberTrials=self.numberTrials, forceOperation=self.forceOperation, operationId=self.operationId)
cleanPath(join(self.localTargetFolder, tempFolder))
self.assertTrue(passTest)
