def tes_copyToPyarchPath(self):
# Can only be tested on a computer which can write to /data/pyarch
testPath = "/data/visitor/mx415/cm01/20180101/RAW_DATA/test2/CWAT_ESRF_RawData_K2/170620_TMV_1/Images-Disc1/GridSquare_20174003/Data/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198.mrc"
if not os.path.exists(os.path.dirname(testPath)):
os.makedirs(testPath)
if not os.path.exists(testPath):
f = open(testPath)
f.write("Test")
f.close()
pyarchFilePath = UtilsPath.getPyarchFilePath(testPath)
print(pyarchFilePath)
pyarchFilePath = UtilsPath.copyToPyarchPath(testPath)
print(pyarchFilePath)
def test_getCtfMetaData(self):
workingDir = "/scisoft/pxsoft/data/cryoem/testRunData/20190715/000126_ProtGctf"
mrcFilePath = "/scisoft/pxsoft/data/cryoem/testRunData/20190715/000064_ProtMotionCorr/extra/GridSquare_7828225_Data_FoilHole_8853794_Data_7832898_7832899_20190711_0913-4443_aligned_mic.mrc"
dictResult = UtilsPath.getCtfMetaData(workingDir, mrcFilePath)
dictRef = {
'Angle':
'35.60',
'CCC':
'-0.076290',
'Defocus_U':
'23173.92',
'Defocus_V':
'22988.54',
'Phase_shift':
None,
'estimatedBfactor':
'82.54',
'logFilePath':
'/scisoft/pxsoft/data/cryoem/testRunData/20190715/000126_ProtGctf/logs/run.log',
'resolutionLimit':
'3.381',
'spectraImageFullPath':
'/scisoft/pxsoft/data/cryoem/testRunData/20190715/000126_ProtGctf/extra/GridSquare_7828225_Data_FoilHole_8853794_Data_7832898_7832899_20190711_0913-4443_aligned_mic_ctf.mrc',
'spectraImageSnapshotFullPath':
'/scisoft/pxsoft/data/cryoem/testRunData/20190715/000126_ProtGctf/extra/GridSquare_7828225_Data_FoilHole_8853794_Data_7832898_7832899_20190711_0913-4443_aligned_mic_ctf.jpeg'
}
self.assertEqual(dictRef, dictResult)
def test_getMovieFileNameParametersFromMotioncorrPath(self):
# Test 1
mrcFullPath1 = "/mntdirect/_data_visitor/mx2006/cm01/20180123/PROCESSED_DATA/sa1mgml/saribs/Runs/000058_ProtMotionCorr/extra/GridSquare_22879601_Data_FoilHole_22888571_Data_22885672_22885673_20180124_0845-4482_aligned_mic.mrc"
dictResult1 = UtilsPath.getMovieFileNameParametersFromMotioncorrPath(
mrcFullPath1)
refDict1 = {
'data': '_Data_',
'date': '20180124',
'directory':
'/mntdirect/_data_visitor/mx2006/cm01/20180123/PROCESSED_DATA/sa1mgml/saribs/Runs/000058_ProtMotionCorr/extra',
'extra': '_aligned_mic',
'gridSquare': 'GridSquare_22879601',
'hour': '0845',
'id1': '22888571',
'id2': '22885672',
'id3': '22885673',
'movieName':
'FoilHole_22888571_Data_22885672_22885673_20180124_0845-4482',
'movieNumber': '4482',
'prefix': 'FoilHole',
'suffix': 'mrc'
}
self.assertEqual(refDict1, dictResult1)
# Test 2
mrcFullPath2 = "/users/svensson/ScipionUserData/projects/TestPipeLine/Runs/000859_ProtMotionCorr/extra/FoilHole_19150795_Data_19148847_19148848_20170619_2101-0344_aligned_mic.mrc"
dictResult2 = UtilsPath.getMovieFileNameParametersFromMotioncorrPath(
mrcFullPath2)
refDict2 = {
'data': None,
'date': '20170619',
'directory':
'/users/svensson/ScipionUserData/projects/TestPipeLine/Runs/000859_ProtMotionCorr/extra',
'extra': '_aligned_mic',
'gridSquare': None,
'hour': '2101',
'id1': '19150795',
'id2': '19148847',
'id3': '19148848',
'movieName':
'FoilHole_19150795_Data_19148847_19148848_20170619_2101-0344',
'movieNumber': '0344',
'prefix': 'FoilHole',
'suffix': 'mrc'
}
self.assertEqual(refDict2, dictResult2)
def test_getShiftData(self):
mrcFilePath = "/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/extra/FoilHole_9208892_Data_9209286_9209287_20171109_1540-0539_aligned_mic.mrc"
dictResult = UtilsPath.getShiftData(mrcFilePath)
dictRef = {
'averageMotionPerFrame': 6.3,
'noPoints': 40,
'totalMotion': 250.6
}
self.assertEqual(dictRef, dictResult)
def test_getMovieFileNameParameters(self):
# Test1
movieFullPath1 = "/users/svensson/cryoem/CWAT_ESRF_RawData_K2/170619_bGal1/Images-Disc1/GridSquare_19141127/Data/FoilHole_19150795_Data_19148847_19148848_20170619_2101-0344.mrc"
dictResult1 = UtilsPath.getMovieFileNameParameters(movieFullPath1)
refDict1 = {
'date': '20170619',
'directory':
'/users/svensson/cryoem/CWAT_ESRF_RawData_K2/170619_bGal1/Images-Disc1/GridSquare_19141127/Data',
'extra': '',
'gridSquare': 'GridSquare_19141127',
'hour': '2101',
'id1': '19150795',
'id2': '19148847',
'id3': '19148848',
'movieName':
'FoilHole_19150795_Data_19148847_19148848_20170619_2101-0344',
'movieNumber': '0344',
'prefix': 'FoilHole',
'suffix': 'mrc'
}
self.assertEqual(refDict1, dictResult1)
# Test 2
mrcFullPath2 = "/data/visitor/mx415/cm01/20180315/RAW_DATA/EPU_BSA_grid5_2mg_2_test8/Images-Disc1/GridSquare_15806527/Data/FoilHole_15814308_Data_15808956_15808957_20180317_1109-17665.mrc"
dictResult2 = UtilsPath.getMovieFileNameParameters(mrcFullPath2)
refDict2 = {
'date': '20180317',
'directory':
'/data/visitor/mx415/cm01/20180315/RAW_DATA/EPU_BSA_grid5_2mg_2_test8/Images-Disc1/GridSquare_15806527/Data',
'extra': '',
'gridSquare': 'GridSquare_15806527',
'hour': '1109',
'id1': '15814308',
'id2': '15808956',
'id3': '15808957',
'movieName':
'FoilHole_15814308_Data_15808956_15808957_20180317_1109-17665',
'movieNumber': '17665',
'prefix': 'FoilHole',
'suffix': 'mrc'
}
self.assertEqual(refDict2, dictResult2)
def test_getMovieJpegMrcXml(self):
movieFullPath = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448-3385.mrc"
jpeg, mrc, xml, gridSquareThumbNail = UtilsPath.getMovieJpegMrcXml(
movieFullPath)
jpegRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.jpg"
self.assertEqual(jpegRef, jpeg)
mrcRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.mrc"
self.assertEqual(mrcRef, mrc)
xmlRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.xml"
self.assertEqual(xmlRef, xml)
gridSquareThumbNailRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/GridSquare_20171123_144119.jpg"
self.assertEqual(gridSquareThumbNailRef, gridSquareThumbNail)
def test_getPyarchFilePath(self):
mrcFilePath = "/data/visitor/mx415/cm01/20171108/RAW_DATA/test2/CWAT_ESRF_RawData_K2/170620_TMV_1/Images-Disc1/GridSquare_20174003/Data/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198.mrc"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2017/cm01/mx415/20171108/RAW_DATA/test2/CWAT_ESRF_RawData_K2/170620_TMV_1/Images-Disc1/GridSquare_20174003/Data/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198.mrc",
pyarchFilePath)
mrcFilePath = "/mntdirect/_data_visitor/mx415/cm01/20171108/PROCESSED_DATA/test2/CWAT_ESRF_RawData_K2/170620_TMV_1/Images-Disc1/GridSquare_20174003/Data/GridSquare_20174003_test2/Runs/000056_ProtMotionCorr/extra/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198_aligned_mic.mrc"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2017/cm01/mx415/20171108/PROCESSED_DATA/test2/CWAT_ESRF_RawData_K2/170620_TMV_1/Images-Disc1/GridSquare_20174003/Data/GridSquare_20174003_test2/Runs/000056_ProtMotionCorr/extra/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198_aligned_mic.mrc",
pyarchFilePath)
# ihls2975...
mrcFilePath = "/mntdirect/_data_cm01_inhouse/Hons/IH-LS-2975/RAW_DATA/grid1/Images-Disc1/GridSquare_20174003/Data/GridSquare_20174003_test2/Runs/000056_ProtMotionCorr/extra/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198_aligned_mic.mrc"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2018/cm01/ihls2975/Hons/RAW_DATA/grid1/Images-Disc1/GridSquare_20174003/Data/GridSquare_20174003_test2/Runs/000056_ProtMotionCorr/extra/FoilHole_20182354_Data_20179605_20179606_20170620_1523-1198_aligned_mic.mrc",
pyarchFilePath)
mrcFilePath = "/mntdirect/_data_cm01_inhouse/opcm01/20171108/RAW_DATA/nicetest/Frame.mrc"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2019/20171108/opcm01/RAW_DATA/nicetest/Frame.mrc",
pyarchFilePath)
# CRYOEM-25 : New paths for inhouse research...
mrcFilePath = "/data/cm01/cmihr2/IH-LS3198/20181203/RAW_DATA/EPU_IH_LS3198/Images-Disc1/GridSquare_3087556/Data/FoilHole_4118111_Data_5127341_5127342_20181205_1023.jpg"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2018/cm01/IH-LS3198/20181203/RAW_DATA/EPU_IH_LS3198/Images-Disc1/GridSquare_3087556/Data/FoilHole_4118111_Data_5127341_5127342_20181205_1023.jpg",
pyarchFilePath)
mrcFilePath = "/mntdirect/_data_cm01_cmihr2/IH-LS3198/20181203/RAW_DATA/EPU_IH_LS3198/Images-Disc1/GridSquare_3087308/GridSquare_20181204_115820.jpg"
pyarchFilePath = UtilsPath.getPyarchFilePath(mrcFilePath)
self.assertEqual(
"/data/pyarch/2019/cm01/IH-LS3198/20181203/RAW_DATA/EPU_IH_LS3198/Images-Disc1/GridSquare_3087308/GridSquare_20181204_115820.jpg",
pyarchFilePath)
def test_getAlignMoviesPngLogFilePath(self):
mrcFilePath = "/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/extra/FoilHole_9208892_Data_9209286_9209287_20171109_1540-0539_aligned_mic.mrc"
dictResult = UtilsPath.getAlignMoviesPngLogFilePath(mrcFilePath)
dictRef = {
'doseWeightMrc':
'/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/extra/FoilHole_9208892_Data_9209286_9209287_20171109_1540-0539_aligned_mic_DW.mrc',
'globalShiftPng':
'/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/extra/FoilHole_9208892_Data_9209286_9209287_20171109_1540-0539_global_shifts.png',
'logFileFullPath':
'/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/logs/run.log',
'thumbnailPng':
'/scisoft/pxsoft/data/cryoem/testRunData/20171113/Runs/000056_ProtMotionCorr/extra/FoilHole_9208892_Data_9209286_9209287_20171109_1540-0539_thumbnail.png'
}
self.assertEqual(dictRef, dictResult)
def test_getXmlMetaData(self):
xmlMetaDataFullPath = "/scisoft/pxsoft/data/cryoem/testMetaData/supervisor_20171115_150623/Images-Disc1/GridSquare_15441375/Data/FoilHole_15445484_Data_15444206_15444207_20171115_1620.xml"
dictResult = UtilsPath.getXmlMetaData(xmlMetaDataFullPath)
dictRef = {
'accelerationVoltage': '300000',
'acquisitionDateTime': '2017-11-15T16:20:52.2530023+01:00',
'dose': '2.3276455783574426E+21',
'nominalMagnification': '130000',
'numberOffractions': '30',
'phasePlateUsed': 'true',
'positionX': '0.00026971729600000011',
'positionY': '0.00014176793600000005',
'superResolutionFactor': '1'
}
self.assertEqual(dictRef, dictResult)
if proposal is None:
print("WARNING! No data will be uploaded to ISPyB.")
db = 3
else:
print("Proposal: {0}".format(proposal))
if proposal == "mx415":
# Use valid data base
print("ISPyB valid data base used")
db = 1
else:
# Use productiond data base
print("ISPyB production data base used")
db = 0
jpeg, mrc, xml, gridSquareThumbNail = UtilsPath.getMovieJpegMrcXml(
firstMovieFullPath)
print("Metadata file: {0}".format(xml))
dictResults = UtilsPath.getXmlMetaData(xml)
doPhaseShiftEstimation = dictResults["phasePlateUsed"]
nominalMagnification = int(dictResults["nominalMagnification"])
superResolutionFactor = int(dictResults["superResolutionFactor"])
if samplingRate is None:
samplingRate = 1.1 / float(superResolutionFactor)
print("doPhaseShiftEstimation: {0}".format(doPhaseShiftEstimation))
print("nominalMagnification: {0}".format(nominalMagnification))
print("superResolutionFactor: {0}".format(superResolutionFactor))
print("samplingRate: {0}".format(samplingRate))
def uploadCTFMicrographs(self, prot):
self.info("ESRF ISPyB upload CTF results")
workingDir = os.path.join(self.currentDir, str(prot.workingDir))
for ctf in self.iter_updated_set(prot.outputCTF):
micrographFullPath = ctf.getMicrograph().getFileName()
dictFileNameParameters = UtilsPath.getMovieFileNameParametersFromMotioncorrPath(micrographFullPath)
movieName = dictFileNameParameters["movieName"]
if movieName in self.allParams and not "CTFid" in self.allParams[movieName]:
self.info("CTF: movie {0}".format(os.path.basename(self.allParams[movieName]["movieFullPath"])))
movieFullPath = self.allParams[movieName]["movieFullPath"]
spectraImageSnapshotFullPath = None
spectraImageSnapshotPyarchPath = None
spectraImageFullPath = None
spectraImagePyarchPath = None
defocusU = None
defocusV = None
angle = None
crossCorrelationCoefficient = None
phaseShift = None
resolutionLimit = None
estimatedBfactor = None
try:
dictResults = UtilsPath.getCtfMetaData(workingDir, micrographFullPath)
spectraImageSnapshotFullPath = dictResults["spectraImageSnapshotFullPath"]
spectraImageSnapshotPyarchPath = UtilsPath.copyToPyarchPath(spectraImageSnapshotFullPath)
spectraImageFullPath = dictResults["spectraImageFullPath"]
spectraImagePyarchPath = None
defocusU = dictResults["Defocus_U"]
defocusV = dictResults["Defocus_V"]
angle = dictResults["Angle"]
crossCorrelationCoefficient = dictResults["CCC"]
phaseShift = dictResults["Phase_shift"]
resolutionLimit = dictResults["resolutionLimit"]
estimatedBfactor = dictResults["estimatedBfactor"]
shutil.copy(spectraImageFullPath, self.allParams[movieName]["processDir"])
shutil.copy(dictResults["logFilePath"], self.allParams[movieName]["processDir"])
except:
self.info("ERROR uploading CTF for movie {0}".format(movieFullPath))
traceback.print_exc()
ctfObject = None
logFilePath = UtilsPath.copyToPyarchPath(dictResults["logFilePath"])
# self.info("proposal : {0}".format(self.proposal))
# self.info("movieFullPath : {0}".format(movieFullPath))
# self.info("spectraImageSnapshotFullPath : {0}".format(spectraImageSnapshotPyarchPath))
# self.info("spectraImageFullPath : {0}".format(spectraImagePyarchPath))
# self.info("defocusU : {0}".format(defocusU))
# self.info("defocusV : {0}".format(defocusV))
# self.info("angle : {0}".format(angle))
# self.info("crossCorrelationCoefficient : {0}".format(crossCorrelationCoefficient))
# self.info("resolutionLimit : {0}".format(resolutionLimit))
# self.info("estimatedBfactor : {0}".format(estimatedBfactor))
# self.info("logFilePath : {0}".format(logFilePath))
try:
ctfObject = self.client.service.addCTF(proposal=self.proposal,
movieFullPath=movieFullPath,
spectraImageSnapshotFullPath=spectraImageSnapshotPyarchPath,
spectraImageFullPath=spectraImagePyarchPath,
defocusU=defocusU,
defocusV=defocusV,
angle=angle,
crossCorrelationCoefficient=crossCorrelationCoefficient,
resolutionLimit=resolutionLimit,
estimatedBfactor=estimatedBfactor,
logFilePath=logFilePath)
except:
self.info("ERROR uploading CTF for movie {0}".format(movieFullPath))
traceback.print_exc()
ctfObject = None
if ctfObject is not None:
CTFid = ctfObject.CTFid
else:
self.info("ERROR: ctfObject is None!")
CTFid = None
self.allParams[movieName]["CTFid"] = CTFid
self.allParams[movieName]["phaseShift"] = phaseShift
self.allParams[movieName]["defocusU"] = defocusU
self.allParams[movieName]["defocusV"] = defocusV
self.allParams[movieName]["angle"] = angle
self.allParams[movieName]["crossCorrelationCoefficient"] = crossCorrelationCoefficient
self.allParams[movieName]["resolutionLimit"] = resolutionLimit
self.info("CTF done, CTFid = {0}".format(CTFid))
def uploadAlignMovies(self, prot):
self.info("ESRF ISPyB upload motion corr results")
for micrograph in self.iter_updated_set(prot.outputMicrographs):
micrographFullPath = os.path.join(self.currentDir, micrograph.getFileName())
#self.info("Motion corr micrographFullPath: {0}".format(micrographFullPath))
dictFileNameParameters = UtilsPath.getMovieFileNameParametersFromMotioncorrPath(micrographFullPath)
movieName = dictFileNameParameters["movieName"]
#self.info("Motion corr movie name: {0}".format(movieName))
if movieName in self.allParams and not "motionCorrectionId" in self.allParams[movieName]:
self.info("Align movies: movie {0}".format(os.path.basename(self.allParams[movieName]["movieFullPath"])))
movieFullPath = self.allParams[movieName]["movieFullPath"]
dictResult = UtilsPath.getAlignMoviesPngLogFilePath(micrographFullPath)
driftPlotFullPath = dictResult["globalShiftPng"]
if "doseWeightMrc" in dictResult:
correctedDoseMicrographFullPath = dictResult["doseWeightMrc"]
else:
correctedDoseMicrographFullPath = None
if "thumbnailPng" in dictResult:
micrographSnapshotFullPath = dictResult["thumbnailPng"]
else:
micrographSnapshotFullPath = None
dictShift = UtilsPath.getShiftData(micrographFullPath)
if "totalMotion" in dictShift:
totalMotion = dictShift["totalMotion"]
else:
totalMotion = None
if "averageMotionPerFrame" in dictShift:
averageMotionPerFrame = dictShift["averageMotionPerFrame"]
else:
averageMotionPerFrame = None
logFileFullPath = dictResult["logFileFullPath"]
firstFrame = 1
lastFrame = self.allParams[movieName]["imagesCount"]
dosePerFrame = self.allParams[movieName]["dosePerFrame"]
doseWeight = None
driftPlotPyarchPath = UtilsPath.copyToPyarchPath(driftPlotFullPath)
micrographPyarchPath = None
correctedDoseMicrographPyarchPath = None
micrographSnapshotPyarchPath = UtilsPath.copyToPyarchPath(micrographSnapshotFullPath)
logFilePyarchPath = UtilsPath.copyToPyarchPath(logFileFullPath)
shutil.copy(micrographFullPath, self.allParams[movieName]["processDir"])
shutil.copy(correctedDoseMicrographFullPath, self.allParams[movieName]["processDir"])
shutil.copy(logFileFullPath, self.allParams[movieName]["processDir"])
try:
motionCorrectionObject = self.client.service.addMotionCorrection(proposal=self.proposal,
movieFullPath=movieFullPath,
firstFrame=firstFrame,
lastFrame=lastFrame,
dosePerFrame=dosePerFrame,
doseWeight=doseWeight,
totalMotion=totalMotion,
averageMotionPerFrame=averageMotionPerFrame,
driftPlotFullPath=driftPlotPyarchPath,
micrographFullPath=micrographPyarchPath,
correctedDoseMicrographFullPath=correctedDoseMicrographPyarchPath,
micrographSnapshotFullPath=micrographSnapshotPyarchPath,
logFileFullPath=logFilePyarchPath)
except:
self.info("ERROR uploading motion correction for movie {0}".format(movieFullPath))
traceback.print_exc()
motionCorrectionObject = None
if motionCorrectionObject is not None:
motionCorrectionId = motionCorrectionObject.motionCorrectionId
else:
self.info("ERROR: motionCorrectionObject is None!")
motionCorrectionId = None
self.allParams[movieName]["motionCorrectionId"] = motionCorrectionId
self.allParams[movieName]["totalMotion"] = totalMotion
self.allParams[movieName]["averageMotionPerFrame"] = averageMotionPerFrame
self.info("Align movies done, motionCorrectionId = {0}".format(motionCorrectionId))
def uploadImportMovies(self, prot):
for movieFullPath in prot.getMatchFiles():
listMovieFullPath = []
for movieName in self.allParams:
if "movieFullPath" in self.allParams[movieName] and "movieId" in self.allParams[movieName]:
if self.allParams[movieName]["movieId"] is not None:
listMovieFullPath.append(self.allParams[movieName]["movieFullPath"])
# listMovieFullPath = [ self.allParams[movieName]["movieFullPath"] for movieName in self.allParams if "movieFullPath" in self.allParams[movieName]]
if movieFullPath in listMovieFullPath:
pass
#self.info("Movie already uploaded: {0}".format(movieFullPath))
else:
self.info("Import movies: movieFullPath: {0}".format(movieFullPath))
# Create "process" directory
dictFileNameParameters = UtilsPath.getMovieFileNameParameters(movieFullPath)
self.info("dictFileNameParameters: {0}".format(dictFileNameParameters))
self.movieDirectory = dictFileNameParameters["directory"]
gridSquare = dictFileNameParameters["gridSquare"]
prefix = dictFileNameParameters["prefix"]
date = dictFileNameParameters["date"]
hour = dictFileNameParameters["hour"]
movieNumber = dictFileNameParameters["movieNumber"]
movieName = dictFileNameParameters["movieName"]
self.info("Import movies: movieName: {0}".format(movieName))
processDir = os.path.join(os.path.dirname(movieFullPath), "process", movieName)
if not os.path.exists(processDir):
os.makedirs(processDir, 0755)
self.movieDirectory = os.path.dirname(movieFullPath)
micrographSnapshotFullPath, micrographFullPath, xmlMetaDataFullPath, gridSquareSnapshotFullPath = \
UtilsPath.getMovieJpegMrcXml(movieFullPath)
startTime = time.time()
doContinue = True
while doContinue:
micrographSnapshotFullPath, micrographFullPath, xmlMetaDataFullPath, gridSquareSnapshotFullPath = \
UtilsPath.getMovieJpegMrcXml(movieFullPath)
if micrographSnapshotFullPath is None or micrographFullPath is None or xmlMetaDataFullPath is None or gridSquareSnapshotFullPath is None:
self.info("Import movies: waiting for meta-data files to appear on disk...")
timeNow = time.time()
deltaTime = timeNow -startTime
if deltaTime > 30:
self.info("Import movies: Timeout waiting for meta-data files to appear on disk!!!")
doContinue = False
else:
time.sleep(5)
else:
doContinue = False
self.info("Import movies: micrographSnapshotFullPath: {0}".format(micrographSnapshotFullPath))
micrographSnapshotPyarchPath = None
micrographPyarchPath = None
xmlMetaDataPyarchPath = None
gridSquareSnapshotPyarchPath = None
voltage = None
magnification = None
imagesCount = None
positionX = None
positionY = None
dosePerImage = None
if micrographFullPath is not None:
micrographSnapshotPyarchPath = UtilsPath.copyToPyarchPath(micrographSnapshotFullPath)
xmlMetaDataPyarchPath = UtilsPath.copyToPyarchPath(xmlMetaDataFullPath)
gridSquareSnapshotPyarchPath = UtilsPath.copyToPyarchPath(gridSquareSnapshotFullPath)
try:
dictMetaData = UtilsPath.getXmlMetaData(xmlMetaDataFullPath)
voltage = dictMetaData["accelerationVoltage"]
magnification = dictMetaData["nominalMagnification"]
imagesCount = dictMetaData["numberOffractions"]
positionX = dictMetaData["positionX"]
positionY = dictMetaData["positionY"]
dosePerImage = round( float(dictMetaData["dose"]) / 10.0**20 / float(imagesCount), 2)
except:
self.info("ERROR reading XML file {0}".format(xmlMetaDataFullPath))
traceback.print_exc()
sphericalAberration = prot.sphericalAberration.get()
amplitudeContrast = prot.amplitudeContrast.get()
samplingRate = prot.samplingRate.get()
doseInitial = prot.doseInitial.get()
dosePerFrame = prot.dosePerFrame.get()
try:
movieObject = self.client.service.addMovie(proposal=self.proposal,
proteinAcronym=self.proteinAcronym,
sampleAcronym=self.sampleAcronym,
movieDirectory=self.movieDirectory,
movieFullPath=movieFullPath,
movieNumber=movieNumber,
micrographFullPath=micrographPyarchPath,
micrographSnapshotFullPath=micrographSnapshotPyarchPath,
xmlMetaDataFullPath=xmlMetaDataPyarchPath,
voltage=voltage,
sphericalAberration=sphericalAberration,
amplitudeContrast=amplitudeContrast,
magnification=magnification,
scannedPixelSize=samplingRate,
imagesCount=imagesCount,
dosePerImage=dosePerImage,
positionX=positionX,
positionY=positionY,
beamlineName=self.beamlineName,
gridSquareSnapshotFullPath=gridSquareSnapshotPyarchPath,
)
except:
self.info("ERROR uploading movie {0}".format(movieFullPath))
traceback.print_exc()
movieObject = None
if movieObject is not None:
movieId = movieObject.movieId
else:
self.info("ERROR: movieObject is None!")
movieId = None
self.allParams[movieName] = {
"movieNumber": movieNumber,
"movieFullPath": movieFullPath,
"processDir": processDir,
"date": date,
"hour": hour,
"movieId": movieId,
"imagesCount": imagesCount,
"dosePerFrame": dosePerFrame,
"proposal": self.proposal,
"gridSquare": gridSquare,
"archived": False,
"positionX": positionX,
"positionY": positionY,
}
if not "EM_meta_data" in self.allParams:
self.allParams["EM_meta_data"] = {
"EM_directory": prot.filesPath.get(),
"EM_protein_acronym": self.proteinAcronym,
"EM_voltage": voltage,
"EM_magnification": magnification,
"EM_images_count": imagesCount,
"EM_position_x": positionX,
"EM_position_y": positionY,
"EM_dose_initial": doseInitial,
"EM_spherical_aberration": sphericalAberration,
"EM_dose_per_frame": dosePerFrame,
"EM_amplitude_contrast": amplitudeContrast,
"EM_sampling_rate": samplingRate,
}
if not gridSquare in self.allParams:
self.allParams[gridSquare] = {}
if not "listGalleryPath" in self.allParams[gridSquare]:
self.allParams[gridSquare]["listGalleryPath"] = [gridSquareSnapshotFullPath]
self.info("Import movies done, movieId = {0}".format(self.allParams[movieName]["movieId"]))
self.currentGridSquareLastMovieTime = time.time()
if self.currentGridSquare is None:
self.currentGridSquare = gridSquare
self.info("New grid square detected: {0}".format(self.currentGridSquare))
elif self.currentGridSquare != gridSquare:
# New grid square, archive previous grid square
self.archiveGridSquare(self.currentGridSquare)
self.currentGridSquare = None
# Check if old grid squares
self.archiveOldGridSquares(gridSquare)