def test_getMovieJpegMrcXml(self):
movieFullPath = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448-3385.mrc"
jpeg, mrc, xml, gridSquareThumbNail = UtilsPath.getMovieJpegMrcXml(
movieFullPath)
jpegRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.jpg"
self.assertEqual(jpegRef, jpeg)
mrcRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.mrc"
self.assertEqual(mrcRef, mrc)
xmlRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/Data/FoilHole_23724105_Data_23724797_23724798_20171123_1448.xml"
self.assertEqual(xmlRef, xml)
gridSquareThumbNailRef = "/scisoft/pxsoft/data/cryoem/20171123/RAW_DATA/testsecretin-grid1/Images-Disc1/GridSquare_23722826/GridSquare_20171123_144119.jpg"
self.assertEqual(gridSquareThumbNailRef, gridSquareThumbNail)
if proposal is None:
print("WARNING! No data will be uploaded to ISPyB.")
db = 3
else:
print("Proposal: {0}".format(proposal))
if proposal == "mx415":
# Use valid data base
print("ISPyB valid data base used")
db = 1
else:
# Use productiond data base
print("ISPyB production data base used")
db = 0
jpeg, mrc, xml, gridSquareThumbNail = UtilsPath.getMovieJpegMrcXml(
firstMovieFullPath)
print("Metadata file: {0}".format(xml))
dictResults = UtilsPath.getXmlMetaData(xml)
doPhaseShiftEstimation = dictResults["phasePlateUsed"]
nominalMagnification = int(dictResults["nominalMagnification"])
superResolutionFactor = int(dictResults["superResolutionFactor"])
if samplingRate is None:
samplingRate = 1.1 / float(superResolutionFactor)
print("doPhaseShiftEstimation: {0}".format(doPhaseShiftEstimation))
print("nominalMagnification: {0}".format(nominalMagnification))
print("superResolutionFactor: {0}".format(superResolutionFactor))
print("samplingRate: {0}".format(samplingRate))
def uploadImportMovies(self, prot):
for movieFullPath in prot.getMatchFiles():
listMovieFullPath = []
for movieName in self.allParams:
if "movieFullPath" in self.allParams[movieName] and "movieId" in self.allParams[movieName]:
if self.allParams[movieName]["movieId"] is not None:
listMovieFullPath.append(self.allParams[movieName]["movieFullPath"])
# listMovieFullPath = [ self.allParams[movieName]["movieFullPath"] for movieName in self.allParams if "movieFullPath" in self.allParams[movieName]]
if movieFullPath in listMovieFullPath:
pass
#self.info("Movie already uploaded: {0}".format(movieFullPath))
else:
self.info("Import movies: movieFullPath: {0}".format(movieFullPath))
# Create "process" directory
dictFileNameParameters = UtilsPath.getMovieFileNameParameters(movieFullPath)
self.info("dictFileNameParameters: {0}".format(dictFileNameParameters))
self.movieDirectory = dictFileNameParameters["directory"]
gridSquare = dictFileNameParameters["gridSquare"]
prefix = dictFileNameParameters["prefix"]
date = dictFileNameParameters["date"]
hour = dictFileNameParameters["hour"]
movieNumber = dictFileNameParameters["movieNumber"]
movieName = dictFileNameParameters["movieName"]
self.info("Import movies: movieName: {0}".format(movieName))
processDir = os.path.join(os.path.dirname(movieFullPath), "process", movieName)
if not os.path.exists(processDir):
os.makedirs(processDir, 0755)
self.movieDirectory = os.path.dirname(movieFullPath)
micrographSnapshotFullPath, micrographFullPath, xmlMetaDataFullPath, gridSquareSnapshotFullPath = \
UtilsPath.getMovieJpegMrcXml(movieFullPath)
startTime = time.time()
doContinue = True
while doContinue:
micrographSnapshotFullPath, micrographFullPath, xmlMetaDataFullPath, gridSquareSnapshotFullPath = \
UtilsPath.getMovieJpegMrcXml(movieFullPath)
if micrographSnapshotFullPath is None or micrographFullPath is None or xmlMetaDataFullPath is None or gridSquareSnapshotFullPath is None:
self.info("Import movies: waiting for meta-data files to appear on disk...")
timeNow = time.time()
deltaTime = timeNow -startTime
if deltaTime > 30:
self.info("Import movies: Timeout waiting for meta-data files to appear on disk!!!")
doContinue = False
else:
time.sleep(5)
else:
doContinue = False
self.info("Import movies: micrographSnapshotFullPath: {0}".format(micrographSnapshotFullPath))
micrographSnapshotPyarchPath = None
micrographPyarchPath = None
xmlMetaDataPyarchPath = None
gridSquareSnapshotPyarchPath = None
voltage = None
magnification = None
imagesCount = None
positionX = None
positionY = None
dosePerImage = None
if micrographFullPath is not None:
micrographSnapshotPyarchPath = UtilsPath.copyToPyarchPath(micrographSnapshotFullPath)
xmlMetaDataPyarchPath = UtilsPath.copyToPyarchPath(xmlMetaDataFullPath)
gridSquareSnapshotPyarchPath = UtilsPath.copyToPyarchPath(gridSquareSnapshotFullPath)
try:
dictMetaData = UtilsPath.getXmlMetaData(xmlMetaDataFullPath)
voltage = dictMetaData["accelerationVoltage"]
magnification = dictMetaData["nominalMagnification"]
imagesCount = dictMetaData["numberOffractions"]
positionX = dictMetaData["positionX"]
positionY = dictMetaData["positionY"]
dosePerImage = round( float(dictMetaData["dose"]) / 10.0**20 / float(imagesCount), 2)
except:
self.info("ERROR reading XML file {0}".format(xmlMetaDataFullPath))
traceback.print_exc()
sphericalAberration = prot.sphericalAberration.get()
amplitudeContrast = prot.amplitudeContrast.get()
samplingRate = prot.samplingRate.get()
doseInitial = prot.doseInitial.get()
dosePerFrame = prot.dosePerFrame.get()
try:
movieObject = self.client.service.addMovie(proposal=self.proposal,
proteinAcronym=self.proteinAcronym,
sampleAcronym=self.sampleAcronym,
movieDirectory=self.movieDirectory,
movieFullPath=movieFullPath,
movieNumber=movieNumber,
micrographFullPath=micrographPyarchPath,
micrographSnapshotFullPath=micrographSnapshotPyarchPath,
xmlMetaDataFullPath=xmlMetaDataPyarchPath,
voltage=voltage,
sphericalAberration=sphericalAberration,
amplitudeContrast=amplitudeContrast,
magnification=magnification,
scannedPixelSize=samplingRate,
imagesCount=imagesCount,
dosePerImage=dosePerImage,
positionX=positionX,
positionY=positionY,
beamlineName=self.beamlineName,
gridSquareSnapshotFullPath=gridSquareSnapshotPyarchPath,
)
except:
self.info("ERROR uploading movie {0}".format(movieFullPath))
traceback.print_exc()
movieObject = None
if movieObject is not None:
movieId = movieObject.movieId
else:
self.info("ERROR: movieObject is None!")
movieId = None
self.allParams[movieName] = {
"movieNumber": movieNumber,
"movieFullPath": movieFullPath,
"processDir": processDir,
"date": date,
"hour": hour,
"movieId": movieId,
"imagesCount": imagesCount,
"dosePerFrame": dosePerFrame,
"proposal": self.proposal,
"gridSquare": gridSquare,
"archived": False,
"positionX": positionX,
"positionY": positionY,
}
if not "EM_meta_data" in self.allParams:
self.allParams["EM_meta_data"] = {
"EM_directory": prot.filesPath.get(),
"EM_protein_acronym": self.proteinAcronym,
"EM_voltage": voltage,
"EM_magnification": magnification,
"EM_images_count": imagesCount,
"EM_position_x": positionX,
"EM_position_y": positionY,
"EM_dose_initial": doseInitial,
"EM_spherical_aberration": sphericalAberration,
"EM_dose_per_frame": dosePerFrame,
"EM_amplitude_contrast": amplitudeContrast,
"EM_sampling_rate": samplingRate,
}
if not gridSquare in self.allParams:
self.allParams[gridSquare] = {}
if not "listGalleryPath" in self.allParams[gridSquare]:
self.allParams[gridSquare]["listGalleryPath"] = [gridSquareSnapshotFullPath]
self.info("Import movies done, movieId = {0}".format(self.allParams[movieName]["movieId"]))
self.currentGridSquareLastMovieTime = time.time()
if self.currentGridSquare is None:
self.currentGridSquare = gridSquare
self.info("New grid square detected: {0}".format(self.currentGridSquare))
elif self.currentGridSquare != gridSquare:
# New grid square, archive previous grid square
self.archiveGridSquare(self.currentGridSquare)
self.currentGridSquare = None
# Check if old grid squares
self.archiveOldGridSquares(gridSquare)