def test():
import datetime
# hole transformer
qhole = leginondata.AcquisitionImageData(
filename='07mar29b_00010gr_00012sq_v02_00011hl')
images = dbdk.query(qhole, readimages=False)
im = images[0]
hlscope = im['scope']
hlcamera = im['camera']
hltrans = getTransformer(hlscope['tem'], hlcamera['ccdcamera'],
hlscope['high tension'], hlscope['magnification'],
im.timestamp)
# square transformer
qsq = leginondata.AcquisitionImageData(filename='07mar29b_00010gr_00012sq')
images = dbdk.query(qsq, readimages=False)
im = images[0]
sqscope = im['scope']
sqcamera = im['camera']
sqtrans = getTransformer(sqscope['tem'], sqcamera['ccdcamera'],
sqscope['high tension'], sqscope['magnification'],
datetime.datetime(2007, 4, 10, 0, 0, 0))
## get stage position of target on hole image
bin = hlcamera['binning']
stage0 = hlscope['stage position']
pixvect = {'row': 0, 'col': 0}
stage = hltrans.transform(pixvect, stage0, bin)
print 'STAGE', stage
## get pixel position of stage position on square image
bin = sqcamera['binning']
stage0 = sqscope['stage position']
pix = sqtrans.itransform(stage, stage0, bin)
print 'PIX', pix
def getCorrectedImageData(self):
if not self.isOldNRAMMData():
return super(GatanK2Processing, self).getCorrectedImageData()
# local change
if self.image['camera']['ccdcamera']['name'] == 'GatanK2Super':
return leginondata.AcquisitionImageData().direct_query(1989908)
elif self.image['camera']['ccdcamera']['name'] == 'GatanK2Counting':
return leginondata.AcquisitionImageData().direct_query(1989725)
else:
return self.image
def getCsFromLeginonSessions():
sessions = leginondata.SessionData().query()
if len(sessions) == 0:
printError(
'No sessions found. New installation does not need to run this')
tems = []
realtems = []
for sessiondata in sessions:
images = leginondata.AcquisitionImageData(session=sessiondata).query(
results=1)
if not images:
continue
temdata = images[0]['scope']['tem']
if temdata is not None and temdata.dbid not in map(
(lambda x: x.dbid), tems):
tems.append(temdata)
# Only consider non-appion tem
if temdata['hostname'] != 'appion':
realtems.append(temdata)
print 'TEM %s.%s used in session %s has Cs value of %.3e m' % (
temdata['hostname'], temdata['name'], sessiondata['name'],
temdata['cs'])
print
if len(tems) == 0:
printWarning('No images acquired with any TEM. Do not worry about it')
if len(realtems) == 0:
printWarning(
'No images acquired with a real TEM. Do not worry about it')
def processImageListData(self, imagelistdata):
if 'images' not in imagelistdata or imagelistdata['images'] is None:
return
querydata = leginondata.AcquisitionImageData(list=imagelistdata)
## research, but don't read images until later
images = self.research(querydata, readimages=False)
targetlist = self.newTargetList(queue=self.settings['queue'])
if self.settings['allow append']:
self.logger.info("will append targets")
for imagedata in images:
self.findTargets(imagedata, targetlist)
self.makeTargetListEvent(targetlist)
if self.settings['queue']:
self.logger.info('Queue is on... not generating event')
pubevent = False
else:
self.logger.info('Queue is off... generating event')
pubevent = True
self.publish(targetlist,
database=True,
dbforce=True,
pubevent=pubevent)
if self.settings['wait for done'] and pubevent:
self.setStatus('waiting')
self.waitForTargetListDone()
self.setStatus('processing')
def publishImageData(self, imagedata, save):
acquisitionimagedata = leginondata.AcquisitionImageData(
initializer=imagedata)
if save:
griddata = leginondata.GridData()
if self.grid is not None:
gridinfo = self.gridmapping[self.grid]
griddata['grid ID'] = gridinfo['gridId']
emgriddata = leginondata.EMGridData(
name=gridinfo['label'], project=gridinfo['projectId'])
griddata['emgrid'] = emgriddata
griddata['insertion'] = self.insertion
acquisitionimagedata['grid'] = griddata
self.gridlabel = gridlabeler.getGridLabel(griddata)
elif self.emgrid is not None:
# New style that uses emgridata only for grid entry
griddata['emgrid'] = self.emgrid
griddata['insertion'] = self.insertion
acquisitionimagedata['grid'] = griddata
else:
self.gridlabel = ''
acquisitionimagedata['label'] = self.settings['image label']
self.setImageFilename(acquisitionimagedata)
acquisitionimagedata.attachPixelSize()
try:
self.publish(imagedata['scope'], database=True)
self.publish(imagedata['camera'], database=True)
self.publish(acquisitionimagedata, database=True)
except RuntimeError:
raise node.PublishError
## publish event even if no save
self.publish(acquisitionimagedata, pubevent=True)
def getSiblings(self): siblings = [] if self.imagedata: targetlistdata = self.imagedata['target']['list'] targetq = leginondata.AcquisitionImageTargetData(list=targetlistdata,type='acquisition') siblings = leginondata.AcquisitionImageData(target=targetq).query() return siblings
def alreadyAcquired(self, targetdata, presetname):
'''
determines if image already acquired using targetdata and presetname
'''
## if image exists with targetdata and presetdata, no acquire
## we expect target to be exact, however, presetdata may have
## changed so we only query on preset name
# seems to have trouple with using original targetdata as
# a query, so use a copy with only some of the fields
presetquery = leginondata.PresetData(name=presetname)
targetquery = leginondata.AcquisitionImageTargetData(initializer=targetdata)
## don't care if drift correction was done on target after image was
## acquired, so ignore version, delta row/col, parentimage, and fromtarget
targetquery['version'] = None
targetquery['delta row'] = None
targetquery['delta column'] = None
targetquery['image'] = None
targetquery['fromtarget'] = None
imagequery = leginondata.AcquisitionImageData(target=targetquery, preset=presetquery)
## other things to fill in
imagequery['scope'] = leginondata.ScopeEMData()
imagequery['camera'] = leginondata.CameraEMData()
imagequery['session'] = leginondata.SessionData()
datalist = self.research(datainstance=imagequery)
if datalist:
## no need to acquire again, but need to republish
self.reportStatus('output', 'Image was acquired previously, republishing')
imagedata = datalist[0]
self.publishDisplayWait(imagedata)
return True
else:
return False
def researchImages(self, **kwargs):
'''
Returns images in order of list and versions
'''
imagequery = leginondata.AcquisitionImageData(**kwargs)
images = imagequery.query()
# organize by list, version
organized = {}
for image in images:
if image['list'] is None:
imagelist = None
else:
imagelist = image['list'].dbid
if imagelist not in organized:
organized[imagelist] = {}
target = image['target']
version = image['version']
organized[imagelist][version] = image
final = []
tls = organized.keys()
tls.sort()
for imagelist in tls:
versions = organized[imagelist].keys()
versions.sort()
for version in versions:
final.append(organized[imagelist][version])
return final
def setFilename(self,inputname):
self.filename = inputname.split('.mrc')[0]
r = leginondata.AcquisitionImageData(filename=self.filename).query(results=1)
if not r:
print 'File %s does not exist in Leginon database' % inputname
sys.exit(1)
self.imagedata = r[0]
def printDriftStats(filenamepattern, apix):
filelist = glob.glob(filenamepattern + '_st_Log.txt')
if len(filelist) == 0:
return
allshifts = []
imgdata = leginondata.AcquisitionImageData(
filename=filelist[0].split('_st_Log.txt')[0]).query()[0]
fps = 1000.0 / imgdata['camera']['frame time']
for file in filelist:
positions = readPositionsFromAlignLog(file)
positions.insert(0, positions[0])
shifts = []
for i in range(len(positions) - 1):
shifts.append(
math.hypot(positions[i + 1][0] - positions[i][0],
positions[i + 1][1] - positions[i][1]))
allshifts.append(shifts)
import numpy
a = numpy.array(allshifts)
for d in range(a.shape[1]):
suba = a[:, d]
print "frame_%d %6.4f %6.4f" % (d, suba.mean() * apix * fps,
suba.std() * apix * fps)
def importInstrument(self):
print "Importing instrument...."
self.is_upload = False
# guess instrument from the last image
sinedon.setConfig('leginondata', db=self.source_dbname)
q = leginondata.AcquisitionImageData(session=self.getSourceSession())
last_image = self.research(q, True)
# we know there are images for the session.
tem = last_image['scope']['tem']
camera = last_image['camera']['ccdcamera']
if tem is None:
# old uploaded session such as 12jun29b
self.is_upload = True
return None, None, 200000
high_tension = last_image['scope']['high tension']
q = leginondata.InstrumentData()
source_temdata = q.direct_query(tem.dbid)
source_cameradata = q.direct_query(camera.dbid)
if 'appion' in source_temdata['name'].lower():
self.is_upload = True
sinedon.setConfig('leginondata', db=self.destination_dbname)
tem.insert(archive=True)
camera.insert(archive=True)
return source_cameradata, source_temdata, high_tension
def newImageVersion(self, oldimagedata, newimagedata, correct):
## store EMData to DB to prevent referencing errors
self.publish(newimagedata['scope'], database=True)
self.publish(newimagedata['camera'], database=True)
## convert CameraImageData to AcquisitionImageData
newimagedata = leginondata.AcquisitionImageData(initializer=newimagedata)
## then add stuff from old imagedata
newimagedata['preset'] = oldimagedata['preset']
newimagedata['label'] = oldimagedata['label']
newimagedata['target'] = oldimagedata['target']
newimagedata['list'] = oldimagedata['list']
newimagedata['emtarget'] = oldimagedata['emtarget']
newimagedata['version'] = oldimagedata['version'] + 1
dim = newimagedata['camera']['dimension']
newimagedata['pixels'] = dim['x'] * dim['y']
newimagedata['pixeltype'] = str(newimagedata['image'].dtype)
target = newimagedata['target']
if target is not None and 'grid' in target and target['grid'] is not None:
newimagedata['grid'] = target['grid']
## set the 'filename' value
if newimagedata['label'] == 'RCT':
rctacquisition.setImageFilename(newimagedata)
else:
acquisition.setImageFilename(newimagedata)
newimagedata.attachPixelSize()
self.logger.info('Publishing new version of image...')
self.publish(newimagedata, database=True, dbforce=True)
return newimagedata
def reacquire(self, targetdata, use_parent_mover=False):
'''
Reacquire parent image that created the targetdata but at current stage z.
'''
### get old image
oldimage = None
targetlist = targetdata['list']
# targetdata may be outdated due to other target adjustment
# This query gives the most recent target of the same specification
tquery = leginondata.AcquisitionImageTargetData(session=self.session, list=targetlist, number=targetdata['number'], type=targetdata['type'])
aquery = leginondata.AcquisitionImageData(target=tquery)
results = aquery.query(readimages=False, results=1)
if len(results) > 0:
oldimage = results[0]
if oldimage is None:
if targetlist:
self.logger.error('No image is acquired with target list %d' % targetlist.dbid)
return None
oldemtarget = oldimage['emtarget']
movetype = oldemtarget['movetype']
try:
emtarget = self.targetToEMTargetData(targetdata,movetype)
except InvalidStagePosition:
self.logger.error('Invalid new emtarget')
return None
oldpresetdata = oldimage['preset']
presetname = oldpresetdata['name']
channel = int(oldimage['correction channel']==0)
self._moveToLastFocusedStageZ()
self.imageMoveAndPreset(oldimage,emtarget,use_parent_mover)
targetdata = emtarget['target']
try:
imagedata = self.acquireCorrectedCameraImageData(channel)
except Exception, exc:
self.logger.error('Reacquire image failed: %s' % (exc,))
def importByImages(self):
source_session = self.getSourceSession()
sinedon.setConfig('leginondata', db=self.source_dbname)
q = leginondata.AcquisitionImageData(session=source_session)
images = q.query()
images.reverse()
print 'number of images in the session = %d' % len(images)
targetlist = {}
for image in images:
q = leginondata.ImageTargetListData(image=image)
targetlist[image.dbid] = self.research(q, True)
self.imageids.append(image.dbid)
sinedon.setConfig('leginondata', db=self.destination_dbname)
skipped = 0
for i, image in enumerate(images):
if not (i + 1) % 20:
print ""
else:
print ".",
if image['label'] in exclude_preset_list:
skipped += 1
continue
imageid = image.dbid
image.insert(archive=True)
if targetlist[imageid]:
targetlist[imageid].insert(archive=True)
print '\nimported %d images' % (len(images) - skipped)
def saveSessionDDinfoToDatabase(sessiondata):
qcam = leginondata.CameraEMData(session=sessiondata)
qcam['save frames'] = True
acqimages = leginondata.AcquisitionImageData(camera=qcam).query()
for imagedata in acqimages:
infopath = os.path.join(sessiondata['image path'],
imagedata['filename'] + '.frames', 'info.path')
saveImageDDinfoToDatabase(imagedata, infopath)
def getLastParentImage(self,newparentimage): ''' Get last version of the newparentimage. If there are target adjustment of the grandparent, the query may returns no results ''' results = leginondata.AcquisitionImageData(target=newparentimage['target'],version=newparentimage['version']-1).query(results=1) if not results: return None return results[0]
def start(self):
sessionname = self.params['sessionname']
runname = self.params['runname']
preset = self.params['preset']
sessionq = leginondata.SessionData(name=sessionname)
presetq = leginondata.PresetData(name=preset)
imgquery = leginondata.AcquisitionImageData()
imgquery['preset'] = presetq
imgquery['session'] = sessionq
imgtree = imgquery.query(readimages=False)
partq = appiondata.ApContourData()
sessiond = sessionq.query()
file = open('contourpickerTubeCircleData-' + sessionname + '.txt', 'w')
file.write('experiment_name ' + sessionname + '\n')
file.write('experiment_description ' + sessiond[0]['comment'] + '\n')
numimages = 0
for imgdata in imgtree:
partq['image'] = imgdata
partd = partq.query()
if len(partd) > 0:
numimages += 1
file.write('nimages ' + str(numimages) + '\n')
numparticles = 0.0
numcircles = 0.0
numtubes = 0.0
for imgdata in imgtree:
partq['image'] = imgdata
partd = partq.query()
maxversion = 0
for part in partd:
if int(part['version']
) > maxversion and part['runname'] == runname:
maxversion = int(part['version'])
for part in partd:
if int(part['version']
) == maxversion and part['runname'] == runname:
numparticles += 1
if part['particleType'] == 'Circle':
numcircles += 1
if part['particleType'] == 'Tube':
numtubes += 1
file.write('nparticles ' + str(numparticles) + '\n')
if numparticles == 0:
precenttubes = 0
file.write('%tubes ' + str(0.0))
else:
percenttubes = numtubes / numparticles
percent = percenttubes * 100
percent *= 100
percent = int(percent)
percent = percent / 100.0
file.write('%tubes ' + str(percent))
def lastTargetNumberOnMosaic(self, imagelist): qimagedata = leginondata.AcquisitionImageData() qimagedata['list'] = imagelist targetquery = leginondata.AcquisitionImageTargetData(image=qimagedata, status='new') targets = self.research(datainstance=targetquery, results=1) if targets: for target in targets: if target['number'] is not None: return target['number'] return 0
def getImageFromDB(self, filename):
# only want filename without path and extension
filename = os.path.split(filename)[1]
filename = '.'.join(filename.split('.')[:-1])
q = leginondata.AcquisitionImageData(filename=filename)
results = self.research(datainstance=q)
if not results:
return None
imagedata = results[0]
return imagedata
def getTiltSeries(self, targetdata, presetdata): ''' targetdata argument is target about to be acquired. Find the tilt series that this new image will belong to if it exists, otherwise create a new tilt series. ''' commontarget = targetdata['image']['target'] targetnumber = targetdata['number'] qimage1 = leginondata.AcquisitionImageData(target=commontarget) qtarget = leginondata.AcquisitionImageTargetData(image=qimage1, number=targetnumber) qpreset = leginondata.PresetData(name=presetdata['name'], session=presetdata['session']) qimage2 = leginondata.AcquisitionImageData(target=qtarget, preset=qpreset, session=presetdata['session']) images = self.research(qimage2, readimages=False) if images: tiltseries = images[0]['tilt series'] defocus = images[0]['scope']['defocus'] else: tiltseries = None defocus = None return tiltseries, defocus
def getImageData(filename_or_id):
if isinstance(filename_or_id, basestring):
q = leginondata.AcquisitionImageData(filename=filename_or_id)
images = q.query(readimages=False, results=1)
if images:
return images[0]
else:
raise RuntimeError('%s does not exist' % (filename_or_id, ))
else:
im = data.AcquisitionImageData.direct_query(filename_or_id,
readimages=False)
return im
def acquire(self, intensity):
imagedata = self.acquireCorrectedCameraImageData()
self.setImage(imagedata['image'])
preset = self.presetsclient.getCurrentPreset()
acqimdata = leginondata.AcquisitionImageData(initializer=imagedata,
session=self.session,
preset=preset)
acqimdata['filename'] = '%s_%s_%s_%04d' % (
self.session['name'], preset['name'], self.settings['label'],
int(intensity * 1000))
self.publish(acqimdata, database=True)
return acqimdata
def acquireAcquisitionImageData(self, range=None):
if range is None:
im = self.acquireImage()
else:
im = self.acquireWithinRange(*range)
im = leginondata.AcquisitionImageData(initializer=im)
im['session'] = self.session
mag = im['scope']['magnification']
magstring = '%06d' % (mag, )
label = self.settings['label']
im['filename'] = self.session['name'] + '-' + label + '-' + magstring
im['label'] = label
return im
def importImageDDinfo(self):
'''
Import DDInfoData based on imported image list.
This must be done after images are imported.
'''
print "Importing image ddinfo...."
#source_session = self.getSourceSession()
for imageid in self.imageids:
sinedon.setConfig('leginondata', db=self.source_dbname)
image = leginondata.AcquisitionImageData().direct_query(imageid)
q = leginondata.DDinfoValueData(camera=image['camera'])
results = self.research(q)
self.publish(results)
def processImageData(self, imagedata):
'''
Gets and publishes target information of specified image leginondata.
'''
if self.settings['ignore images']:
return
for target_name in self.targetnames:
self.setTargets([], target_name, block=True)
self.setTargetImageVector(imagedata)
# check if there is already a target list for this image
# or any other versions of this image (all from same target/preset)
# exclude sublists (like rejected target lists)
qtarget = imagedata['target']
try:
pname = imagedata['preset']['name']
qpreset = leginondata.PresetData(name=pname)
except:
qpreset = None
qimage = leginondata.AcquisitionImageData(target=qtarget,
preset=qpreset)
previouslists = self.researchTargetLists(image=qimage, sublist=False)
if previouslists:
# I hope you can only have one target list on an image, right?
targetlist = previouslists[0]
db = False
self.logger.info('Existing target list on this image...')
self.displayPreviousTargets(targetlist)
else:
# no previous list, so create one and fill it with targets
targetlist = self.newTargetList(image=imagedata,
queue=self.settings['queue'])
db = True
if self.settings['allow append'] or len(previouslists) == 0:
self.findTargets(imagedata, targetlist)
self.logger.debug('Publishing targetlist...')
## if queue is turned on, do not notify other nodes of each target list publish
if self.settings['queue']:
pubevent = False
else:
pubevent = True
self.publish(targetlist, database=db, pubevent=pubevent)
self.logger.debug('Published targetlist %s' % (targetlist.dbid, ))
if self.settings['wait for done'] and not self.settings['queue']:
self.makeTargetListEvent(targetlist)
self.setStatus('waiting')
self.waitForTargetListDone()
self.setStatus('processing')
def acquireCCD(self, presetdata, emtarget=None,channel=None):
targetdata = emtarget['target']
## set correction channel
## in the future, may want this based on preset or something
if channel is None:
channel = 0
else:
channel = channel
## acquire image
self.reportStatus('acquisition', 'acquiring image...')
self.startTimer('acquire getData')
correctimage = self.settings['correct image']
if correctimage:
imagedata = self.acquireCorrectedCameraImageData(channel=channel)
else:
imagedata = self.acquireCameraImageData()
self.reportStatus('acquisition', 'image acquired')
self.stopTimer('acquire getData')
if imagedata is None:
raise BadImageAcquirePause('failed acquire camera image')
if imagedata['image'] is None:
raise BadImageAcquirePause('Acquired array is None. Possible camera problem')
if self.settings['bad stats response'] != 'Continue':
self.recheck_counter = itertools.count()
# For ring collapse testing
#c = self.acq_counter.next()
#imagearray = imagedata['image'] - c*c
imagearray = imagedata['image']
self.evaluateStats(imagearray)
## convert float to uint16
if self.settings['save integer']:
imagedata['image'] = numpy.clip(imagedata['image'], 0, 2**16-1)
imagedata['image'] = numpy.asarray(imagedata['image'], numpy.uint16)
## convert CameraImageData to AcquisitionImageData
dim = imagedata['camera']['dimension']
pixels = dim['x'] * dim['y']
try:
pixeltype = str(imagedata['image'].dtype)
except:
self.logger.error('array not returned from camera')
return
imagedata = leginondata.AcquisitionImageData(initializer=imagedata, preset=presetdata, label=self.name, target=targetdata, list=self.imagelistdata, emtarget=emtarget, pixels=pixels, pixeltype=pixeltype)
imagedata['version'] = 0
## store EMData to DB to prevent referencing errors
self.publish(imagedata['scope'], database=True)
self.publish(imagedata['camera'], database=True)
return imagedata
def recentImageVersion(self, imagedata):
# find most recent version of this image
p = leginondata.PresetData(name=imagedata['preset']['name'])
q = leginondata.AcquisitionImageData()
q['session'] = imagedata['session']
q['target'] = imagedata['target']
q['list'] = imagedata['list']
q['preset'] = p
allimages = q.query()
version = 0
for im in allimages:
if im['version'] > version:
version = im['version']
return version
def getLastFocusedStageZ(self, targetdata):
if not targetdata or not targetdata['last_focused']:
return None
# FIX ME: This only works if images are taken with the last_focused
# ImageTargetListData. Not ideal. Can not rely on FocusResultData since
# manual z change is not recorded and it is not possible to distinguish
# true failuer from "fail" at eucentric focus as the contrast is lost.
qt = leginondata.AcquisitionImageTargetData(
list=targetdata['last_focused'])
images = leginondata.AcquisitionImageData(
target=qt, session=targetdata['session']).query(results=1)
if images:
z = images[0]['scope']['stage position']['z']
return z
def reacquire(self, targetdata, use_parent_mover=False):
'''
Reacquire parent image that created the targetdata but at current stage z.
'''
### get old image
oldimage = None
targetlist = targetdata['list']
# targetdata may be outdated due to other target adjustment
# This query gives the most recent target of the same specification
tquery = leginondata.AcquisitionImageTargetData(
session=self.session,
list=targetlist,
number=targetdata['number'],
type=targetdata['type'])
aquery = leginondata.AcquisitionImageData(target=tquery)
results = aquery.query(readimages=False, results=1)
if len(results) > 0:
oldimage = results[0]
if oldimage is None:
if targetlist:
self.logger.error('No image is acquired with target list %d' %
targetlist.dbid)
return None
oldemtarget = oldimage['emtarget']
movetype = oldemtarget['movetype']
try:
emtarget = self.targetToEMTargetData(targetdata, movetype)
except InvalidStagePosition:
self.logger.error('Invalid new emtarget')
return None
oldpresetdata = oldimage['preset']
presetname = oldpresetdata['name']
channel = int(oldimage['correction channel'] == 0)
self._moveToLastFocusedStageZ()
stagenow = self.instrument.tem.StagePosition
msg = 'after moveToLastFocusedStageZ z %.6f' % stagenow['z']
self.logger.debug(msg)
# z is not changed within imageMoveAndPreset
status = self.imageMoveAndPreset(oldimage, emtarget, use_parent_mover)
targetdata = emtarget['target']
# extra wait for falcon protector or normalization
self.logger.info('Wait for %.1f second before reaquire' %
self.settings['pause time'])
time.sleep(self.settings['pause time'])
try:
imagedata = self.acquireCorrectedCameraImageData(channel)
except Exception, exc:
self.logger.error('Reacquire image failed: %s' % (exc, ))
def copyImage(self, oldimage):
imagedata = leginondata.AcquisitionImageData()
imagedata.update(oldimage)
version = self.recentImageVersion(oldimage)
imagedata['version'] = version + 1
imagedata['filename'] = None
imagedata['image'] = oldimage['image']
## set the 'filename' value
if imagedata['label'] == 'RCT':
rctacquisition.setImageFilename(imagedata)
else:
acquisition.setImageFilename(imagedata)
self.logger.info('Publishing new copied image...')
self.publish(imagedata, database=True)
return imagedata
