def test(): import datetime # hole transformer qhole = leginondata.AcquisitionImageData( filename='07mar29b_00010gr_00012sq_v02_00011hl') images = dbdk.query(qhole, readimages=False) im = images[0] hlscope = im['scope'] hlcamera = im['camera'] hltrans = getTransformer(hlscope['tem'], hlcamera['ccdcamera'], hlscope['high tension'], hlscope['magnification'], im.timestamp) # square transformer qsq = leginondata.AcquisitionImageData(filename='07mar29b_00010gr_00012sq') images = dbdk.query(qsq, readimages=False) im = images[0] sqscope = im['scope'] sqcamera = im['camera'] sqtrans = getTransformer(sqscope['tem'], sqcamera['ccdcamera'], sqscope['high tension'], sqscope['magnification'], datetime.datetime(2007, 4, 10, 0, 0, 0)) ## get stage position of target on hole image bin = hlcamera['binning'] stage0 = hlscope['stage position'] pixvect = {'row': 0, 'col': 0} stage = hltrans.transform(pixvect, stage0, bin) print 'STAGE', stage ## get pixel position of stage position on square image bin = sqcamera['binning'] stage0 = sqscope['stage position'] pix = sqtrans.itransform(stage, stage0, bin) print 'PIX', pix
def getCorrectedImageData(self): if not self.isOldNRAMMData(): return super(GatanK2Processing, self).getCorrectedImageData() # local change if self.image['camera']['ccdcamera']['name'] == 'GatanK2Super': return leginondata.AcquisitionImageData().direct_query(1989908) elif self.image['camera']['ccdcamera']['name'] == 'GatanK2Counting': return leginondata.AcquisitionImageData().direct_query(1989725) else: return self.image
def getCsFromLeginonSessions(): sessions = leginondata.SessionData().query() if len(sessions) == 0: printError( 'No sessions found. New installation does not need to run this') tems = [] realtems = [] for sessiondata in sessions: images = leginondata.AcquisitionImageData(session=sessiondata).query( results=1) if not images: continue temdata = images[0]['scope']['tem'] if temdata is not None and temdata.dbid not in map( (lambda x: x.dbid), tems): tems.append(temdata) # Only consider non-appion tem if temdata['hostname'] != 'appion': realtems.append(temdata) print 'TEM %s.%s used in session %s has Cs value of %.3e m' % ( temdata['hostname'], temdata['name'], sessiondata['name'], temdata['cs']) print if len(tems) == 0: printWarning('No images acquired with any TEM. Do not worry about it') if len(realtems) == 0: printWarning( 'No images acquired with a real TEM. Do not worry about it')
def processImageListData(self, imagelistdata): if 'images' not in imagelistdata or imagelistdata['images'] is None: return querydata = leginondata.AcquisitionImageData(list=imagelistdata) ## research, but don't read images until later images = self.research(querydata, readimages=False) targetlist = self.newTargetList(queue=self.settings['queue']) if self.settings['allow append']: self.logger.info("will append targets") for imagedata in images: self.findTargets(imagedata, targetlist) self.makeTargetListEvent(targetlist) if self.settings['queue']: self.logger.info('Queue is on... not generating event') pubevent = False else: self.logger.info('Queue is off... generating event') pubevent = True self.publish(targetlist, database=True, dbforce=True, pubevent=pubevent) if self.settings['wait for done'] and pubevent: self.setStatus('waiting') self.waitForTargetListDone() self.setStatus('processing')
def publishImageData(self, imagedata, save): acquisitionimagedata = leginondata.AcquisitionImageData( initializer=imagedata) if save: griddata = leginondata.GridData() if self.grid is not None: gridinfo = self.gridmapping[self.grid] griddata['grid ID'] = gridinfo['gridId'] emgriddata = leginondata.EMGridData( name=gridinfo['label'], project=gridinfo['projectId']) griddata['emgrid'] = emgriddata griddata['insertion'] = self.insertion acquisitionimagedata['grid'] = griddata self.gridlabel = gridlabeler.getGridLabel(griddata) elif self.emgrid is not None: # New style that uses emgridata only for grid entry griddata['emgrid'] = self.emgrid griddata['insertion'] = self.insertion acquisitionimagedata['grid'] = griddata else: self.gridlabel = '' acquisitionimagedata['label'] = self.settings['image label'] self.setImageFilename(acquisitionimagedata) acquisitionimagedata.attachPixelSize() try: self.publish(imagedata['scope'], database=True) self.publish(imagedata['camera'], database=True) self.publish(acquisitionimagedata, database=True) except RuntimeError: raise node.PublishError ## publish event even if no save self.publish(acquisitionimagedata, pubevent=True)
def getSiblings(self): siblings = [] if self.imagedata: targetlistdata = self.imagedata['target']['list'] targetq = leginondata.AcquisitionImageTargetData(list=targetlistdata,type='acquisition') siblings = leginondata.AcquisitionImageData(target=targetq).query() return siblings
def alreadyAcquired(self, targetdata, presetname): ''' determines if image already acquired using targetdata and presetname ''' ## if image exists with targetdata and presetdata, no acquire ## we expect target to be exact, however, presetdata may have ## changed so we only query on preset name # seems to have trouple with using original targetdata as # a query, so use a copy with only some of the fields presetquery = leginondata.PresetData(name=presetname) targetquery = leginondata.AcquisitionImageTargetData(initializer=targetdata) ## don't care if drift correction was done on target after image was ## acquired, so ignore version, delta row/col, parentimage, and fromtarget targetquery['version'] = None targetquery['delta row'] = None targetquery['delta column'] = None targetquery['image'] = None targetquery['fromtarget'] = None imagequery = leginondata.AcquisitionImageData(target=targetquery, preset=presetquery) ## other things to fill in imagequery['scope'] = leginondata.ScopeEMData() imagequery['camera'] = leginondata.CameraEMData() imagequery['session'] = leginondata.SessionData() datalist = self.research(datainstance=imagequery) if datalist: ## no need to acquire again, but need to republish self.reportStatus('output', 'Image was acquired previously, republishing') imagedata = datalist[0] self.publishDisplayWait(imagedata) return True else: return False
def researchImages(self, **kwargs): ''' Returns images in order of list and versions ''' imagequery = leginondata.AcquisitionImageData(**kwargs) images = imagequery.query() # organize by list, version organized = {} for image in images: if image['list'] is None: imagelist = None else: imagelist = image['list'].dbid if imagelist not in organized: organized[imagelist] = {} target = image['target'] version = image['version'] organized[imagelist][version] = image final = [] tls = organized.keys() tls.sort() for imagelist in tls: versions = organized[imagelist].keys() versions.sort() for version in versions: final.append(organized[imagelist][version]) return final
def setFilename(self,inputname): self.filename = inputname.split('.mrc')[0] r = leginondata.AcquisitionImageData(filename=self.filename).query(results=1) if not r: print 'File %s does not exist in Leginon database' % inputname sys.exit(1) self.imagedata = r[0]
def printDriftStats(filenamepattern, apix): filelist = glob.glob(filenamepattern + '_st_Log.txt') if len(filelist) == 0: return allshifts = [] imgdata = leginondata.AcquisitionImageData( filename=filelist[0].split('_st_Log.txt')[0]).query()[0] fps = 1000.0 / imgdata['camera']['frame time'] for file in filelist: positions = readPositionsFromAlignLog(file) positions.insert(0, positions[0]) shifts = [] for i in range(len(positions) - 1): shifts.append( math.hypot(positions[i + 1][0] - positions[i][0], positions[i + 1][1] - positions[i][1])) allshifts.append(shifts) import numpy a = numpy.array(allshifts) for d in range(a.shape[1]): suba = a[:, d] print "frame_%d %6.4f %6.4f" % (d, suba.mean() * apix * fps, suba.std() * apix * fps)
def importInstrument(self): print "Importing instrument...." self.is_upload = False # guess instrument from the last image sinedon.setConfig('leginondata', db=self.source_dbname) q = leginondata.AcquisitionImageData(session=self.getSourceSession()) last_image = self.research(q, True) # we know there are images for the session. tem = last_image['scope']['tem'] camera = last_image['camera']['ccdcamera'] if tem is None: # old uploaded session such as 12jun29b self.is_upload = True return None, None, 200000 high_tension = last_image['scope']['high tension'] q = leginondata.InstrumentData() source_temdata = q.direct_query(tem.dbid) source_cameradata = q.direct_query(camera.dbid) if 'appion' in source_temdata['name'].lower(): self.is_upload = True sinedon.setConfig('leginondata', db=self.destination_dbname) tem.insert(archive=True) camera.insert(archive=True) return source_cameradata, source_temdata, high_tension
def newImageVersion(self, oldimagedata, newimagedata, correct): ## store EMData to DB to prevent referencing errors self.publish(newimagedata['scope'], database=True) self.publish(newimagedata['camera'], database=True) ## convert CameraImageData to AcquisitionImageData newimagedata = leginondata.AcquisitionImageData(initializer=newimagedata) ## then add stuff from old imagedata newimagedata['preset'] = oldimagedata['preset'] newimagedata['label'] = oldimagedata['label'] newimagedata['target'] = oldimagedata['target'] newimagedata['list'] = oldimagedata['list'] newimagedata['emtarget'] = oldimagedata['emtarget'] newimagedata['version'] = oldimagedata['version'] + 1 dim = newimagedata['camera']['dimension'] newimagedata['pixels'] = dim['x'] * dim['y'] newimagedata['pixeltype'] = str(newimagedata['image'].dtype) target = newimagedata['target'] if target is not None and 'grid' in target and target['grid'] is not None: newimagedata['grid'] = target['grid'] ## set the 'filename' value if newimagedata['label'] == 'RCT': rctacquisition.setImageFilename(newimagedata) else: acquisition.setImageFilename(newimagedata) newimagedata.attachPixelSize() self.logger.info('Publishing new version of image...') self.publish(newimagedata, database=True, dbforce=True) return newimagedata
def reacquire(self, targetdata, use_parent_mover=False): ''' Reacquire parent image that created the targetdata but at current stage z. ''' ### get old image oldimage = None targetlist = targetdata['list'] # targetdata may be outdated due to other target adjustment # This query gives the most recent target of the same specification tquery = leginondata.AcquisitionImageTargetData(session=self.session, list=targetlist, number=targetdata['number'], type=targetdata['type']) aquery = leginondata.AcquisitionImageData(target=tquery) results = aquery.query(readimages=False, results=1) if len(results) > 0: oldimage = results[0] if oldimage is None: if targetlist: self.logger.error('No image is acquired with target list %d' % targetlist.dbid) return None oldemtarget = oldimage['emtarget'] movetype = oldemtarget['movetype'] try: emtarget = self.targetToEMTargetData(targetdata,movetype) except InvalidStagePosition: self.logger.error('Invalid new emtarget') return None oldpresetdata = oldimage['preset'] presetname = oldpresetdata['name'] channel = int(oldimage['correction channel']==0) self._moveToLastFocusedStageZ() self.imageMoveAndPreset(oldimage,emtarget,use_parent_mover) targetdata = emtarget['target'] try: imagedata = self.acquireCorrectedCameraImageData(channel) except Exception, exc: self.logger.error('Reacquire image failed: %s' % (exc,))
def importByImages(self): source_session = self.getSourceSession() sinedon.setConfig('leginondata', db=self.source_dbname) q = leginondata.AcquisitionImageData(session=source_session) images = q.query() images.reverse() print 'number of images in the session = %d' % len(images) targetlist = {} for image in images: q = leginondata.ImageTargetListData(image=image) targetlist[image.dbid] = self.research(q, True) self.imageids.append(image.dbid) sinedon.setConfig('leginondata', db=self.destination_dbname) skipped = 0 for i, image in enumerate(images): if not (i + 1) % 20: print "" else: print ".", if image['label'] in exclude_preset_list: skipped += 1 continue imageid = image.dbid image.insert(archive=True) if targetlist[imageid]: targetlist[imageid].insert(archive=True) print '\nimported %d images' % (len(images) - skipped)
def saveSessionDDinfoToDatabase(sessiondata): qcam = leginondata.CameraEMData(session=sessiondata) qcam['save frames'] = True acqimages = leginondata.AcquisitionImageData(camera=qcam).query() for imagedata in acqimages: infopath = os.path.join(sessiondata['image path'], imagedata['filename'] + '.frames', 'info.path') saveImageDDinfoToDatabase(imagedata, infopath)
def getLastParentImage(self,newparentimage): ''' Get last version of the newparentimage. If there are target adjustment of the grandparent, the query may returns no results ''' results = leginondata.AcquisitionImageData(target=newparentimage['target'],version=newparentimage['version']-1).query(results=1) if not results: return None return results[0]
def start(self): sessionname = self.params['sessionname'] runname = self.params['runname'] preset = self.params['preset'] sessionq = leginondata.SessionData(name=sessionname) presetq = leginondata.PresetData(name=preset) imgquery = leginondata.AcquisitionImageData() imgquery['preset'] = presetq imgquery['session'] = sessionq imgtree = imgquery.query(readimages=False) partq = appiondata.ApContourData() sessiond = sessionq.query() file = open('contourpickerTubeCircleData-' + sessionname + '.txt', 'w') file.write('experiment_name ' + sessionname + '\n') file.write('experiment_description ' + sessiond[0]['comment'] + '\n') numimages = 0 for imgdata in imgtree: partq['image'] = imgdata partd = partq.query() if len(partd) > 0: numimages += 1 file.write('nimages ' + str(numimages) + '\n') numparticles = 0.0 numcircles = 0.0 numtubes = 0.0 for imgdata in imgtree: partq['image'] = imgdata partd = partq.query() maxversion = 0 for part in partd: if int(part['version'] ) > maxversion and part['runname'] == runname: maxversion = int(part['version']) for part in partd: if int(part['version'] ) == maxversion and part['runname'] == runname: numparticles += 1 if part['particleType'] == 'Circle': numcircles += 1 if part['particleType'] == 'Tube': numtubes += 1 file.write('nparticles ' + str(numparticles) + '\n') if numparticles == 0: precenttubes = 0 file.write('%tubes ' + str(0.0)) else: percenttubes = numtubes / numparticles percent = percenttubes * 100 percent *= 100 percent = int(percent) percent = percent / 100.0 file.write('%tubes ' + str(percent))
def lastTargetNumberOnMosaic(self, imagelist): qimagedata = leginondata.AcquisitionImageData() qimagedata['list'] = imagelist targetquery = leginondata.AcquisitionImageTargetData(image=qimagedata, status='new') targets = self.research(datainstance=targetquery, results=1) if targets: for target in targets: if target['number'] is not None: return target['number'] return 0
def getImageFromDB(self, filename): # only want filename without path and extension filename = os.path.split(filename)[1] filename = '.'.join(filename.split('.')[:-1]) q = leginondata.AcquisitionImageData(filename=filename) results = self.research(datainstance=q) if not results: return None imagedata = results[0] return imagedata
def getTiltSeries(self, targetdata, presetdata): ''' targetdata argument is target about to be acquired. Find the tilt series that this new image will belong to if it exists, otherwise create a new tilt series. ''' commontarget = targetdata['image']['target'] targetnumber = targetdata['number'] qimage1 = leginondata.AcquisitionImageData(target=commontarget) qtarget = leginondata.AcquisitionImageTargetData(image=qimage1, number=targetnumber) qpreset = leginondata.PresetData(name=presetdata['name'], session=presetdata['session']) qimage2 = leginondata.AcquisitionImageData(target=qtarget, preset=qpreset, session=presetdata['session']) images = self.research(qimage2, readimages=False) if images: tiltseries = images[0]['tilt series'] defocus = images[0]['scope']['defocus'] else: tiltseries = None defocus = None return tiltseries, defocus
def getImageData(filename_or_id): if isinstance(filename_or_id, basestring): q = leginondata.AcquisitionImageData(filename=filename_or_id) images = q.query(readimages=False, results=1) if images: return images[0] else: raise RuntimeError('%s does not exist' % (filename_or_id, )) else: im = data.AcquisitionImageData.direct_query(filename_or_id, readimages=False) return im
def acquire(self, intensity): imagedata = self.acquireCorrectedCameraImageData() self.setImage(imagedata['image']) preset = self.presetsclient.getCurrentPreset() acqimdata = leginondata.AcquisitionImageData(initializer=imagedata, session=self.session, preset=preset) acqimdata['filename'] = '%s_%s_%s_%04d' % ( self.session['name'], preset['name'], self.settings['label'], int(intensity * 1000)) self.publish(acqimdata, database=True) return acqimdata
def acquireAcquisitionImageData(self, range=None): if range is None: im = self.acquireImage() else: im = self.acquireWithinRange(*range) im = leginondata.AcquisitionImageData(initializer=im) im['session'] = self.session mag = im['scope']['magnification'] magstring = '%06d' % (mag, ) label = self.settings['label'] im['filename'] = self.session['name'] + '-' + label + '-' + magstring im['label'] = label return im
def importImageDDinfo(self): ''' Import DDInfoData based on imported image list. This must be done after images are imported. ''' print "Importing image ddinfo...." #source_session = self.getSourceSession() for imageid in self.imageids: sinedon.setConfig('leginondata', db=self.source_dbname) image = leginondata.AcquisitionImageData().direct_query(imageid) q = leginondata.DDinfoValueData(camera=image['camera']) results = self.research(q) self.publish(results)
def processImageData(self, imagedata): ''' Gets and publishes target information of specified image leginondata. ''' if self.settings['ignore images']: return for target_name in self.targetnames: self.setTargets([], target_name, block=True) self.setTargetImageVector(imagedata) # check if there is already a target list for this image # or any other versions of this image (all from same target/preset) # exclude sublists (like rejected target lists) qtarget = imagedata['target'] try: pname = imagedata['preset']['name'] qpreset = leginondata.PresetData(name=pname) except: qpreset = None qimage = leginondata.AcquisitionImageData(target=qtarget, preset=qpreset) previouslists = self.researchTargetLists(image=qimage, sublist=False) if previouslists: # I hope you can only have one target list on an image, right? targetlist = previouslists[0] db = False self.logger.info('Existing target list on this image...') self.displayPreviousTargets(targetlist) else: # no previous list, so create one and fill it with targets targetlist = self.newTargetList(image=imagedata, queue=self.settings['queue']) db = True if self.settings['allow append'] or len(previouslists) == 0: self.findTargets(imagedata, targetlist) self.logger.debug('Publishing targetlist...') ## if queue is turned on, do not notify other nodes of each target list publish if self.settings['queue']: pubevent = False else: pubevent = True self.publish(targetlist, database=db, pubevent=pubevent) self.logger.debug('Published targetlist %s' % (targetlist.dbid, )) if self.settings['wait for done'] and not self.settings['queue']: self.makeTargetListEvent(targetlist) self.setStatus('waiting') self.waitForTargetListDone() self.setStatus('processing')
def acquireCCD(self, presetdata, emtarget=None,channel=None): targetdata = emtarget['target'] ## set correction channel ## in the future, may want this based on preset or something if channel is None: channel = 0 else: channel = channel ## acquire image self.reportStatus('acquisition', 'acquiring image...') self.startTimer('acquire getData') correctimage = self.settings['correct image'] if correctimage: imagedata = self.acquireCorrectedCameraImageData(channel=channel) else: imagedata = self.acquireCameraImageData() self.reportStatus('acquisition', 'image acquired') self.stopTimer('acquire getData') if imagedata is None: raise BadImageAcquirePause('failed acquire camera image') if imagedata['image'] is None: raise BadImageAcquirePause('Acquired array is None. Possible camera problem') if self.settings['bad stats response'] != 'Continue': self.recheck_counter = itertools.count() # For ring collapse testing #c = self.acq_counter.next() #imagearray = imagedata['image'] - c*c imagearray = imagedata['image'] self.evaluateStats(imagearray) ## convert float to uint16 if self.settings['save integer']: imagedata['image'] = numpy.clip(imagedata['image'], 0, 2**16-1) imagedata['image'] = numpy.asarray(imagedata['image'], numpy.uint16) ## convert CameraImageData to AcquisitionImageData dim = imagedata['camera']['dimension'] pixels = dim['x'] * dim['y'] try: pixeltype = str(imagedata['image'].dtype) except: self.logger.error('array not returned from camera') return imagedata = leginondata.AcquisitionImageData(initializer=imagedata, preset=presetdata, label=self.name, target=targetdata, list=self.imagelistdata, emtarget=emtarget, pixels=pixels, pixeltype=pixeltype) imagedata['version'] = 0 ## store EMData to DB to prevent referencing errors self.publish(imagedata['scope'], database=True) self.publish(imagedata['camera'], database=True) return imagedata
def recentImageVersion(self, imagedata): # find most recent version of this image p = leginondata.PresetData(name=imagedata['preset']['name']) q = leginondata.AcquisitionImageData() q['session'] = imagedata['session'] q['target'] = imagedata['target'] q['list'] = imagedata['list'] q['preset'] = p allimages = q.query() version = 0 for im in allimages: if im['version'] > version: version = im['version'] return version
def getLastFocusedStageZ(self, targetdata): if not targetdata or not targetdata['last_focused']: return None # FIX ME: This only works if images are taken with the last_focused # ImageTargetListData. Not ideal. Can not rely on FocusResultData since # manual z change is not recorded and it is not possible to distinguish # true failuer from "fail" at eucentric focus as the contrast is lost. qt = leginondata.AcquisitionImageTargetData( list=targetdata['last_focused']) images = leginondata.AcquisitionImageData( target=qt, session=targetdata['session']).query(results=1) if images: z = images[0]['scope']['stage position']['z'] return z
def reacquire(self, targetdata, use_parent_mover=False): ''' Reacquire parent image that created the targetdata but at current stage z. ''' ### get old image oldimage = None targetlist = targetdata['list'] # targetdata may be outdated due to other target adjustment # This query gives the most recent target of the same specification tquery = leginondata.AcquisitionImageTargetData( session=self.session, list=targetlist, number=targetdata['number'], type=targetdata['type']) aquery = leginondata.AcquisitionImageData(target=tquery) results = aquery.query(readimages=False, results=1) if len(results) > 0: oldimage = results[0] if oldimage is None: if targetlist: self.logger.error('No image is acquired with target list %d' % targetlist.dbid) return None oldemtarget = oldimage['emtarget'] movetype = oldemtarget['movetype'] try: emtarget = self.targetToEMTargetData(targetdata, movetype) except InvalidStagePosition: self.logger.error('Invalid new emtarget') return None oldpresetdata = oldimage['preset'] presetname = oldpresetdata['name'] channel = int(oldimage['correction channel'] == 0) self._moveToLastFocusedStageZ() stagenow = self.instrument.tem.StagePosition msg = 'after moveToLastFocusedStageZ z %.6f' % stagenow['z'] self.logger.debug(msg) # z is not changed within imageMoveAndPreset status = self.imageMoveAndPreset(oldimage, emtarget, use_parent_mover) targetdata = emtarget['target'] # extra wait for falcon protector or normalization self.logger.info('Wait for %.1f second before reaquire' % self.settings['pause time']) time.sleep(self.settings['pause time']) try: imagedata = self.acquireCorrectedCameraImageData(channel) except Exception, exc: self.logger.error('Reacquire image failed: %s' % (exc, ))
def copyImage(self, oldimage): imagedata = leginondata.AcquisitionImageData() imagedata.update(oldimage) version = self.recentImageVersion(oldimage) imagedata['version'] = version + 1 imagedata['filename'] = None imagedata['image'] = oldimage['image'] ## set the 'filename' value if imagedata['label'] == 'RCT': rctacquisition.setImageFilename(imagedata) else: acquisition.setImageFilename(imagedata) self.logger.info('Publishing new copied image...') self.publish(imagedata, database=True) return imagedata