def createReferenceSession(self):
session_name = None
for suffix in 'abcdefghijklmnopqrstuvwxyz':
maybe_name = time.strftime('%y%b%d_ref_'+suffix).lower()
try:
leginon.session.makeReservation(maybe_name)
except leginon.session.ReservationFailed:
continue
else:
session_name = maybe_name
break
if session_name is None:
raise RuntimeError('no reference session name determined')
directory = leginon.leginonconfig.mapPath(leginon.leginonconfig.IMAGE_PATH)
imagedirectory = os.path.join(leginon.leginonconfig.unmapPath(directory), session_name, 'rawdata').replace('\\', '/')
initializer = {
'name': session_name,
'comment': 'reference images',
'user': self.session['user'],
'image path': imagedirectory,
'hidden' : True,
}
session = leginondata.SessionData(initializer=initializer)
session.insert()
refsession = leginondata.ReferenceSessionData(session=session)
refsession.insert()
return session
def researchSettings(self, classname, **kwargs):
'''
Find Legion Settings that may be used by the source session.
This could be those in the session or the last one in a previous
session by the user.
'''
sessiondata = self.getSession()
t = sessiondata.timestamp
sessiontime = self.makeTimeStringFromTimeStamp(t)
# search in the session
q = self.makequery(classname, kwargs)
q['session'] = self.session
r1 = q.query(results=1)
# search by user
found_by_user = False
if sessiondata['user'] and sessiondata.dbid:
sessionq = leginondata.SessionData(user=sessiondata['user'])
q = self.makequery(classname, kwargs)
q['session'] = sessionq
r2 = q.query(results=1,
timelimit='19000000000000\t%s' % (sessiontime, ))
if r2:
r1.append(r2[0])
found_by_user = True
# search by isdefault
if found_by_user == False:
q = self.makequery(classname, kwargs)
q['isdefault'] = True
r2 = q.query(results=1, timelimit='19000000000000\tsessiontime')
if r2:
r1.append(r2[0])
return r1
def updateAcquisitionSettings():
settingsclasses = []
for value in leginondata.__dict__.values():
try:
if issubclass(value, leginondata.AcquisitionSettingsData):
settingsclasses.append(value)
except:
pass
print 'Settings Classes:'
for s in settingsclasses:
print ' %s' % (s, )
qusers = leginondata.UserData()
users = qusers.query()
for user in users:
print 'USER', user['name']
qsession = leginondata.SessionData(user=user)
for cls in settingsclasses:
print '=========================='
print 'CLS', cls
temp_add_drift(cls)
qset = cls(session=qsession)
settings = qset.query()
recent = {}
for s in settings:
if s['name'] not in recent:
recent[s['name']] = s
drift2transform(s)
def alreadyAcquired(self, targetdata, presetname):
'''
determines if image already acquired using targetdata and presetname
'''
## if image exists with targetdata and presetdata, no acquire
## we expect target to be exact, however, presetdata may have
## changed so we only query on preset name
# seems to have trouple with using original targetdata as
# a query, so use a copy with only some of the fields
presetquery = leginondata.PresetData(name=presetname)
targetquery = leginondata.AcquisitionImageTargetData(initializer=targetdata)
## don't care if drift correction was done on target after image was
## acquired, so ignore version, delta row/col, parentimage, and fromtarget
targetquery['version'] = None
targetquery['delta row'] = None
targetquery['delta column'] = None
targetquery['image'] = None
targetquery['fromtarget'] = None
imagequery = leginondata.AcquisitionImageData(target=targetquery, preset=presetquery)
## other things to fill in
imagequery['scope'] = leginondata.ScopeEMData()
imagequery['camera'] = leginondata.CameraEMData()
imagequery['session'] = leginondata.SessionData()
datalist = self.research(datainstance=imagequery)
if datalist:
## no need to acquire again, but need to republish
self.reportStatus('output', 'Image was acquired previously, republishing')
imagedata = datalist[0]
self.publishDisplayWait(imagedata)
return True
else:
return False
def upgradeLeginonDB(self):
if not self.leginon_dbupgrade.tableExists('SessionData'):
return
while True:
while True:
sessionname = raw_input('Enter the name of the first session of which\n frames are saved with Gatan K2 Summit under DM 2.31\n (Press RETURN if none): ')
if not sessionname:
return
results = leginondata.SessionData(name=sessionname).query()
if not results:
print '\033[31mSession not found, Try again.\033[0m'
continue
self.sessiondata = results[0]
q = leginondata.CameraEMData(session=self.sessiondata)
q['save frames'] = True
results = q.query(results=1)
if not results:
print '\033[31mNo image with saved frame found in Session. Try again.\033[0m'
continue
camdata = results[0]
if not camdata['ccdcamera']['name'] in VALID_CAMERA_NAMES:
print '\033[31mThis is not a camera that needs changing. Try again.\033[0m'
continue
first_image = self. getFirstImageInSession()
if not first_image:
print '\033[31mThis session has no image saved. Try agin.\033[0m'
continue
k2cameras = self.getRelatedFrameCameras(camdata)
self.saveFrameOrientation(k2cameras,first_image)
break
answer = raw_input('Another K2 camera ? (Y/N)')
if answer.upper() =='N':
return
def onInit(self):
if 'sessionname' not in self.params.keys():
self.params['sessionname'] = leginondata.SessionData(
).direct_query(self.params['expid'])['name']
self.dd = apDDprocess.initializeDDFrameprocess(
self.params['sessionname'], self.params['wait'])
self.dd.setRunDir(self.params['rundir'])
# The gain/dark corrected ddstack is unlikely to be on local disk
if 'tempdir' not in self.params.keys():
self.dd.setTempDir()
else:
self.dd.setTempDir(self.params['tempdir'])
self.dd.setNewBinning(self.rundata['params']['bin'])
# Get the unfinished ddstack run parameters to apply them here
jobdata = apDatabase.getJobDataFromPathAndType(
self.rundata['path']['path'], 'makeddrawframestack')
self.ddstack_script_params = apScriptLog.getScriptParamValuesFromRunname(
self.rundata['runname'], self.rundata['path'], jobdata)
if 'no-keepstack' in self.ddstack_script_params.keys():
self.dd.setKeepStack(False)
self.dd.setDoseFDriftCorrOptions(self.ddstack_script_params)
# Give an unique lockname
self.setLockname('ddalign')
self.success_count = 0
def getCsFromLeginonSessions():
sessions = leginondata.SessionData().query()
if len(sessions) == 0:
printError(
'No sessions found. New installation does not need to run this')
tems = []
realtems = []
for sessiondata in sessions:
images = leginondata.AcquisitionImageData(session=sessiondata).query(
results=1)
if not images:
continue
temdata = images[0]['scope']['tem']
if temdata is not None and temdata.dbid not in map(
(lambda x: x.dbid), tems):
tems.append(temdata)
# Only consider non-appion tem
if temdata['hostname'] != 'appion':
realtems.append(temdata)
print 'TEM %s.%s used in session %s has Cs value of %.3e m' % (
temdata['hostname'], temdata['name'], sessiondata['name'],
temdata['cs'])
print
if len(tems) == 0:
printWarning('No images acquired with any TEM. Do not worry about it')
if len(realtems) == 0:
printWarning(
'No images acquired with a real TEM. Do not worry about it')
def setSession(self, sessionname):
q = leginondata.SessionData(name=sessionname)
r = q.query(results=1)
if r:
self.session = r[0]
else:
print 'No session named %s' % (sessionname, )
sys.exit(1)
def getSessionByName(self, name):
qsession = leginondata.SessionData(name=name)
sessions = self.research(qsession, results=1)
if sessions:
session = sessions[0]
else:
session = None
return session
def setDestinationSession(self, sessionname):
self.destination_session = None
sinedon.setConfig('leginondata', db=self.destination_dbname)
q = leginondata.SessionData(name=sessionname)
r = q.query()
if r:
session = r[0]
self.destination_session = session
self.reset()
def start(self):
sessionname = self.params['sessionname']
runname = self.params['runname']
preset = self.params['preset']
sessionq = leginondata.SessionData(name=sessionname)
presetq = leginondata.PresetData(name=preset)
imgquery = leginondata.AcquisitionImageData()
imgquery['preset'] = presetq
imgquery['session'] = sessionq
imgtree = imgquery.query(readimages=False)
partq = appiondata.ApContourData()
sessiond = sessionq.query()
file = open('contourpickerTubeCircleData-' + sessionname + '.txt', 'w')
file.write('experiment_name ' + sessionname + '\n')
file.write('experiment_description ' + sessiond[0]['comment'] + '\n')
numimages = 0
for imgdata in imgtree:
partq['image'] = imgdata
partd = partq.query()
if len(partd) > 0:
numimages += 1
file.write('nimages ' + str(numimages) + '\n')
numparticles = 0.0
numcircles = 0.0
numtubes = 0.0
for imgdata in imgtree:
partq['image'] = imgdata
partd = partq.query()
maxversion = 0
for part in partd:
if int(part['version']
) > maxversion and part['runname'] == runname:
maxversion = int(part['version'])
for part in partd:
if int(part['version']
) == maxversion and part['runname'] == runname:
numparticles += 1
if part['particleType'] == 'Circle':
numcircles += 1
if part['particleType'] == 'Tube':
numtubes += 1
file.write('nparticles ' + str(numparticles) + '\n')
if numparticles == 0:
precenttubes = 0
file.write('%tubes ' + str(0.0))
else:
percenttubes = numtubes / numparticles
percent = percenttubes * 100
percent *= 100
percent = int(percent)
percent = percent / 100.0
file.write('%tubes ' + str(percent))
def setNewSession(self,applicationname):
adminuser = self.getAdministratorUser()
timename = time.strftime('%Y%m%d%H%M%S', time.localtime())
sessionname = 'importsettings%s' % (timename)
q = leginondata.SessionData(name=sessionname,user=adminuser)
q['comment'] = 'import settings for %s' % (applicationname,)
q['hidden'] = True
q.insert()
# q becomes the data once inserted
self.session = q
def getDoseFromSessionPresetNames(sessionname, presetname):
''' returns dose, in electrons per Angstrom '''
sessiondata = leginondata.SessionData(name=sessionname).query()[0]
presetdata = leginondata.PresetData(
name=presetname, session=sessiondata).query(results=1)[0]
dose = presetdata['dose']
if not dose:
raise RunTimeError("dose not available for %s session and preset %s" %
(sessionname, presetname))
return dose / 1e20
def __init__(self, sessionname, new_path):
if not os.path.isdir(new_path):
raise ValueError('New image path must exists')
# find session(s) that is named as sessionname. It should be unique
results = leginondata.SessionData(name=sessionname).query()
if len(results) != 1:
raise ValueError('Found %d session data record for %s' %
(len(results), sessionname))
self.sessiondata = results[0]
self.new_path = new_path
def scanLeginonDB(self):
'''
Use sinedon data object to query
'''
# get all session data objects
allsessiondata = leginondata.SessionData().query()
for sessiondata in allsessiondata:
# when the selected field is not filled, it gives None, not empty string
if not sessiondata['image path']:
continue
# text search
if self.searchdrive in sessiondata['image path']:
self.leginon_sessions.append(sessiondata['name'])
def archiveAdminSettings():
'''
Archive all users and sessions in the project identified by id number
'''
adminid = archiveAdminUser()
if adminid is None:
print "No administration user found, Abort"
sys.exit(1)
u = leginondata.UserData().direct_query(adminid)
recent_session = leginondata.SessionData(user=u).query()[0]
app = SettingArchiver(recent_session['name'])
app.run()
app = None
def upgradeLeginonDB(self):
if not self.leginon_dbupgrade.columnExists('AcquisitionSettingsData', 'adjust for transform'):
return
users = leginondata.UserData().query()
for userdata in users:
sessionq = leginondata.SessionData(user=userdata)
asettingsq = leginondata.AcquisitionSettingsData(session=sessionq,name='Centered Square')
# only need to change the most recent one
asettings = asettingsq.query(results=1)
if asettings:
asettingsdata = asettings[0]
if asettingsdata['adjust for transform'] == 'no':
print asettingsdata.dbid,asettingsdata['session']['name'],asettingsdata['session']['user']['username']
self.leginon_dbupgrade.updateColumn('AcquisitionSettingsData', 'adjust for transform',"'one'",'`DEF_id`=%d' % asettingsdata.dbid,True)
def upgradeLeginonDB(self):
if not self.leginon_dbupgrade.columnExists('SessionData',
'frame path'):
return
sessions = leginondata.SessionData().query()
for sessiondata in sessions:
if sessiondata['frame path']:
continue
imagepath = sessiondata['image path']
if not imagepath:
continue
framepath = ddinfo.getRawFrameSessionPathFromImagePath(imagepath)
self.leginon_dbupgrade.updateColumn(
'SessionData', 'frame path', "'" + framepath + "'",
'`DEF_id`=%d' % sessiondata.dbid, True)
def getApplicationHistory(self):
initializer = {
'session': leginondata.SessionData(user=self.session['user']),
'application': leginondata.ApplicationData()
}
appdata = leginondata.LaunchedApplicationData(initializer=initializer)
appdatalist = self.research(appdata, timelimit='-90 0:0:0')
history = []
map = {}
for a in appdatalist:
name = a['application']['name']
if name not in history:
history.append(name)
map[name] = a['launchers']
return history, map
def importSession(self, comment=''):
print "Importing session...."
session = self.getSourceSession()
source_sessionid = session.dbid
# change session description if needed
if comment:
self.replaceItem(session, 'comment', comment)
sinedon.setConfig('leginondata', db=self.destination_dbname)
session.insert(force=False, archive=True)
q = leginondata.SessionData()
sessiondata = q.direct_query(session.dbid)
if not sessiondata:
self.escape("Session Not Inserted Successfully")
return
self.setDestinationSession(sessiondata)
def reseachDBSettings(self, settingsclass, inst_alias, user=None):
# load the requested user settings
if user is None:
user = self.session['user']
qsession = leginondata.SessionData(initializer={'user': user})
qdata = settingsclass(initializer={
'session': qsession,
'name': inst_alias
})
settings = self.research(qdata, results=1)
# if that failed, try to load default settings from DB
if not settings:
qdata = settingsclass(initializer={
'isdefault': True,
'name': self.name
})
settings = self.research(qdata, results=1)
return settings
def researchSettings(self, classname, **kwargs):
'''
Find Legion Settings that may be used by the source session.
This could be those in the session or the last one in a previous
session by the user.
'''
if classname not in dir(leginondata):
return []
source_session = self.getSourceSession()
t = source_session.timestamp
sessiontime = self.makeTimeStringFromTimeStamp(t)
# search in the session
q = self.makequery(classname, kwargs)
q['session'] = source_session
r1 = q.query()
# search by user
found_by_user = False
# query session again since for some reason it is mapped to destination
source_session = self.getSourceSession()
if source_session['user'] and source_session.dbid:
sessionq = leginondata.SessionData(user=source_session['user'])
q = self.makequery(classname, kwargs)
q['session'] = sessionq
r2 = q.query(results=1,
timelimit='19000000000000\t%s' % (sessiontime, ))
if r2:
r1.append(r2[0])
found_by_user = True
# search by isdefault
if found_by_user == False:
q = self.makequery(classname, kwargs)
q['isdefault'] = True
r2 = q.query(results=1, timelimit='19000000000000\tsessiontime')
if r2:
r1.append(r2[0])
r1.reverse()
return r1
def initializeSettings(self, user=None):
if not hasattr(self, 'settingsclass'):
return
# load the requested user settings
if user is None:
user = self.session['user']
qsession = leginondata.SessionData(initializer={'user': user})
qdata = self.settingsclass(initializer={
'session': qsession,
'name': self.name
})
settings = self.research(qdata, results=1)
# if that failed, try to load default settings from DB
if not settings:
qdata = self.settingsclass(initializer={
'isdefault': True,
'name': self.name
})
settings = self.research(qdata, results=1)
# if that failed, use hard coded defaults
if not settings:
self.settings = copy.deepcopy(self.defaultsettings)
else:
# get query result into usable form
settings = settings[0]
self.settings = settings.toDict(dereference=True)
del self.settings['session']
del self.settings['name']
# check if None in any fields
for key, value in self.settings.items():
if value is None:
if key in self.defaultsettings:
self.settings[key] = copy.deepcopy(
self.defaultsettings[key])
def getImages(session_name):
ses = leginondata.SessionData(name=session_name)
qim = leginondata.AcquisitionImageData(session=ses)
images = qim.query()
return images
#!/usr/bin/env python
import sys
if len(sys.argv) != 2:
print '%s %s' % (sys.argv[0], 'session_name')
sys.exit(1)
from leginon import leginondata
# look up session with name given by user
session_name = sys.argv[1]
sessionq = leginondata.SessionData(name=session_name)
sessions = sessionq.query()
if not sessions:
print 'No session named %s' % (session_name, )
print 'asdf'
sys.exit(0)
sessiondata = sessions[0]
# prompt user to confirm session before modifying database
prompt = """Found the following session:
Name: %s
User: %s %s
Comment: %s
Type "ok" if this is the correct session: """ % (
sessiondata['name'], sessiondata['user']['firstname'],
sessiondata['user']['lastname'], sessiondata['comment'])
response = raw_input(prompt)
if response != 'ok':
print 'no change to database'
del sortclasses[sortcls]
except KeyError:
pass
for nodes in sortclasses.values():
nodeorder += nodes
return nodeorder
def depth(parent, map):
l = [parent]
for child in map[parent]:
l += depth(child, map)
return l
if __name__ == '__main__':
import sys
import time
try:
session = sys.argv[1]
except IndexError:
session = time.strftime('%Y-%m-%d-%H-%M')
initializer = {'name': session}
m = Manager(('manager', ),
leginondata.SessionData(initializer=initializer))
m.start()
def setSourceSession(self, sessionname):
sinedon.setConfig('leginondata', db=self.source_dbname)
q = leginondata.SessionData(name=sessionname)
self.source_session = self.research(q, most_recent=True)
self.logger.info('%s done waiting' % (self.name,))
## if waiting for more than one, only returns status of final one
return status
if __name__ == '__main__':
import node
import sinedon
class TestNode(node.Node, TargetHandler):
pass
def __init__(self, id, session=None, managerlocation=None):
node.Node.__init__(self, id, session, managerlocation)
db = sinedon.getConnection('leginondata')
t = TestNode('testnode')
s = leginondata.SessionData(name='04may26a')
s = db.query(s, results=1)
s = s[0]
print 's', s
im = db.direct_query(leginondata.AcquisitionImageData, 49780)
print 'DONE DIRECT', im.dbid
tar = t.researchTargets(session=s, image=im)
print 'LEN', len(tar)
print 'DBID', tar[0].dbid
#print 'TAR0', tar[0]
print 'IM REF', tar[0].special_getitem('image', dereference=False)
def getSession(session_name):
ses = leginondata.SessionData(name=session_name)
session = ses.query(results=1)[0]
return session
datatypes = {'leginon': 'image', 'frames': 'frame'}
new_path_base = os.path.abspath(new_path_base)
datatype = None
for key in datatypes.keys():
if key in new_path_base:
datatype = datatypes[key]
print 'datatype set to %s' % (datatype)
if not datatype:
print 'Can not determine datatype'
sys.exit(1)
new_sessionnames = os.listdir(new_path_base)
for sessionname in new_sessionnames:
# find session(s) that is named as sessionname. It should be unique
results = leginondata.SessionData(name=sessionname).query()
if len(results) != 1:
print 'Found %d session data record for %s' % (len(results),
sessionname)
continue
new_path = os.path.join(new_path_base, sessionname, 'rawdata')
if not os.path.isdir(new_path):
print 'New path %s must exists' % (new_path)
continue
# DEF_id field in SessionData table is used to choose which row of data to update
sessionid = results[0].dbid
# update
query = "update `SessionData` set `%s path` = '%s' where `SessionData`.`DEF_id`=%d" % (
