def __init__(self, mirrorName='BNL-prod', FType=None, XtraOpts=None,
testName=None, CCDType=None, sensorId=None, run=None, outputFile=None,
dataType=None, site='slac.lca.archive', Print=False, db='Prod',
prodServer='Dev', appSuffix='-jrb'):
if mirrorName == 'vendor': chk_list = (sensorId)
else: chk_list = (mirrorName, testName, sensorId, run)
if None in chk_list:
print 'Error: missing input to findCCD'
raise ValueError
self.mirrorName = mirrorName
self.FType = FType
self.XtraOpts = XtraOpts
self.testName = testName
self.sensorId = sensorId
self.outputFile = outputFile
self.dataType = dataType
self.site = site
self.Print = Print
self.run = run
self.db = db
self.prodServer = prodServer
if 'ITL' in self.sensorId: self.CCDType = "ITL-CCD"
if 'E2V' in self.sensorId: self.CCDType = "e2v-CCD"
pS = True
if self.prodServer == 'Dev': pS = False
self.connect = Connection(operator='richard', db=db, exp='LSST-CAMERA', prodServer=pS, appSuffix=appSuffix)
def __init__(self, prodServer="Prod"):
self.mirrorName = None
self.FType = None
self.XtraOpts = None
self.testName = None
self.site = "SLAC"
self.run = None
self.db = None
self.prodServer = prodServer
self.connect_Prod = Connection(operator='richard',
db="Prod",
exp='LSST-CAMERA',
prodServer=prodServer)
self.connect_Dev = Connection(operator='richard',
db="Dev",
exp='LSST-CAMERA',
prodServer=prodServer)
self.connections = {}
o = self.connections.setdefault("connect", {})
o["Prod"] = self.connect_Prod
o["Dev"] = self.connect_Dev
class exploreFocalPlane:
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
self.SR_htype = "LCA-11021_RTM"
self.CR_htype = "LCA-10692_CRTM"
self.raft_types = [self.SR_htype, self.CR_htype]
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def focalPlaneContents(self,
parentName="LCA-10134_Cryostat-0001",
htype='LCA-10134_Cryostat',
when=None,
run=None):
# when format: %Y-%m-%dT%H:%M:%S.%f eg: 2016-11-11T04:24:35.0
kwds = {
'experimentSN': parentName,
'htype': htype,
'noBatched': 'true'
}
if run is not None:
run_info = self.connect.getRunSummary(run=run)
kwds['timestamp'] = run_info['begin']
elif when is not None:
kwds['timestamp'] = when
response = self.connect.getHardwareHierarchy(**kwds)
raft_list = []
for ind in response:
raft = ind['child_experimentSN']
# ignore mechanical rafts
if 'MTR' in raft or ind[
"child_hardwareTypeName"] not in self.raft_types:
continue
slot = ind['slotName']
# kludge for first cryo definition with slot names as BayXY, not RXY
if 'Bay' in slot:
slot = slot.replace("Bay", "R")
if ind['parent_experimentSN'] == parentName:
raft_list.append([raft, slot])
return raft_list
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def query_file_paths(self, run):
"""
Parameters
----------
run: str
Run number that was assigned by eT.
"""
if run is None:
return
db_name = 'Dev' if run.endswith('D') else 'Prod'
conn = Connection(self.user, db_name, prodServer=self.prodServer)
self.resp[run] = conn.getRunFilepaths(run=str(run))
class exploreREB():
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def REBContents(self, REBName=None):
kwds = {
'experimentSN': REBName,
'htype': 'LCA-13574',
'noBatched': 'true'
}
response = self.connect.getHardwareHierarchy(**kwds)
# LCA-11721 is the ASPIC.
aspic_list = []
for row in response:
kid = row['child_experimentSN']
if '11721' in kid:
aspic_list.append((kid, row['slotName']))
return aspic_list
def ASPIC_parent(self, ASPIC_name=None, htype='LCA-11721'):
# now find ASPIC for a REB
kwds = {
'experimentSN': ASPIC_name,
'htype': htype,
'noBatched': 'true'
}
response = self.connect.getContainingHardware(**kwds)
for child in response:
if '13574' in child['parent_experimentSN']:
parentREB = child['parent_experimentSN']
break
return parentREB
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
self.SR_htype = "LCA-11021_RTM"
self.CR_htype = "LCA-10692_CRTM"
self.raft_types = [self.SR_htype, self.CR_htype]
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def __init__(self, db="Prod", eTserver="Prod"):
self.eTserver = eTserver
self.db = db
self.runData = None
if self.eTserver == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(
operator='richard',
db=self.db,
exp='LSST-CAMERA',
prodServer=pS)
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
self.SR_htype = "LCA-11021_RTM"
self.CR_htype = "LCA-10692_CRTM"
self.REB_htype = "LCA-13574"
self.WREB_htype = "LCA-13537"
self.GREB_htype = "LCA-13540"
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def __init__(self, run, user='******', prodServer=True):
"""
Parameters
----------
run: str
Run number. If it ends in 'D', the Dev database will be
queried.
user: str ['ccs']
User id to pass to the etravelerAPI.connection.Connnection
initializer.
prodServer: bool [True]
Flag to use the prod or dev eT server.
"""
super(ETResults, self).__init__()
db_name = 'Dev' if run.endswith('D') else 'Prod'
conn = Connection(user, db_name, prodServer=prodServer)
self.results = conn.getRunResults(run=run)
self._extract_schema_values()
def __init__(self, db='Prod', server='Prod', base_dir=None):
self.traveler_name = {}
self.test_type = "fe55_raft_analysis"
self.traveler_name['I&T-Raft'] = 'INT-SR-EOT-01'
self.traveler_name['BNL-Raft'] = 'SR-RTM-EOT-03'
self.base_dir = base_dir
self.db = db
self.server = server
if server == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS)
class get_steps_schema:
def __init__(self, db="Prod", eTserver="Prod"):
self.eTserver = eTserver
self.db = db
self.runData = None
if self.eTserver == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(
operator='richard',
db=self.db,
exp='LSST-CAMERA',
prodServer=pS)
def get(self, run=None):
if run is None:
print('Error: missing run number')
raise ValueError
self.runData = self.connect.getRunResults(run=run)
return self.runData
def get_test_info(self, runData=None):
if runData is None:
r = self.runData
else:
r = runData
self.test_dict = OrderedDict()
for steps in r["steps"]:
if "acq" in steps:
continue
schema_list = r["steps"][steps]
for schema in schema_list:
if "info" in schema or "versions" in schema:
continue
test_list = schema_list[schema][0]
for item in test_list:
if 'amp' in item or "slot" in item or "raft" in item or "sensor" in item \
or "schema" in item or "job" in item or \
"subset" in item or "file" in item or "host" in item or "wavelength" in item:
continue
self.test_dict[item] = [steps, schema]
return self.test_dict
class exploreRun():
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def hardware_sn(self, run):
run_info = self.connect.getRunSummary(run=run)
return run_info['experimentSN']
def slot_raft_map(self, run, parentName="LCA-10134_Cryostat-0001"):
run_info = self.connect.getRunSummary(run=run)
kwds = {
'experimentSN': run_info['experimentSN'],
'htype': 'LCA-10134_Cryostat',
'noBatched': 'true',
'timestamp': run_info['begin']
}
response = self.connect.getHardwareHierarchy(**kwds)
raft_list = []
for ind in response:
raft = ind['child_experimentSN']
# ignore mechanical rafts
if 'MTR' in raft or ind[
"child_hardwareTypeName"] != "LCA-11021_RTM":
continue
slot = ind['slotName']
# kludge for first cryo definition with slot names as BayXY, not RXY
if 'Bay' in slot:
slot = slot.replace("Bay", "R")
if ind['parent_experimentSN'] == parentName:
raft_list.append([raft, slot])
return response
def get_raft_slot_info():
"""
Return a dict, keyed by raft slot (e.g., 'R01') of tuples containing
raft LSST ID (e.g., 'LCA-11021_RTM-017') and the "good" TS8 run number
from
https://confluence.slac.stanford.edu/display/LSSTCAM/List+of+Good+Runs
"""
cryostat_id = 'LCA-10134_Cryostat-0001'
cryo_htype = 'LCA-10134_Cryostat'
conn = Connection('jchiang', 'Prod', prodServer=True)
resp = conn.getHardwareHierarchy(experimentSN=cryostat_id,
htype=cryo_htype)
raft_htype = 'LCA-11021_RTM'
raft_slot_info = dict()
for item in resp:
if item['child_hardwareTypeName'] == raft_htype:
raft_slot = f'R{item["slotName"][-2:]}'
lsst_id = item['child_experimentSN']
raft_slot_info[raft_slot] \
= RaftInfo(lsst_id, ts8_good_runs[lsst_id])
return raft_slot_info
def get_tests(self, htype = None, db = 'Prod', server = 'Prod', appSuffix = None):
# Find all instances of the SR-EOT-02 traveler.
if server == 'Prod': pS = True
else: pS = False
connect = Connection(operator='richard', db=db, exp='LSST-CAMERA', prodServer=pS)
hardwareLabels = ['SR_Grade:', 'SR_Contract:']
filePaths = connect.getFilepathsJH(htype='ITL-CCD', stepName='SR-CCD-EOT-05_Preflight-Check', travelerName='SR-CCD-EOT-05')#, hardwareLabels=hardwareLabels)
# Defects
# this step gives us bright columns and bright pixels,
bright_defects_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'bright_defects_offline', travelerName='SR-EOT-02')
# this step gives us dark columns and dark pixels
dark_defects_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'dark_defects_offline', travelerName='SR-EOT-02')
# this step gives us traps
traps_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'traps_offline', travelerName='SR-EOT-02')
# CTE
# this step gives us cti_low_serial, cti_high_serial, cti_low_parallel, cti_high_parallel
cte_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'cte_offline', travelerName='SR-EOT-02')
# Read read_noise
# this step gives us read_noise
readnoise_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'read_noise_offline', travelerName='SR-EOT-02')
nonlinearity_data = connect.getResultsJH(htype='ITL-CCD', stepName = 'flat_pairs_offline', travelerName='SR-EOT-02')
ccd_list = []
# Get a list of ccd's
for ccd in filePaths:
ccd_list.append(ccd)
expDict = {}
expDict['brightdefects'] = bright_defects_data
expDict['darkdefects'] = dark_defects_data
expDict['traps'] = traps_data
expDict['cte'] = cte_data
expDict['readnoise'] = readnoise_data
expDict['nonlinearity'] = nonlinearity_data
return ccd_list, expDict, filePaths
def __init__(self,
run=None,
step=None,
imgtype=None,
db='Prod',
prodServer='Prod',
appSuffix='',
site='slac.lca.archive'):
chk_list = (run, step)
if None in chk_list:
print('Error: missing input to raft_observation')
raise ValueError
if prodServer == 'Prod':
pS = True
else:
pS = False
self.db = db
self.prodServer = prodServer
self.appSuffix = appSuffix
self.site = site
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix,
debug=False)
self.run = run
self.step = step
self.imgtype = imgtype
def __init__(self, db='Prod', server='Prod', base_dir=None):
self.traveler_name = {}
self.test_type = ""
self.db = db
self.server = server
self.output_spec = ""
self.slot_names = [
"S00", "S01", "S02", "S10", "S11", "S12", "S20", "S21", "S22"
]
if server == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS)
self.requirements = {}
self.requirements['total_noise'] = 9. # C-SRFT-073
def __init__(self,
run=None,
raft=None,
step=None,
imgtype=None,
db='Prod',
prodServer='Prod',
appSuffix=''):
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
self.run = run
self.raft = raft
self.step = step
self.imgtype = imgtype
def execute(self):
source_list = []
args = self.args
uname = args.u
if uname is None:
uname = os.getenv('USER')
source_db = self.args.source_db
print('Connecting to database ', source_db)
self.conn = Connection(uname, source_db, prodServer=True, debug=False)
rsp = []
try:
print("Results from getHardwareHierarchy unfiltered:")
iDict = 0
rsp = self.conn.getHardwareHierarchy(experimentSN=self.lsst_id,
htype=args.htype,
noBatched='true')
source_list = self.formList(rsp, args.child_types)
print('#Components found is ', len(source_list))
for elt in source_list:
print('An element: ', elt['experimentSN'],
elt['manufacturerId'])
if elt['level'] > 0:
print(' ', elt['parent'], elt['slotName'])
except Exception as msg:
print('Query of source db failed with exception: ')
print(msg)
sys.exit(1)
### Stuff to be properly indented
# Make new connection to dest db
print('Connecting to database ', args.dest_db)
forReal = args.forreal
if args.forreal:
print('Continue for real? Y or N?')
try:
ans = raw_input('--> ')
except NameError:
ans = input('--> ')
if ans[0] != 'Y':
print('Switching to dry-run behavior')
forReal = False
self.conn = Connection(uname,
args.dest_db,
prodServer=True,
debug=False)
# SHOULD check that hardware types exist, but for now just hope for the best
# Query will fail
# we get the wrong answer.)
# Query dest db to find out which ones exist; create any that don't
# For now do this one by one. Could instead make 1 query per htype
for elt in source_list:
print('Working on component ', elt['experimentSN'])
#print('Value for key leaf is: ', elt['leaf'])
try:
rs = self.conn.getHardwareInstances(
htype=elt['htype'], experimentSN=elt['experimentSN'])
print('Component found in dest db. No register action')
except ETClientAPINoDataException:
print('Did not find corresponding component in dest db')
if (forReal):
self.register(elt)
else:
print('Dryrun. No attempt to create entry')
pass
except:
raise
if elt['leaf'] is True:
self.setReady(elt, forReal)
class CopyAssembly:
def __init__(self, args):
'''
Verify and store information passed in command-line argumens
'''
self.lsst_id = None
self.site = None
self.loc = None
self.conn = None
self.args = args
# Must of one of lsstId, number arguments
if args.lsst_id is not None:
self.lsst_id = args.lsst_id
elif args.cmpnt_num is not None:
self.lsst_id = ''.join([args.htype, '-', args.cmpnt_num])
else:
raise NameError('Neither lsstId nor number supplied')
# if --location option was used, check it has proper syntax
if (args.siteloc) is not None:
try:
self.site, self.loc = (args.siteloc).split(':')
except:
print('--location argument of improper form')
raise
def formList(self, response, good_types):
'''
response is the response from the getHierarchy query
good_types are the hardware types of interest
Returns a list of dicts, each containing information to be used
in registering one component
'''
outlist = []
didRoot = False
print('Good types are ')
for g in good_types:
print(g)
for r in response:
chtype = r['child_hardwareTypeName']
if chtype not in good_types:
print(chtype, ' not in list of hardware types')
continue
if r['level'] == '0':
if not didRoot:
dict0 = {}
dict0['level'] = 0
dict0['found'] = False
phtype = r['parent_hardwareTypeName']
pexpSN = r['parent_experimentSN']
dict0['htype'] = phtype
dict0['leaf'] = False
try:
rootInfos = self.conn.getHardwareInstances(
htype=phtype, experimentSN=pexpSN)
rootInfo = rootInfos[0]
for k in rootInfo:
dict0[k] = rootInfo[k]
outlist.append(dict0)
didRoot = True
print("Appended to outlist")
except:
raise
print('processing type ', chtype)
cdict = {}
cdict['parent'] = r['parent_experimentSN']
cdict['slotName'] = r['slotName']
cdict['relationshipName'] = r['relationshipTypeName']
cdict['level'] = int(r['level']) + 1
cdict['htype'] = chtype
cdict['leaf'] = True # default assumption
cexpSN = r['child_experimentSN']
try:
infos = self.conn.getHardwareInstances(htype=chtype,
experimentSN=cexpSN)
info = infos[0]
for k in info:
cdict[k] = info[k]
outlist.append(cdict)
print("Appended to outlist")
except:
raise
print('len of out list is ', len(outlist))
for cdict in outlist:
for r in response:
if r['parent_experimentSN'] == cdict['experimentSN'] and r[
'parent_hardwareTypeName'] == cdict['htype']:
cdict['leaf'] = False
return outlist
def register(self, elt):
if self.site is None:
site, location = elt['location'].split(':')
else:
site = self.site
location = self.loc
newId = self.conn.registerHardware(
htype=elt['htype'],
site=site,
location=location,
experimentSN=elt['experimentSN'],
manufacturerId=elt['manufacturerId'],
manufacturer=elt['manufacturer'],
model=elt['model'])
def setReady(self, elt, forReal):
if forReal:
self.conn.setHardwareStatus(experimentSN=elt['experimentSN'],
htype=elt['htype'],
status='READY',
reason='Set by copy_assembly')
print('Set status to READY for component ', elt['experimentSN'])
else:
print('Would have set status to READY for component ',
elt['experimentSN'])
def execute(self):
source_list = []
args = self.args
uname = args.u
if uname is None:
uname = os.getenv('USER')
source_db = self.args.source_db
print('Connecting to database ', source_db)
self.conn = Connection(uname, source_db, prodServer=True, debug=False)
rsp = []
try:
print("Results from getHardwareHierarchy unfiltered:")
iDict = 0
rsp = self.conn.getHardwareHierarchy(experimentSN=self.lsst_id,
htype=args.htype,
noBatched='true')
source_list = self.formList(rsp, args.child_types)
print('#Components found is ', len(source_list))
for elt in source_list:
print('An element: ', elt['experimentSN'],
elt['manufacturerId'])
if elt['level'] > 0:
print(' ', elt['parent'], elt['slotName'])
except Exception as msg:
print('Query of source db failed with exception: ')
print(msg)
sys.exit(1)
### Stuff to be properly indented
# Make new connection to dest db
print('Connecting to database ', args.dest_db)
forReal = args.forreal
if args.forreal:
print('Continue for real? Y or N?')
try:
ans = raw_input('--> ')
except NameError:
ans = input('--> ')
if ans[0] != 'Y':
print('Switching to dry-run behavior')
forReal = False
self.conn = Connection(uname,
args.dest_db,
prodServer=True,
debug=False)
# SHOULD check that hardware types exist, but for now just hope for the best
# Query will fail
# we get the wrong answer.)
# Query dest db to find out which ones exist; create any that don't
# For now do this one by one. Could instead make 1 query per htype
for elt in source_list:
print('Working on component ', elt['experimentSN'])
#print('Value for key leaf is: ', elt['leaf'])
try:
rs = self.conn.getHardwareInstances(
htype=elt['htype'], experimentSN=elt['experimentSN'])
print('Component found in dest db. No register action')
except ETClientAPINoDataException:
print('Did not find corresponding component in dest db')
if (forReal):
self.register(elt)
else:
print('Dryrun. No attempt to create entry')
pass
except:
raise
if elt['leaf'] is True:
self.setReady(elt, forReal)
import glob, os
from eTraveler.clientAPI.connection import Connection
data_dir="/sps/lsst/DataBE/ASPIC_production/"
archive_dir="/lsst-fr/data/camera/ASPIC_data"
#chipdirs = glob.glob(os.path.join(data_dir,"CHIP*"))
chipdirs = glob.glob(os.path.join(data_dir,"CHIP010*"))
logdir = os.path.join(data_dir,'Logs')
logfiles = glob.glob(os.path.join(logdir,"log-*.txt"))
logfiles = filter(lambda x: (not 'try' in x)and(not 'Test' in x)and(('PreScreening' in x)or('PostScreening' in x)or('ClearPending' in x)), logfiles)
myConn = Connection('cohen', 'Dev', prodServer=False)
#for chipdir in chipdirs[2:3]:
for chipdir in chipdirs:
unit_str = chipdir.split('CHIP')[1]
snid = unit_str
#Promex 01xx need to be turned into LCA-11721-ASPIC-P1xx
if snid[0:2]=='01':
snid='P1'+snid[2:]
snid='LCA-11721-ASPIC-'+snid
chiplogs = filter(lambda x: 'log-%s'%unit_str in x, logfiles)
#register hardware
try:
assembly_template_file = 'assembly_template_simple.yml'
relTaskPos = 5
print 'Number of steps before RelationshipTask = ', relTaskPos
if not os.path.exists(savedir):
os.mkdir(savedir)
print "yaml files will be saved in dir ", savedir
print "yaml files will be saved in dir ", savedir
print "Excel file = ", excel_file
print "Subsystem = ", subsystem
myConn = Connection(operator,
db='Dev',
exp='LSST-CAMERA',
prodServer=False,
localServer=False,
appSuffix='',
cnfPath='~/.ssh/.etapi.cnf',
debug=False)
class yamlAssembly():
"""Main class of this module.
Provide tools to create yaml files to assembly Filter Exchange System hardware.
Yaml files are created from a template and excel file provided by meca team.
"""
def __init__(self, record):
self._yaml = yaml.load(open(assembly_template_file))
self._name = (record[3].value).encode('utf-8')
self._yaml['Name'] = subsystem + "_Assembly_" + self._name
import glob, os, subprocess
from eTraveler.clientAPI.connection import Connection
data_dir="/sps/lsst/DataBE/ASPIC_production/"
archive_dir="/lsst-fr/data/camera/ASPIC_data"
#chipdirs = glob.glob(os.path.join(data_dir,"CHIP*"))
chipdirs = glob.glob(os.path.join(data_dir,"CHIP0226"))
logdir = os.path.join(data_dir,'Logs')
logfiles = glob.glob(os.path.join(logdir,"log-*.txt"))
logfiles = filter(lambda x: (not 'try' in x)and(not 'Test' in x)and(('PreScreening' in x)or('PostScreening' in x)or('ClearPending' in x)), logfiles)
myConn = Connection('cohen', 'Dev', prodServer=False)
#for chipdir in chipdirs[2:3]:
for chipdir in chipdirs:
unit_str = chipdir.split('CHIP')[1]
snid = unit_str
manufacturer = "QuikPak"
#Promex 01xx need to be turned into LCA-11721-ASPIC-P1xx
if snid[0:2]=='01':
snid='P1'+snid[2:]
manufacturer = "Promex"
snid='LCA-11721-ASPIC-'+snid
chiplogs = filter(lambda x: 'log-%s'%unit_str in x, logfiles)
class plotGoodRaftRuns():
def __init__(self, db='Prod', server='Prod', base_dir=None):
self.traveler_name = {}
self.test_type = "fe55_raft_analysis"
self.traveler_name['I&T-Raft'] = 'INT-SR-EOT-01'
self.traveler_name['BNL-Raft'] = 'SR-RTM-EOT-03'
self.base_dir = base_dir
self.db = db
self.server = server
if server == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS)
def find_runs(self, site_type=None, runs=None):
#data = self.connect.getResultsJH(htype="LCA-11021_RTM",
# stepName=self.test_type,
# travelerName=self.traveler_name)
if runs is not None:
runs = runs
return runs
def make_run_pages(self, site_type=None, runs=None):
run_list = self.find_runs(site_type=site_type, runs=runs)
raft_list = []
for run in run_list:
data = self.connect.getRunResults(run=run)
raft = data["experimentSN"]
raft_list.append(raft)
self.write_run_plot(run=run, site_type=site_type, raft=raft)
return run_list, raft_list
def write_run_plot(self, run=None, site_type=None, raft=None):
print('Operating on run ', run)
g = get_EO_analysis_results(db=self.db, server=self.server)
raft_list, data = g.get_tests(site_type=site_type,
test_type="gain",
run=run)
res = g.get_results(test_type="gain", data=data, device=raft)
raft_list_ptc, data_ptc = g.get_tests(site_type=site_type,
test_type="ptc_gain",
run=run)
res_ptc_gain = g.get_results(test_type='ptc_gain',
data=data_ptc,
device=raft)
raft_list_psf, data_psf = g.get_tests(site_type=site_type,
test_type="psf_sigma",
run=run)
res_psf = g.get_results(test_type='psf_sigma',
data=data_psf,
device=raft)
raft_list_rn, data_rn = g.get_tests(site_type=site_type,
test_type="read_noise",
run=run)
res_rn = g.get_results(test_type='read_noise',
data=data_rn,
device=raft)
raft_list_cti_low_serial, data_cls = g.get_tests(
site_type=site_type, test_type="cti_low_serial", run=run)
res_cls = g.get_results(test_type='cti_low_serial',
data=data_cls,
device=raft)
raft_list_cti_high_serial, data_chs = g.get_tests(
site_type=site_type, test_type="cti_high_serial", run=run)
res_chs = g.get_results(test_type='cti_high_serial',
data=data_chs,
device=raft)
raft_list_cti_low_parallel, data_clp = g.get_tests(
site_type=site_type, test_type="cti_low_parallel", run=run)
res_clp = g.get_results(test_type='cti_low_parallel',
data=data_clp,
device=raft)
raft_list_cti_high_parallel, data_chp = g.get_tests(
site_type=site_type, test_type="cti_high_parallel", run=run)
res_chp = g.get_results(test_type='cti_high_parallel',
data=data_chp,
device=raft)
raft_list_dark_pixels, data_dp = g.get_tests(site_type=site_type,
test_type="dark_pixels",
run=run)
res_dp = g.get_results(test_type='dark_pixels',
data=data_dp,
device=raft)
raft_list_dark_columns, data_dc = g.get_tests(site_type=site_type,
test_type="dark_columns",
run=run)
res_dc = g.get_results(test_type='dark_columns',
data=data_dc,
device=raft)
raft_list_bright_pixels, data_bp = g.get_tests(
site_type=site_type, test_type="bright_pixels", run=run)
res_bp = g.get_results(test_type='bright_pixels',
data=data_bp,
device=raft)
raft_list_bright_columns, data_bc = g.get_tests(
site_type=site_type, test_type="bright_columns", run=run)
res_bc = g.get_results(test_type='bright_columns',
data=data_bc,
device=raft)
raft_list_traps, data_tp = g.get_tests(site_type=site_type,
test_type="num_traps",
run=run)
res_tp = g.get_results(test_type='num_traps',
data=data_tp,
device=raft)
raft_list_qe, data_qe = g.get_tests(site_type=site_type,
test_type="QE",
run=run)
res_qe = g.get_results(test_type='QE', data=data_qe, device=raft)
raft_list_drkC, data_drkC = g.get_tests(site_type=site_type,
test_type="dark_current_95CL",
run=run)
res_drkC = g.get_results(test_type='dark_current_95CL',
data=data_drkC,
device=raft)
raft_list_fw, data_fw = g.get_tests(site_type=site_type,
test_type="full_well",
run=run)
res_fw = g.get_results(test_type='full_well',
data=data_fw,
device=raft)
raft_list_nonl, data_nonl = g.get_tests(site_type=site_type,
test_type="max_frac_dev",
run=run)
res_nonl = g.get_results(test_type='max_frac_dev',
data=data_nonl,
device=raft)
test_list = []
test_list_ptc = []
test_list_psf = []
test_list_rn = []
test_list_cls = []
test_list_chs = []
test_list_clp = []
test_list_chp = []
test_list_dp = []
test_list_dc = []
test_list_bp = []
test_list_bc = []
test_list_tp = []
test_list_drkC = []
test_list_fw = []
test_list_nonl = []
# treat qe specially since all filters are in the same list
test_list_qe_u = []
test_list_qe_g = []
test_list_qe_r = []
test_list_qe_i = []
test_list_qe_z = []
test_list_qe_y = []
for ccd in res:
test_list.extend(res[ccd])
test_list_ptc.extend(res_ptc_gain[ccd])
test_list_psf.extend(res_psf[ccd])
test_list_rn.extend(res_rn[ccd])
test_list_cls.extend(res_cls[ccd])
test_list_chs.extend(res_chs[ccd])
test_list_clp.extend(res_clp[ccd])
test_list_chp.extend(res_chp[ccd])
test_list_bp.extend(res_bp[ccd])
test_list_bc.extend(res_bc[ccd])
test_list_dp.extend(res_dp[ccd])
test_list_dc.extend(res_dc[ccd])
test_list_tp.extend(res_tp[ccd])
test_list_drkC.extend(res_drkC[ccd])
test_list_fw.extend(res_fw[ccd])
test_list_nonl.extend(res_nonl[ccd])
test_list_qe_u.append(res_qe[ccd][0])
test_list_qe_g.append(res_qe[ccd][1])
test_list_qe_r.append(res_qe[ccd][2])
test_list_qe_i.append(res_qe[ccd][3])
test_list_qe_z.append(res_qe[ccd][4])
test_list_qe_y.append(res_qe[ccd][5])
# NEW: create a column data source for the plots to share
source = ColumnDataSource(data=dict(x=range(0, len(test_list)),
gain=test_list,
ptc=test_list_ptc,
psf=test_list_psf,
rn=test_list_rn,
cls=test_list_cls,
chs=test_list_chs,
clp=test_list_clp,
chp=test_list_chp,
bp=test_list_bp,
bc=test_list_bc,
dp=test_list_dp,
dc=test_list_dc,
tp=test_list_tp,
drkC=test_list_drkC,
fw=test_list_fw,
nonl=test_list_nonl))
source_qe = ColumnDataSource(data=dict(x=range(0, len(test_list_qe_u)),
u=test_list_qe_u,
g=test_list_qe_g,
r=test_list_qe_r,
i=test_list_qe_i,
z=test_list_qe_z,
y=test_list_qe_y))
TOOLS = "pan,wheel_zoom,box_zoom,reset,save,box_select,lasso_select"
# create a new plot with a title and axis labels
p = figure(tools=TOOLS,
title="gains",
x_axis_label='amp',
y_axis_label='gain')
ptc = figure(tools=TOOLS,
title="ptc gains",
x_axis_label='amp',
y_axis_label='gain')
psf = figure(tools=TOOLS,
title="psf",
x_axis_label='amp',
y_axis_label='psf')
rn = figure(tools=TOOLS,
title="Read noise",
x_axis_label='amp',
y_axis_label='Noise')
cls = figure(tools=TOOLS,
title="CTI low serial",
x_axis_label='amp',
y_axis_label='CTI')
chs = figure(tools=TOOLS,
title="CTI high serial",
x_axis_label='amp',
y_axis_label='CTI')
clp = figure(tools=TOOLS,
title="CTI low parallel",
x_axis_label='amp',
y_axis_label='CTI')
chp = figure(tools=TOOLS,
title="CTI high parallel",
x_axis_label='amp',
y_axis_label='CTI')
bp = figure(tools=TOOLS,
title="Bright Pixels",
x_axis_label='amp',
y_axis_label='Bright Pixels')
bc = figure(tools=TOOLS,
title="Bright Columns",
x_axis_label='amp',
y_axis_label='Bright Columns')
dp = figure(tools=TOOLS,
title="Dark Pixels",
x_axis_label='amp',
y_axis_label='Dark Pixels')
dc = figure(tools=TOOLS,
title="Dark Columns",
x_axis_label='amp',
y_axis_label='Dark Columns')
tp = figure(tools=TOOLS,
title="Traps",
x_axis_label='amp',
y_axis_label='Traps')
drkC = figure(tools=TOOLS,
title="Dark Current",
x_axis_label='amp',
y_axis_label='Current')
fw = figure(tools=TOOLS,
title="Full Well",
x_axis_label='amp',
y_axis_label='Full Well')
nonl = figure(tools=TOOLS,
title="Non-linearity",
x_axis_label='amp',
y_axis_label='Max dev')
qe_u = figure(tools=TOOLS,
title="QE: u band",
x_axis_label='sensor',
y_axis_label='QE')
qe_g = figure(tools=TOOLS,
title="QE: g band",
x_axis_label='sensor',
y_axis_label='QE')
qe_r = figure(tools=TOOLS,
title="QE: r band",
x_axis_label='sensor',
y_axis_label='QE')
qe_i = figure(tools=TOOLS,
title="QE: i band",
x_axis_label='sensor',
y_axis_label='QE')
qe_z = figure(tools=TOOLS,
title="QE: z band",
x_axis_label='sensor',
y_axis_label='QE')
qe_y = figure(tools=TOOLS,
title="QE: y band",
x_axis_label='sensor',
y_axis_label='QE')
# add a line renderer with legend and line thickness
#sensor_lines = [sensor_start, sensor_end, sensor_third]
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
p.circle('x',
'gain',
source=source,
legend="Gain: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
p.add_layout(sensor_line)
my_label = Label(x=0, y=10, text='S00')
p.add_layout(my_label)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
ptc.circle('x',
'ptc',
source=source,
legend="ptc Gain: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
ptc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
psf.circle('x',
'psf',
source=source,
legend="PSF: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
psf.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
rn.circle('x',
'rn',
source=source,
legend="Read Noise: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
rn.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
cls.circle('x',
'cls',
source=source,
legend="CTI low serial: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
cls.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
chs.circle('x',
'chs',
source=source,
legend="CTI high serial: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
chs.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
clp.circle('x',
'clp',
source=source,
legend="CTI low parallel: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
clp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
chp.circle('x',
'chp',
source=source,
legend="CTI high parallel: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
chp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
bp.circle('x',
'bp',
source=source,
legend="Bright Pixels: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
bp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
bc.circle('x',
'bc',
source=source,
legend="Bright Columns: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
bc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
dp.circle('x',
'dp',
source=source,
legend="Dark Pixels: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
dp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
dc.circle('x',
'dc',
source=source,
legend="Dark Columns: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
dc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
tp.circle('x',
'tp',
source=source,
legend="Traps: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
tp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
drkC.circle('x',
'drkC',
source=source,
legend="Dark Current: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
drkC.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
fw.circle('x',
'fw',
source=source,
legend="Full Well: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
fw.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
nonl.circle('x',
'nonl',
source=source,
legend="Non-linearity: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
nonl.add_layout(sensor_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_u.circle('x',
'u',
source=source_qe,
legend="QE u band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_u.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_g.circle('x',
'g',
source=source_qe,
legend="QE g band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_g.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_r.circle('x',
'r',
source=source_qe,
legend="QE r band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_r.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_i.circle('x',
'i',
source=source_qe,
legend="QE i band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_i.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_z.circle('x',
'z',
source=source_qe,
legend="QE z band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_z.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_y.circle('x',
'y',
source=source_qe,
legend="QE y band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_y.add_layout(raft_line)
# NEW: put the subplots in a gridplot
grid = gridplot([[p, ptc], [psf, rn]])
grid_cti = gridplot([[cls, chs], [clp, chp]])
grid_def = gridplot([[bp, bc, None], [dp, dc, tp]])
grid_qe = gridplot([[qe_u, qe_g, qe_r], [qe_i, qe_z, qe_y]])
grid_misc = gridplot([[fw, drkC], [nonl, None]])
raft_text = "Plots for I&T Raft run " + str(run) + " " + raft
pre = PreText(text=raft_text, width=1000, height=100)
tab1 = Panel(child=grid, title="Gains, PSF, Noise")
tab2 = Panel(child=grid_cti, title="CTI")
tab3 = Panel(child=grid_def, title="Defects")
tab4 = Panel(child=grid_qe, title="QE 6-band")
tab5 = Panel(child=grid_misc, title="Dark C, Full well, Non-lin")
tabs = Tabs(tabs=[tab1, tab2, tab3, tab4, tab5])
l = layout([widgetbox(pre), tabs], sizing_mode='scale_both')
o_file = self.base_dir + raft + "-" + str(run) + ".html"
output_file(o_file)
save(l)
def write_table(self, run_list=None, raft_list=None, type_list=None):
data = dict(rafts=raft_list, runs=run_list, types=type_list)
dashboard = ColumnDataSource(data)
columns = [
TableColumn(field="rafts", title="Raft"),
TableColumn(field="types", title="Type"),
TableColumn(field="runs",
title="Run",
formatter=HTMLTemplateFormatter(template="<a href= \
'http://slac.stanford.edu/~richard/LSST/<%= rafts %>-<%= runs %>.html' \
><%= runs %></a>"))
]
data_table = DataTable(source=dashboard,
columns=columns,
width=400,
height=280)
return data_table
if not topdir:
raise RuntimeError, 'cannot determine top-level data directory'
# Save list of subdirectories. Their names are the manufacturer id's
# of the components to be processed
#manIds = list(os.listdir(topdir))
# Connect to eTraveler (prod) server with intent to use Dev database
limsurl = ""
try:
limsurl = os.environ['LCATR_LIMS_URL']
except:
pass
if ('/Prod' in limsurl):
print "Connecting to eTraveler Prod"
conn = Connection('homer', 'Prod', prodServer=True)
else:
print "Connecting to eTraveler Dev"
conn = Connection('homer', 'Dev', prodServer=True)
if not conn:
raise RuntimeError, 'unable to authenticate'
rsp = []
try:
# act = 23278
# act = siteUtils.extractJobId(topdir)
act = int(activity_id)
print "Activity ID = %d" % act
rsp = conn.getRunInfo(activityId=act)
class findCCD():
def __init__(self,
mirrorName='BNL-prod',
FType=None,
XtraOpts=None,
testName=None,
sensorId=None,
run=None,
outputFile=None,
site='slac.lca.archive',
Print=False,
db='Prod',
prodServer='Dev',
appSuffix='-jrb'):
if mirrorName == 'vendor':
chk_list = (sensorId)
else:
chk_list = (mirrorName, testName, sensorId, run)
if None in chk_list:
print 'Error: missing input to findCCD'
raise ValueError
self.mirrorName = mirrorName
self.FType = FType
self.XtraOpts = XtraOpts
self.testName = testName
self.sensorId = sensorId
self.outputFile = outputFile
self.site = site
self.Print = Print
self.run = run
self.db = db
self.prodServer = prodServer
if 'ITL' in self.sensorId:
self.CCDType = "ITL-CCD"
if 'E2V' in self.sensorId:
self.CCDType = "e2v-CCD"
pS = True
if self.prodServer == 'Dev':
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def find(self):
sourceMap = {
'BNL-prod': 'BNL-prod/prod/',
'BNL-test': 'BNL-test/test/',
'vendorCopy-prod': 'SLAC-prod/prod/',
'vendorCopy-test': 'SLAC-test/test/',
'vendor': 'vendorData/',
'SAWG-BNL': 'BNL-SAWG/SAWG/'
}
folder = '/LSST/'
use_latest_activity = False
query = ''
site = self.site
use_query_eT = True
if (self.mirrorName == 'vendorCopy'):
site = "SLAC"
elif (self.mirrorName == 'vendor'):
folder = folder + \
sourceMap['vendor'] + \
self.CCDType.split('-')[0] + '/' + self.sensorId + '/' + self.db + '/'
use_latest_activity = True
site = "slac.lca.archive"
use_query_eT = False
elif (self.mirrorName == 'SAWG-BNL'):
folder = folder + 'mirror/' + \
sourceMap[self.mirrorName] + self.CCDType + \
'/' + self.sensorId + '/' + self.testName
use_latest_activity = True
use_query_eT = False
folderList = []
if use_query_eT is True:
kwds = {'run': self.run, 'stepName': self.testName}
filePaths = self.connect.getRunFilepaths(**kwds)
# get the unique directory paths
for test in filePaths:
for f in filePaths[test]:
dirpath = os.path.dirname(f['virtualPath']) + '/'
if dirpath not in folderList:
if self.sensorId in os.path.basename(f['virtualPath']):
folderList.append(dirpath)
else:
folderList.append(folder)
if self.XtraOpts is not None:
if query == '':
query = self.XtraOpts
else:
query += "&&" + self.XtraOpts
dsList = []
for folder in folderList:
datacatalog = DataCatalog(folder=folder,
experiment='LSST',
site=site,
use_newest_subfolder=use_latest_activity)
datasets = datacatalog.find_datasets(query)
if len(datasets) != 0:
dsList.append(datasets)
files = []
for ds in dsList:
pathsList = ds.full_paths()
for item in pathsList:
if (self.FType is None) or (self.FType is not None
and item.endswith(self.FType)):
if item not in files:
files.append(item)
if self.Print:
print "File paths for files at %s:" % site
for item in files:
print item
# Write file with list of found data files, if requested
if self.outputFile is not None and len(datasets) > 0:
print 'Writing output file ', self.outputFile, '...'
ofile = open(self.outputFile, 'w')
for line in files:
ofile.write(line + '\n')
pass
ofile.close()
elif self.outputFile is not None:
print "Result file requested, but no files found"
pass
return files
from openpyxl import load_workbook
from eTraveler.clientAPI.connection import Connection
import yaml, os, sys
demoStr = "_Test_997"
savedir='test'
#excel_file='AutoChanger_BOM_simplifiee_20151009.xlsx'
excel_file='AutoChanger_BOM_demo_20160613.xlsx'
subsystem = 'CHGR'
if not os.path.exists(savedir):
os.mkdir(savedir)
print "yaml files will be saved in dir ", savedir
myConn = Connection('heather', 'Raw', prodServer=True)
class yamlAssembly():
def __init__(self, record):
self._yaml = yaml.load(open("assembly_template.yml"))
self._name = (record[3].value+demoStr).encode('utf-8')
self._yaml['HardwareGroup'] = self._name
self.registerHardwareType(self._name, record)
def registerHardwareType(self, name, rec):
newId = ''
try:
newId = myConn.defineHardwareType(name=name,
description=rec[7].value,
subsystem=subsystem,
batchedFlag=1,
sequenceWidth='0')
if not topdir:
raise RuntimeError, 'cannot determine top-level data directory'
# Save list of subdirectories. Their names are the manufacturer id's
# of the components to be processed
#manIds = list(os.listdir(topdir))
# Connect to eTraveler (prod) server with intent to use Dev database
limsurl = ""
try:
limsurl = os.environ['LCATR_LIMS_URL']
except :
pass
if ('/Dev' in limsurl) :
print "Connecting to eTraveler Dev"
conn = Connection('homer', 'Dev', prodServer=True)
else :
print "Connecting to eTraveler Prod"
conn = Connection('homer', 'Prod', prodServer=True)
if not conn:
raise RuntimeError, 'unable to authenticate'
rsp = []
try:
# act = 23278
# act = siteUtils.extractJobId(topdir)
act = int(activity_id)
print "Activity ID = %d" % act
rsp = conn.getRunInfo(activityId=act)
class exploreRaft():
def __init__(self, db='Prod', prodServer='Dev', appSuffix='-jrb'):
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def raftContents(self, raftName=None):
kwds = {
'experimentSN': raftName,
'htype': 'LCA-11021_RTM',
'noBatched': 'true'
}
response = self.connect.getHardwareHierarchy(**kwds)
# LCA-13574 is the REB.
reb_list = []
for row in response:
kid = row['child_experimentSN']
if '13574' in kid:
reb_list.append((kid, row['slotName']))
# match up the CCD to the REB via REB and slot numbering. The CCD in slot
# Sxy is on REBx. Note that the CCD is actually
# assembled onto the RSA.
ccd_list = []
for child in response:
# print child['parent_experimentSN'], child['relationshipTypeName'],
# child['child_experimentSN'], child['slotName']
kid = child['child_experimentSN']
if 'ITL' in kid.upper() or 'E2V' in kid.upper():
slotName = child['slotName']
rebNumber = slotName[1]
for reb in reb_list:
rebLoc = reb[1][3]
if rebLoc == rebNumber:
rebId = reb[0]
break
ccd_list.append((kid, slotName, rebId))
return ccd_list
def CCD_parent(self, CCD_name=None, htype='ITL-CCD'):
# now find raft for a CCD
kwds = {'experimentSN': CCD_name, 'htype': htype, 'noBatched': 'true'}
# connect = connection.Connection('richard', db='Dev', exp='LSST-CAMERA', prodServer=True)
response = self.connect.getContainingHardware(**kwds)
for child in response:
if 'RTM' in child['parent_experimentSN']:
parentRTM = child['parent_experimentSN']
break
return parentRTM
def REB_parent(self, REB_name=None):
# now find raft for a REB
kwds = {
'experimentSN': REB_name,
'htype': 'LCA-13574',
'noBatched': 'true'
} # need to fix REB htype!
response = self.connect.getContainingHardware(**kwds)
for child in response:
if 'RTM' in child['parent_experimentSN']:
parentRTM = child['parent_experimentSN']
break
return parentRTM
def REB_CCD(self, REB_name=None):
raft = self.REB_parent(REB_name)
ccd_list = self.raftContents(raft)
ccd_in_reb = []
for ccd in ccd_list:
if REB_name == ccd[2]:
ccd_in_reb.append(ccd[0])
return ccd_in_reb
from openpyxl import load_workbook
from eTraveler.clientAPI.connection import Connection
import yaml, os, sys
savedir = 'test'
excel_file = 'AutoChanger_BOM_simplifiee_20151009.xlsx'
subsystem = 'CHGR'
if not os.path.exists(savedir):
os.mkdir(savedir)
print "yaml files will be saved in dir ", savedir
myConn = Connection('jrb', 'Raw', prodServer=True)
class yamlAssembly():
def __init__(self, record):
self._yaml = yaml.load(open("assembly_template.yml"))
self._name = (record[3].value).encode('utf-8')
self._yaml['HardwareGroup'] = self._name
self.registerHardwareType(self._name, record)
def registerHardwareType(self, name, rec):
newId = ''
try:
newId = myConn.defineHardwareType(name=name,
description=rec[7].value,
subsystem=subsystem,
batchedFlag=1,
sequenceWidth='0')
print 'New hardware type defined. Returned id is ', newId
except Exception, msg:
writer.writerow(h)
entry +=1
sys.exit(0)
except ETClientAPINoDataException as msg:
print('No data exception: ',msg)
sys.exit(1)
except Exception as msg:
print('Operation failed with exception: ')
print(msg)
sys.exit(1)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description="""
Generate csv of NCR info for specified component
""")
parser.add_argument('--component','-c', help='experimentSN',
default='LCA-11021_RTM-004')
parser.add_argument('--htype', '-ht', help='hardware type',
default='LCA-11021_RTM')
myConn = Connection('jrb', 'Prod', debug=False, prodServer=True,
appSuffix='')
args = parser.parse_args()
#get_NCRs(myConn, component='LCA-11021_RTM-005')
print("Arguments:")
adict = vars(args)
print(adict)
get_NCRs(myConn, htype=adict['htype'], component=adict['component'])
class raft_observation():
def __init__(self,
run=None,
step=None,
imgtype=None,
db='Prod',
prodServer='Prod',
appSuffix='',
site='slac.lca.archive'):
chk_list = (run, step)
if None in chk_list:
print('Error: missing input to raft_observation')
raise ValueError
if prodServer == 'Prod':
pS = True
else:
pS = False
self.db = db
self.prodServer = prodServer
self.appSuffix = appSuffix
self.site = site
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix,
debug=False)
self.run = run
self.step = step
self.imgtype = imgtype
def find(self, run=None, step=None, imgtype=None):
if run is not None:
self.run = run
if step is not None:
self.step = step
if imgtype is not None:
self.imgtype = imgtype
rsp = self.connect.getRunSummary(run=self.run)
raft = rsp['experimentSN']
eR = exploreRaft(db=self.db,
prodServer=self.prodServer,
appSuffix=self.appSuffix)
ccd_list = eR.raftContents(raft)
obs_dict = {}
XtraOpts = ''
if self.imgtype is not None:
XtraOpts = 'IMGTYPE=="' + self.imgtype + '"'
for row in ccd_list:
ccd = str(row[0])
self.fCCD = findCCD(FType='fits',
testName=self.step,
sensorId=ccd,
run=str(self.run),
db=self.db,
prodServer=self.prodServer,
appSuffix=self.appSuffix,
site=self.site,
XtraOpts=XtraOpts)
files = self.fCCD.find()
for f in files:
# grab the timestamp from the end of the filename
name_split = os.path.splitext(
os.path.basename(f).split('_')[-1])[0]
if name_split not in obs_dict:
obs_dict[name_split] = []
obs_dict[name_split].append(f)
return obs_dict
class exploreRaft():
def __init__(self, db='Prod', prodServer='Prod', appSuffix=''):
self.SR_htype = "LCA-11021_RTM"
self.CR_htype = "LCA-10692_CRTM"
self.REB_htype = "LCA-13574"
self.WREB_htype = "LCA-13537"
self.GREB_htype = "LCA-13540"
if prodServer == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def raftContents(self, raftName=None, when=None, run=None):
# when format: %Y-%m-%dT%H:%M:%S.%f eg: 2016-11-11T04:24:35.0
raft_htype = self.SR_htype
if self.CR_htype in raftName:
raft_htype = self.CR_htype
kwds = {
'experimentSN': raftName,
'htype': raft_htype,
'noBatched': 'true'
}
if run is not None:
run_info = self.connect.getRunSummary(run=run)
kwds['timestamp'] = run_info['begin']
elif when is not None:
kwds['timestamp'] = when
response = self.connect.getHardwareHierarchy(**kwds)
# LCA-13574 is the REB.
reb_list = []
for row in response:
kid = row['child_experimentSN']
if self.REB_htype in kid or self.WREB_htype in kid or self.GREB_htype in kid:
reb_list.append((kid, row['slotName']))
# match up the CCD to the REB via REB and slot numbering. The CCD in slot
# Sxy is on REBx. Note that the CCD is actually
# assembled onto the RSA.
ccd_list = []
for child in response:
# print child['parent_experimentSN'], child['relationshipTypeName'],
# child['child_experimentSN'], child['slotName']
rebId = ""
kid = child['child_experimentSN']
if 'ITL' in kid.upper() or 'E2V' in kid.upper():
slotName = child['slotName']
if raft_htype == self.SR_htype:
slotNumber = 3
else:
slotNumber = 0
for reb in reb_list:
rebLoc = reb[1][slotNumber]
# two CCDs per GREB, but treated as 1
if self.GREB_htype in reb[0] and "3800C" in kid:
rebId = reb[0]
slotName = "guider"
break
# 2 CCDs per WREB
elif self.WREB_htype in reb[0] and "4400B" in kid:
rebId = reb[0]
break
elif self.REB_htype in reb[0]:
# CCD slotname first digit encodes the REB number
rebNumber = slotName[1]
if rebLoc == rebNumber:
rebId = reb[0]
break
else:
continue
ccd_list.append((kid, slotName, rebId))
return ccd_list
def raft_type(self, raft=None):
ccd_list = self.raftContents(raftName=raft)
if 'ITL' in ccd_list[0][0]:
type = 'ITL'
else:
type = 'e2v'
return type
def CCD_parent(self, CCD_name=None, htype='ITL-CCD', when=None, run=None):
# now find raft for a CCD
kwds = {'experimentSN': CCD_name, 'htype': htype, 'noBatched': 'true'}
if run is not None:
run_info = self.connect.getRunSummary(run=run)
kwds['timestamp'] = run_info['begin']
elif when is not None:
kwds['timestamp'] = when
# connect = connection.Connection('richard', db='Dev', exp='LSST-CAMERA', prodServer=True)
response = self.connect.getContainingHardware(**kwds)
parentRTM = ""
for child in response:
if 'RTM' in child['parent_experimentSN']:
parentRTM = child['parent_experimentSN']
break
return parentRTM
def REB_parent(self, REB_name=None, when=None, run=None):
# now find raft for a REB
htype = REB_name.rsplit("-", 1)[0]
kwds = {
'experimentSN': REB_name,
'htype': htype,
'noBatched': 'true'
} # need to fix REB htype!
if run is not None:
run_info = self.connect.getRunSummary(run=run)
kwds['timestamp'] = run_info['begin']
elif when is not None:
kwds['timestamp'] = when
response = self.connect.getContainingHardware(**kwds)
parentRTM = ""
for child in response:
if 'RTM' in child['parent_experimentSN']:
parentRTM = child['parent_experimentSN']
break
return parentRTM
def REB_CCD(self, REB_name=None, when=None, run=None):
raft = self.REB_parent(REB_name)
if run is not None:
run_info = self.connect.getRunSummary(run=run)
when = run_info['begin']
ccd_list = self.raftContents(raftName=raft, when=when)
ccd_in_reb = []
for ccd in ccd_list:
if REB_name == ccd[2]:
ccd_in_reb.append(ccd[0])
return ccd_in_reb