def query_file_paths(self, run):
"""
Parameters
----------
run: str
Run number that was assigned by eT.
"""
if run is None:
return
db_name = 'Dev' if run.endswith('D') else 'Prod'
conn = Connection(self.user, db_name, prodServer=self.prodServer)
self.resp[run] = conn.getRunFilepaths(run=str(run))
class findCCD():
def __init__(self,
mirrorName='BNL-prod',
FType=None,
XtraOpts=None,
testName=None,
sensorId=None,
run=None,
outputFile=None,
site='slac.lca.archive',
Print=False,
db='Prod',
prodServer='Dev',
appSuffix='-jrb'):
if mirrorName == 'vendor':
chk_list = (sensorId)
else:
chk_list = (mirrorName, testName, sensorId, run)
if None in chk_list:
print 'Error: missing input to findCCD'
raise ValueError
self.mirrorName = mirrorName
self.FType = FType
self.XtraOpts = XtraOpts
self.testName = testName
self.sensorId = sensorId
self.outputFile = outputFile
self.site = site
self.Print = Print
self.run = run
self.db = db
self.prodServer = prodServer
if 'ITL' in self.sensorId:
self.CCDType = "ITL-CCD"
if 'E2V' in self.sensorId:
self.CCDType = "e2v-CCD"
pS = True
if self.prodServer == 'Dev':
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
def find(self):
sourceMap = {
'BNL-prod': 'BNL-prod/prod/',
'BNL-test': 'BNL-test/test/',
'vendorCopy-prod': 'SLAC-prod/prod/',
'vendorCopy-test': 'SLAC-test/test/',
'vendor': 'vendorData/',
'SAWG-BNL': 'BNL-SAWG/SAWG/'
}
folder = '/LSST/'
use_latest_activity = False
query = ''
site = self.site
use_query_eT = True
if (self.mirrorName == 'vendorCopy'):
site = "SLAC"
elif (self.mirrorName == 'vendor'):
folder = folder + \
sourceMap['vendor'] + \
self.CCDType.split('-')[0] + '/' + self.sensorId + '/' + self.db + '/'
use_latest_activity = True
site = "slac.lca.archive"
use_query_eT = False
elif (self.mirrorName == 'SAWG-BNL'):
folder = folder + 'mirror/' + \
sourceMap[self.mirrorName] + self.CCDType + \
'/' + self.sensorId + '/' + self.testName
use_latest_activity = True
use_query_eT = False
folderList = []
if use_query_eT is True:
kwds = {'run': self.run, 'stepName': self.testName}
filePaths = self.connect.getRunFilepaths(**kwds)
# get the unique directory paths
for test in filePaths:
for f in filePaths[test]:
dirpath = os.path.dirname(f['virtualPath']) + '/'
if dirpath not in folderList:
if self.sensorId in os.path.basename(f['virtualPath']):
folderList.append(dirpath)
else:
folderList.append(folder)
if self.XtraOpts is not None:
if query == '':
query = self.XtraOpts
else:
query += "&&" + self.XtraOpts
dsList = []
for folder in folderList:
datacatalog = DataCatalog(folder=folder,
experiment='LSST',
site=site,
use_newest_subfolder=use_latest_activity)
datasets = datacatalog.find_datasets(query)
if len(datasets) != 0:
dsList.append(datasets)
files = []
for ds in dsList:
pathsList = ds.full_paths()
for item in pathsList:
if (self.FType is None) or (self.FType is not None
and item.endswith(self.FType)):
if item not in files:
files.append(item)
if self.Print:
print "File paths for files at %s:" % site
for item in files:
print item
# Write file with list of found data files, if requested
if self.outputFile is not None and len(datasets) > 0:
print 'Writing output file ', self.outputFile, '...'
ofile = open(self.outputFile, 'w')
for line in files:
ofile.write(line + '\n')
pass
ofile.close()
elif self.outputFile is not None:
print "Result file requested, but no files found"
pass
return files