def __init__(self, name, camera, detrend, idDict, detrendIdDict, rerun, time=1200, validity=30,
**kwargs):
self.camera = camera
self.detrend = detrend
self.rerun = rerun
exeNames = {'bias': 'reduceBias.py',
'dark': 'reduceDark.py',
'flat': 'reduceFlat.py',
}
if not detrend in exeNames:
raise RuntimeError("Unrecognised detrend type: %s vs %s" % (detrend, exeNames.keys()))
cameraInfo = getCameraInfo(self.camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', exeNames[detrend]) + " "
command += " @WORKDIR@/" + cameraInfo.addDir + " "
command += " --id " + " ".join("=".join(map(str,kv)) for kv in idDict.items())
command += " --detrendId " + " ".join("=".join(map(str,kv)) for kv in detrendIdDict.items())
command += " --job=" + name
command += " --time=%f" % time
command += " @PBSARGS@"
command += " --rerun=" + rerun
# XXX The calibs used to end up in the root CALIB directory, but since Jim changed the rerun handling,
# they go in a new rerun CALIB directory. I'm more concerned to get things working than fix things up,
# so here's a workaround.
# create = "mkdir -p @WORKDIR@/" + cameraInfo.addDir + "/CALIB"
# link = "ln -s ../rerun/" + detrend + "/CALIB/" + detrend.upper()
# link += " @WORKDIR@/" + cameraInfo.addDir + "/CALIB/" + detrend.upper()
super(ReduceDetrendsTest, self).__init__(name, ["pbs", "calib", detrend, camera],
[command], **kwargs)
def __init__(self,
name,
camera,
visit,
ccd,
rerun=None,
addOptions="",
**kwargs):
self.camera = camera
self.visit = visit
self.ccd = ccd
self.dataId = {'visit': visit, 'ccd': ccd}
self.rerun = rerun
cameraInfo = getCameraInfo(camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'scProcessCcd.py') + \
" @WORKDIR@/" + cameraInfo.addDir + \
" --doraise " + addOptions + \
" --id visit=%d ccd=%d" % (self.visit, self.ccd)
if rerun is not None:
command += " --rerun=" + rerun
super(ProcessCcdTest, self).__init__(name, ["process", camera],
[command], **kwargs)
def __init__(self, name, camera, dbHost, dbName, dbUser, dbPass, source,
**kwargs):
cameraInfo = getCameraInfo(camera)
commandList = []
self.fileList = []
for dirpath, dirnames, filenames in os.walk(source):
for f in filenames:
if not f.endswith('.fits'):
continue
self.fileList.append(f)
command = os.path.join(os.environ['HSCDB_DIR'], 'bin',
cameraInfo.dbRaw)
command += " --execute --relocate=none --root=@WORKDIR@"
command += " --dbhost " + dbHost
command += " --dbname " + dbName
command += " --dbuser " + dbUser
command += " --dbpass " + dbPass
command += " " + os.path.join(dirpath, f)
commandList.append(command)
query = "SELECT * FROM %s" % cameraInfo.fileTable
numRowsValidate = lambda num: num == len(self.fileList)
super(DbRawTest, self).__init__(name, ["db", camera],
commandList,
dbHost=dbHost,
dbName=dbName,
dbUser=dbUser,
dbPass=dbPass,
query=query,
numRowsValidate=numRowsValidate,
**kwargs)
def validate(self, workDir=".", **kwargs):
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera,
rerun=self.rerun,
root=os.path.join(workDir,
cameraInfo.addDir))
self.validateCcd(butler, self.dataId)
def __init__(self, name, camera, source, dataIdList, datasetType="raw"):
self.camera = camera
self.dataIdList = dataIdList
self.datasetType = datasetType
self.fitsFiles = set()
inputs = []
for dirpath, dirnames, filenames in os.walk(source):
for f in filenames:
if f.endswith('.fits'):
inputs.append(os.path.join(dirpath, f))
self.fitsFiles.add(f)
cameraInfo = getCameraInfo(camera)
self.registryDir = cameraInfo.addDir
commandList = [[
os.path.join(os.environ['OBS_SUBARU_DIR'], "bin",
cameraInfo.refileScript), "--link", "--execute",
"--root=@WORKDIR@"
] + inputs,
[
os.path.join(os.environ['OBS_SUBARU_DIR'], "bin",
"genInputRegistry.py"), "--create",
"--root=@WORKDIR@/" + self.registryDir,
"--camera=" + cameraInfo.abbrev
]]
super(DataTest, self).__init__(name, ["setup", camera], commandList)
def __init__(self,
name,
camera,
dbType,
dbHost,
dbName,
dbUser,
dbPass,
dbPort=5432,
**kwargs):
command = os.path.join(os.environ['HSCDB_DIR'], 'bin',
'create_HscDb.py')
command += " --dbhost=" + dbHost
command += " --dbtype=" + dbType
command += " --dbname=" + dbName
command += " --dbuser="******" --dbpass="******" --drop"
self.dbPort = dbPort
cameraInfo = getCameraInfo(camera)
query = "SELECT * FROM %s" % cameraInfo.fileTable
numRowsValidate = lambda num: num == 0
super(DbCreateTest, self).__init__(name, ["db", camera], [command],
dbHost=dbHost,
dbName=dbName,
dbUser=dbUser,
dbPass=dbPass,
query=query,
numRowsValidate=numRowsValidate,
**kwargs)
def __init__(self,
name,
camera,
idDictList,
dataIdList,
time=1200,
rerun=None,
**kwargs):
self.camera = camera
self.idDictList = idDictList
self.dataIdList = dataIdList
self.rerun = rerun
cameraInfo = getCameraInfo(self.camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin',
'reduceFrames.py')
command += " @WORKDIR@/" + cameraInfo.addDir + " "
for idDict in idDictList:
command += "--id " + " ".join("=".join(map(str, kv))
for kv in idDict.items())
command += " --job=" + name
command += " --time=%f" % time
if rerun is not None:
command += " --rerun=" + rerun
command += " @PBSARGS@"
super(ReduceFramesTest,
self).__init__(name, ["pbs", "process", camera], [command],
**kwargs)
def validate(self, workDir=".", **kwargs):
self.validatePbs()
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera, rerun=self.rerun,
root=os.path.join(workDir, cameraInfo.addDir))
numCcds = hscCamera.getNumCcds(self.camera)
for dataId in self.dataIdList:
self.validateCcd(butler, dataId)
def validate(self, workDir=".", **kwargs):
target = self.getTargetDir(workDir)
registry = os.path.join(target, "calibRegistry.sqlite3")
self.assertTrue("Registry created", os.path.isfile(registry))
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera, root=os.path.join(workDir, cameraInfo.addDir))
for dataId in self.dataIdList:
self.validateDataset(butler, dataId, self.detrend)
return True
def __init__(self, name, camera, visit, rerun=None, **kwargs):
self.camera = camera
self.visit = visit
self.rerun = rerun
cameraInfo = getCameraInfo(camera)
command = os.path.join(os.environ['SOLVETANSIP_DIR'], 'bin', 'solvetansip.py')
command += " @WORKDIR@/" + cameraInfo.addDir + " --id visit=%d" % visit
if rerun is not None:
command += " --rerun=" + rerun
super(SolveTansipTest, self).__init__(name, ["data", "astrometry", camera], [command], **kwargs)
def validate(self, workDir=".", **kwargs):
self.validatePbs()
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera,
rerun=self.rerun,
root=os.path.join(workDir,
cameraInfo.addDir))
numCcds = hscCamera.getNumCcds(self.camera)
for dataId in self.dataIdList:
self.validateCcd(butler, dataId)
def validate(self, workDir=".", **kwargs):
target = self.getTargetDir(workDir)
registry = os.path.join(target, "calibRegistry.sqlite3")
self.assertTrue("Registry created", os.path.isfile(registry))
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera,
root=os.path.join(workDir,
cameraInfo.addDir))
for dataId in self.dataIdList:
self.validateDataset(butler, dataId, self.detrend)
return True
def __init__(self, name, camera, visit, rerun=None, **kwargs):
self.camera = camera
self.visit = visit
self.rerun = rerun
cameraInfo = getCameraInfo(camera)
command = os.path.join(os.environ['SOLVETANSIP_DIR'], 'bin',
'solvetansip.py')
command += " @WORKDIR@/" + cameraInfo.addDir + " --id visit=%d" % visit
if rerun is not None:
command += " --rerun=" + rerun
super(SolveTansipTest,
self).__init__(name, ["data", "astrometry", camera], [command],
**kwargs)
def __init__(self,
name,
camera,
detrend,
idDict,
detrendIdDict,
rerun,
time=1200,
validity=30,
**kwargs):
self.camera = camera
self.detrend = detrend
self.rerun = rerun
exeNames = {
'bias': 'reduceBias.py',
'dark': 'reduceDark.py',
'flat': 'reduceFlat.py',
}
if not detrend in exeNames:
raise RuntimeError("Unrecognised detrend type: %s vs %s" %
(detrend, exeNames.keys()))
cameraInfo = getCameraInfo(self.camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin',
exeNames[detrend]) + " "
command += " @WORKDIR@/" + cameraInfo.addDir + " "
command += " --id " + " ".join("=".join(map(str, kv))
for kv in idDict.items())
command += " --detrendId " + " ".join("=".join(map(str, kv))
for kv in detrendIdDict.items())
command += " --job=" + name
command += " --time=%f" % time
command += " @PBSARGS@"
command += " --rerun=" + rerun
# XXX The calibs used to end up in the root CALIB directory, but since Jim changed the rerun handling,
# they go in a new rerun CALIB directory. I'm more concerned to get things working than fix things up,
# so here's a workaround.
# create = "mkdir -p @WORKDIR@/" + cameraInfo.addDir + "/CALIB"
# link = "ln -s ../rerun/" + detrend + "/CALIB/" + detrend.upper()
# link += " @WORKDIR@/" + cameraInfo.addDir + "/CALIB/" + detrend.upper()
super(ReduceDetrendsTest,
self).__init__(name, ["pbs", "calib", detrend, camera],
[command], **kwargs)
def validate(self, workDir=".", **kwargs):
found = set()
for dirpath, dirnames, filenames in os.walk(os.path.join(workDir, self.registryDir)):
for f in filenames:
if f.endswith('.fits'):
found.add(f)
self.assertEqual("Number of FITS files", len(found), len(self.fitsFiles))
registry = os.path.join(workDir, self.registryDir, "registry.sqlite3")
self.assertTrue("Registry created", os.path.isfile(registry))
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera, root=os.path.join(workDir, cameraInfo.addDir))
for dataId in self.dataIdList:
self.validateDataset(butler, dataId, self.datasetType)
return True
def __init__(self, name, camera, visit, ccd, rerun=None, addOptions="", **kwargs):
self.camera = camera
self.visit = visit
self.ccd = ccd
self.dataId = {'visit': visit, 'ccd': ccd}
self.rerun = rerun
cameraInfo = getCameraInfo(camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'scProcessCcd.py') + \
" @WORKDIR@/" + cameraInfo.addDir + \
" --doraise " + addOptions + \
" --id visit=%d ccd=%d" % (self.visit, self.ccd)
if rerun is not None:
command += " --rerun=" + rerun
super(ProcessCcdTest, self).__init__(name, ["process", camera], [command], **kwargs)
def __init__(self, name, camera, idDictList, dataIdList, time=1200, rerun=None, **kwargs):
self.camera = camera
self.idDictList = idDictList
self.dataIdList = dataIdList
self.rerun = rerun
cameraInfo = getCameraInfo(self.camera)
command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'reduceFrames.py')
command += " @WORKDIR@/" + cameraInfo.addDir + " "
for idDict in idDictList:
command += "--id " + " ".join("=".join(map(str,kv)) for kv in idDict.items())
command += " --job=" + name
command += " --time=%f" % time
if rerun is not None:
command += " --rerun=" + rerun
command += " @PBSARGS@"
super(ReduceFramesTest, self).__init__(name, ["pbs", "process", camera], [command], **kwargs)
def __init__(self, name, camera, source, dataIdList, datasetType="raw"):
self.camera = camera
self.dataIdList = dataIdList
self.datasetType = datasetType
self.fitsFiles = set()
inputs = []
for dirpath, dirnames, filenames in os.walk(source):
for f in filenames:
if f.endswith('.fits'):
inputs.append(os.path.join(dirpath, f))
self.fitsFiles.add(f)
cameraInfo = getCameraInfo(camera)
self.registryDir = cameraInfo.addDir
commandList = [[os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", cameraInfo.refileScript),
"--link", "--execute", "--root=@WORKDIR@"] + inputs,
[os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genInputRegistry.py"),
"--create", "--root=@WORKDIR@/" + self.registryDir, "--camera=" + cameraInfo.abbrev]
]
super(DataTest, self).__init__(name, ["setup", camera], commandList)
def validate(self, workDir=".", **kwargs):
found = set()
for dirpath, dirnames, filenames in os.walk(
os.path.join(workDir, self.registryDir)):
for f in filenames:
if f.endswith('.fits'):
found.add(f)
self.assertEqual("Number of FITS files", len(found),
len(self.fitsFiles))
registry = os.path.join(workDir, self.registryDir, "registry.sqlite3")
self.assertTrue("Registry created", os.path.isfile(registry))
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera,
root=os.path.join(workDir,
cameraInfo.addDir))
for dataId in self.dataIdList:
self.validateDataset(butler, dataId, self.datasetType)
return True
def __init__(self,
name,
camera,
detrend,
dataIdList,
source=None,
validity=None):
self.camera = camera
self.detrend = detrend
self.dataIdList = dataIdList
self.source = source
commandList = []
cameraInfo = getCameraInfo(camera)
if source is not None:
# Get calibrations from the source
for dirpath, dirnames, filenames in os.walk(source):
targetDir = os.path.join("@WORKDIR@", cameraInfo.addDir,
"CALIB",
os.path.relpath(dirpath, source))
for f in filenames:
commandList.append([
"ln", "-s",
os.path.join(dirpath, f),
os.path.join(targetDir, f)
])
generate = [
os.path.join(os.environ['OBS_SUBARU_DIR'], "bin",
"genCalibRegistry.py"), "--create",
"--root=@WORKDIR@/" + os.path.join(cameraInfo.addDir, "CALIB"),
"--camera=" + camera
]
if validity is not None:
generate += ["--validity=%d" % validity]
commandList.append(generate)
super(CalibTest, self).__init__(name, ["calib", "setup", camera],
commandList)
def __init__(self, name, camera, detrend, dataIdList, source=None, validity=None):
self.camera = camera
self.detrend = detrend
self.dataIdList = dataIdList
self.source = source
commandList = []
cameraInfo = getCameraInfo(camera)
if source is not None:
# Get calibrations from the source
for dirpath, dirnames, filenames in os.walk(source):
targetDir = os.path.join("@WORKDIR@", cameraInfo.addDir, "CALIB",
os.path.relpath(dirpath, source))
for f in filenames:
commandList.append(["ln", "-s", os.path.join(dirpath, f), os.path.join(targetDir, f)])
generate = [os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genCalibRegistry.py"),
"--create", "--root=@WORKDIR@/" + os.path.join(cameraInfo.addDir, "CALIB"),
"--camera=" + camera]
if validity is not None:
generate += ["--validity=%d" % validity]
commandList.append(generate)
super(CalibTest, self).__init__(name, ["calib", "setup", camera], commandList)
def validate(self, workDir=".", **kwargs):
self.validatePbs()
cameraInfo = getCameraInfo(self.camera)
butler = hscCamera.getButler(self.camera, rerun=self.rerun,
root=os.path.join(workDir, cameraInfo.addDir))
def getTargetDir(self, workDir):
cameraInfo = getCameraInfo(self.camera)
return os.path.join(workDir, cameraInfo.addDir, "CALIB")
def getTargetDir(self, workDir):
cameraInfo = getCameraInfo(self.camera)
return os.path.join(workDir, cameraInfo.addDir, "CALIB")
