def __init__(self, name, camera, detrend, idDict, detrendIdDict, rerun, time=1200, validity=30, **kwargs): self.camera = camera self.detrend = detrend self.rerun = rerun exeNames = {'bias': 'reduceBias.py', 'dark': 'reduceDark.py', 'flat': 'reduceFlat.py', } if not detrend in exeNames: raise RuntimeError("Unrecognised detrend type: %s vs %s" % (detrend, exeNames.keys())) cameraInfo = getCameraInfo(self.camera) command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', exeNames[detrend]) + " " command += " @WORKDIR@/" + cameraInfo.addDir + " " command += " --id " + " ".join("=".join(map(str,kv)) for kv in idDict.items()) command += " --detrendId " + " ".join("=".join(map(str,kv)) for kv in detrendIdDict.items()) command += " --job=" + name command += " --time=%f" % time command += " @PBSARGS@" command += " --rerun=" + rerun # XXX The calibs used to end up in the root CALIB directory, but since Jim changed the rerun handling, # they go in a new rerun CALIB directory. I'm more concerned to get things working than fix things up, # so here's a workaround. # create = "mkdir -p @WORKDIR@/" + cameraInfo.addDir + "/CALIB" # link = "ln -s ../rerun/" + detrend + "/CALIB/" + detrend.upper() # link += " @WORKDIR@/" + cameraInfo.addDir + "/CALIB/" + detrend.upper() super(ReduceDetrendsTest, self).__init__(name, ["pbs", "calib", detrend, camera], [command], **kwargs)
def __init__(self, name, camera, visit, ccd, rerun=None, addOptions="", **kwargs): self.camera = camera self.visit = visit self.ccd = ccd self.dataId = {'visit': visit, 'ccd': ccd} self.rerun = rerun cameraInfo = getCameraInfo(camera) command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'scProcessCcd.py') + \ " @WORKDIR@/" + cameraInfo.addDir + \ " --doraise " + addOptions + \ " --id visit=%d ccd=%d" % (self.visit, self.ccd) if rerun is not None: command += " --rerun=" + rerun super(ProcessCcdTest, self).__init__(name, ["process", camera], [command], **kwargs)
def __init__(self, name, camera, dbHost, dbName, dbUser, dbPass, source, **kwargs): cameraInfo = getCameraInfo(camera) commandList = [] self.fileList = [] for dirpath, dirnames, filenames in os.walk(source): for f in filenames: if not f.endswith('.fits'): continue self.fileList.append(f) command = os.path.join(os.environ['HSCDB_DIR'], 'bin', cameraInfo.dbRaw) command += " --execute --relocate=none --root=@WORKDIR@" command += " --dbhost " + dbHost command += " --dbname " + dbName command += " --dbuser " + dbUser command += " --dbpass " + dbPass command += " " + os.path.join(dirpath, f) commandList.append(command) query = "SELECT * FROM %s" % cameraInfo.fileTable numRowsValidate = lambda num: num == len(self.fileList) super(DbRawTest, self).__init__(name, ["db", camera], commandList, dbHost=dbHost, dbName=dbName, dbUser=dbUser, dbPass=dbPass, query=query, numRowsValidate=numRowsValidate, **kwargs)
def validate(self, workDir=".", **kwargs): cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, rerun=self.rerun, root=os.path.join(workDir, cameraInfo.addDir)) self.validateCcd(butler, self.dataId)
def __init__(self, name, camera, source, dataIdList, datasetType="raw"): self.camera = camera self.dataIdList = dataIdList self.datasetType = datasetType self.fitsFiles = set() inputs = [] for dirpath, dirnames, filenames in os.walk(source): for f in filenames: if f.endswith('.fits'): inputs.append(os.path.join(dirpath, f)) self.fitsFiles.add(f) cameraInfo = getCameraInfo(camera) self.registryDir = cameraInfo.addDir commandList = [[ os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", cameraInfo.refileScript), "--link", "--execute", "--root=@WORKDIR@" ] + inputs, [ os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genInputRegistry.py"), "--create", "--root=@WORKDIR@/" + self.registryDir, "--camera=" + cameraInfo.abbrev ]] super(DataTest, self).__init__(name, ["setup", camera], commandList)
def __init__(self, name, camera, dbType, dbHost, dbName, dbUser, dbPass, dbPort=5432, **kwargs): command = os.path.join(os.environ['HSCDB_DIR'], 'bin', 'create_HscDb.py') command += " --dbhost=" + dbHost command += " --dbtype=" + dbType command += " --dbname=" + dbName command += " --dbuser="******" --dbpass="******" --drop" self.dbPort = dbPort cameraInfo = getCameraInfo(camera) query = "SELECT * FROM %s" % cameraInfo.fileTable numRowsValidate = lambda num: num == 0 super(DbCreateTest, self).__init__(name, ["db", camera], [command], dbHost=dbHost, dbName=dbName, dbUser=dbUser, dbPass=dbPass, query=query, numRowsValidate=numRowsValidate, **kwargs)
def __init__(self, name, camera, idDictList, dataIdList, time=1200, rerun=None, **kwargs): self.camera = camera self.idDictList = idDictList self.dataIdList = dataIdList self.rerun = rerun cameraInfo = getCameraInfo(self.camera) command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'reduceFrames.py') command += " @WORKDIR@/" + cameraInfo.addDir + " " for idDict in idDictList: command += "--id " + " ".join("=".join(map(str, kv)) for kv in idDict.items()) command += " --job=" + name command += " --time=%f" % time if rerun is not None: command += " --rerun=" + rerun command += " @PBSARGS@" super(ReduceFramesTest, self).__init__(name, ["pbs", "process", camera], [command], **kwargs)
def validate(self, workDir=".", **kwargs): self.validatePbs() cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, rerun=self.rerun, root=os.path.join(workDir, cameraInfo.addDir)) numCcds = hscCamera.getNumCcds(self.camera) for dataId in self.dataIdList: self.validateCcd(butler, dataId)
def validate(self, workDir=".", **kwargs): target = self.getTargetDir(workDir) registry = os.path.join(target, "calibRegistry.sqlite3") self.assertTrue("Registry created", os.path.isfile(registry)) cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, root=os.path.join(workDir, cameraInfo.addDir)) for dataId in self.dataIdList: self.validateDataset(butler, dataId, self.detrend) return True
def __init__(self, name, camera, visit, rerun=None, **kwargs): self.camera = camera self.visit = visit self.rerun = rerun cameraInfo = getCameraInfo(camera) command = os.path.join(os.environ['SOLVETANSIP_DIR'], 'bin', 'solvetansip.py') command += " @WORKDIR@/" + cameraInfo.addDir + " --id visit=%d" % visit if rerun is not None: command += " --rerun=" + rerun super(SolveTansipTest, self).__init__(name, ["data", "astrometry", camera], [command], **kwargs)
def __init__(self, name, camera, detrend, idDict, detrendIdDict, rerun, time=1200, validity=30, **kwargs): self.camera = camera self.detrend = detrend self.rerun = rerun exeNames = { 'bias': 'reduceBias.py', 'dark': 'reduceDark.py', 'flat': 'reduceFlat.py', } if not detrend in exeNames: raise RuntimeError("Unrecognised detrend type: %s vs %s" % (detrend, exeNames.keys())) cameraInfo = getCameraInfo(self.camera) command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', exeNames[detrend]) + " " command += " @WORKDIR@/" + cameraInfo.addDir + " " command += " --id " + " ".join("=".join(map(str, kv)) for kv in idDict.items()) command += " --detrendId " + " ".join("=".join(map(str, kv)) for kv in detrendIdDict.items()) command += " --job=" + name command += " --time=%f" % time command += " @PBSARGS@" command += " --rerun=" + rerun # XXX The calibs used to end up in the root CALIB directory, but since Jim changed the rerun handling, # they go in a new rerun CALIB directory. I'm more concerned to get things working than fix things up, # so here's a workaround. # create = "mkdir -p @WORKDIR@/" + cameraInfo.addDir + "/CALIB" # link = "ln -s ../rerun/" + detrend + "/CALIB/" + detrend.upper() # link += " @WORKDIR@/" + cameraInfo.addDir + "/CALIB/" + detrend.upper() super(ReduceDetrendsTest, self).__init__(name, ["pbs", "calib", detrend, camera], [command], **kwargs)
def validate(self, workDir=".", **kwargs): found = set() for dirpath, dirnames, filenames in os.walk(os.path.join(workDir, self.registryDir)): for f in filenames: if f.endswith('.fits'): found.add(f) self.assertEqual("Number of FITS files", len(found), len(self.fitsFiles)) registry = os.path.join(workDir, self.registryDir, "registry.sqlite3") self.assertTrue("Registry created", os.path.isfile(registry)) cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, root=os.path.join(workDir, cameraInfo.addDir)) for dataId in self.dataIdList: self.validateDataset(butler, dataId, self.datasetType) return True
def __init__(self, name, camera, idDictList, dataIdList, time=1200, rerun=None, **kwargs): self.camera = camera self.idDictList = idDictList self.dataIdList = dataIdList self.rerun = rerun cameraInfo = getCameraInfo(self.camera) command = os.path.join(os.environ['HSCPIPE_DIR'], 'bin', 'reduceFrames.py') command += " @WORKDIR@/" + cameraInfo.addDir + " " for idDict in idDictList: command += "--id " + " ".join("=".join(map(str,kv)) for kv in idDict.items()) command += " --job=" + name command += " --time=%f" % time if rerun is not None: command += " --rerun=" + rerun command += " @PBSARGS@" super(ReduceFramesTest, self).__init__(name, ["pbs", "process", camera], [command], **kwargs)
def __init__(self, name, camera, source, dataIdList, datasetType="raw"): self.camera = camera self.dataIdList = dataIdList self.datasetType = datasetType self.fitsFiles = set() inputs = [] for dirpath, dirnames, filenames in os.walk(source): for f in filenames: if f.endswith('.fits'): inputs.append(os.path.join(dirpath, f)) self.fitsFiles.add(f) cameraInfo = getCameraInfo(camera) self.registryDir = cameraInfo.addDir commandList = [[os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", cameraInfo.refileScript), "--link", "--execute", "--root=@WORKDIR@"] + inputs, [os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genInputRegistry.py"), "--create", "--root=@WORKDIR@/" + self.registryDir, "--camera=" + cameraInfo.abbrev] ] super(DataTest, self).__init__(name, ["setup", camera], commandList)
def validate(self, workDir=".", **kwargs): found = set() for dirpath, dirnames, filenames in os.walk( os.path.join(workDir, self.registryDir)): for f in filenames: if f.endswith('.fits'): found.add(f) self.assertEqual("Number of FITS files", len(found), len(self.fitsFiles)) registry = os.path.join(workDir, self.registryDir, "registry.sqlite3") self.assertTrue("Registry created", os.path.isfile(registry)) cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, root=os.path.join(workDir, cameraInfo.addDir)) for dataId in self.dataIdList: self.validateDataset(butler, dataId, self.datasetType) return True
def __init__(self, name, camera, detrend, dataIdList, source=None, validity=None): self.camera = camera self.detrend = detrend self.dataIdList = dataIdList self.source = source commandList = [] cameraInfo = getCameraInfo(camera) if source is not None: # Get calibrations from the source for dirpath, dirnames, filenames in os.walk(source): targetDir = os.path.join("@WORKDIR@", cameraInfo.addDir, "CALIB", os.path.relpath(dirpath, source)) for f in filenames: commandList.append([ "ln", "-s", os.path.join(dirpath, f), os.path.join(targetDir, f) ]) generate = [ os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genCalibRegistry.py"), "--create", "--root=@WORKDIR@/" + os.path.join(cameraInfo.addDir, "CALIB"), "--camera=" + camera ] if validity is not None: generate += ["--validity=%d" % validity] commandList.append(generate) super(CalibTest, self).__init__(name, ["calib", "setup", camera], commandList)
def __init__(self, name, camera, detrend, dataIdList, source=None, validity=None): self.camera = camera self.detrend = detrend self.dataIdList = dataIdList self.source = source commandList = [] cameraInfo = getCameraInfo(camera) if source is not None: # Get calibrations from the source for dirpath, dirnames, filenames in os.walk(source): targetDir = os.path.join("@WORKDIR@", cameraInfo.addDir, "CALIB", os.path.relpath(dirpath, source)) for f in filenames: commandList.append(["ln", "-s", os.path.join(dirpath, f), os.path.join(targetDir, f)]) generate = [os.path.join(os.environ['OBS_SUBARU_DIR'], "bin", "genCalibRegistry.py"), "--create", "--root=@WORKDIR@/" + os.path.join(cameraInfo.addDir, "CALIB"), "--camera=" + camera] if validity is not None: generate += ["--validity=%d" % validity] commandList.append(generate) super(CalibTest, self).__init__(name, ["calib", "setup", camera], commandList)
def validate(self, workDir=".", **kwargs): self.validatePbs() cameraInfo = getCameraInfo(self.camera) butler = hscCamera.getButler(self.camera, rerun=self.rerun, root=os.path.join(workDir, cameraInfo.addDir))
def getTargetDir(self, workDir): cameraInfo = getCameraInfo(self.camera) return os.path.join(workDir, cameraInfo.addDir, "CALIB")