def run( rerun, # Rerun name reference, # Reference stack name measureList, # List of stack names to measure skytile, # Sky tile number config, # Configuration ): io = pipReadWrite.ReadWrite(suprimecam.SuprimecamMapper, ['visit', 'ccd'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], rerun) photProc = pipMultiPhot.MultiPhot(config=config) #import lsst.pex.logging as pexLog #pexLog.Log.getDefaultLog().setThreshold(pexLog.Log.DEBUG) refId = {'stack': reference, 'skytile': skytile, 'filter': "r"} refStack = photProc.read(io.inButler, refId, ['stack'])[0] measStackList = list() for name in measureList: dataId = {'stack': name, 'skytile': skytile, 'filter': "r"} measStack = refStack if dataId == refId else photProc.read( io.inButler, dataId, ['stack'])[0] measStackList.append(measStack) sourceList = photProc.run(refStack, measStackList) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) for name, sources in zip(measureList, sourceList): catalog.writeSources(basename + '-' + name + '.sources', sources, 'sources')
def run( rerun, # Rerun name frameList, # Frame numbers ccdList, # CCD numbers config, # Configuration command=None, # Command for PBS queueName=None, # Queue name submit=False, # Submit to queue? ): imports = [ ("lsst.obs.suprime", "suprime"), ("lsst.pipette.readwrite", "pipReadWrite"), ("lsst.pipette.processCcd", "pipProcCcd"), ("lsst.pipette.specific.suprimecam", "pipSuprimeCam"), ("lsst.pex.logging", "pexLog"), ] script = """ log = pexLog.Log.getDefaultLog() log.addDestination(basename + '.log') io = pipReadWrite.ReadWrite(suprime.SuprimeMapper, ['visit', 'ccd'], config=config) raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) ccdProc = pipProcCcd.Ccd(config=config, Isr=pipSuprimeCam.IsrSuprimeCam) exposure, psf, apcorr, sources, matches = ccdProc.run(raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') """ roots = config['roots'] catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) queue = pipQueue.PbsQueue(script, command=command, importList=imports, resourceList="walltime=300", queue=queueName) roots = config['roots'] for frame in frameList: for ccd in ccdList: basename = os.path.join(roots['output'], '%s-%d%d' % (rerun, frame, ccd)) dataId = {'visit': frame, 'ccd': ccd} if submit: queue.sub( basename, basename=basename, config=config, dataId=dataId, catalog=catalog, ) else: print "Not queuing %s: dataId=%s" % (basename, dataId) return
def run( rerun, # Rerun name visit, # Visit number snap, # Snap number raft, # Raft id sensor, # Sensor id config, # Configuration log=pexLog.Log.getDefaultLog() # Log object ): io = pipReadWrite.ReadWrite( lsstSim.LsstSimMapper, ['visit', 'snap', 'raft', 'sensor'], fileKeys=['visit', 'snap', 'raft', 'sensor', 'channel'], config=config) roots = config['roots'] basename = os.path.join( roots['output'], '%s-%d-%d-%s-%s' % (rerun, visit, snap, raft, sensor)) ccdProc = pipProcCcd.ProcessCcd(config=config, log=log) dataId = {'visit': visit, 'snap': snap, 'raft': raft, 'sensor': sensor} detrends = io.detrends(dataId, config) if len([x for x in detrends if x]): # We need to run at least part of the ISR raws = io.readRaw(dataId) else: io.fileKeys = ['visit', 'raft', 'sensor'] raws = io.read('calexp', dataId) detrends = None exposure, psf, brightSources, apcorr, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run( rerun, # Rerun name frame, # Frame number ccd, # CCD number config, # Configuration log=pexLog.Log.getDefaultLog(), # Log object ): roots = config['roots'] registry = os.path.join(roots['data'], 'registry.sqlite3') imapp = suprimecam.SuprimecamMapper(rerun=rerun, root=roots['data'], calibRoot=roots['calib']) omapp = suprimecam.SuprimecamMapper(rerun=rerun, root=roots['output'], calibRoot=roots['calib'], registry=registry) io = pipReadWrite.ReadWrite([imapp, omapp], ['visit', 'ccd'], config=config) ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=pipSuprimeCam.IsrSuprimeCam, log=log) dataId = {'visit': frame, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) basename = os.path.join(roots['output'], '%s-%d%d' % (rerun, frame, ccd)) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run(rerun, # Rerun name visit, # Visit number ccd, # CCD number config, # Configuration log = pexLog.Log.getDefaultLog(), # Log object ): io = pipReadWrite.ReadWrite(cfht.CfhtMapper, ['visit', 'ccd'], fileKeys=['amp'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], '%s-%d-%d' % (rerun, visit, ccd)) ccdProc = pipProcCcd.ProcessCcd(config=config, log=log) dataId = {'visit': visit, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) exposure, psf, brightSources, apcorr, sources, matches, matchMeta = ccdProc.run(raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run( rerun, # Rerun name frame, # Frame number ccd, # CCD number config, # Configuration log=pexLog.Log.getDefaultLog(), # Log object ): # Make our own mappers for now mapperArgs = {'rerun': rerun} # Arguments for mapper instantiation if config.has_key('roots'): roots = config['roots'] for key, value in { 'data': 'root', 'calib': 'calibRoot', 'output': 'outRoot' }.iteritems(): if roots.has_key(key): mapperArgs[value] = roots[key] camera = config['camera'] if camera.lower() in ("hsc"): mapper = obsHsc.HscSimMapper(**mapperArgs) ccdProc = pipCcd.ProcessCcd(config=config, Calibrate=CalibrateHscDc2, log=log) elif camera.lower() in ("suprimecam-mit", "sc-mit", "scmit", "suprimecam-old", "sc-old", "scold"): mapper = obsSc.SuprimecamMapper(mit=True, **mapperArgs) ccdProc = ProcessCcdSuprimeCam(config=config, log=log) elif camera.lower() in ("suprimecam", "suprime-cam", "sc"): mapper = obsSc.SuprimecamMapper(**mapperArgs) ccdProc = ProcessCcdSuprimeCam(config=config, log=log) io = pipReadWrite.ReadWrite(mapper, ['visit', 'ccd'], config=config) oldUmask = os.umask(2) if oldUmask != 2: io.log.log(io.log.WARN, "pipette umask started as: %s" % (os.umask(2))) dataId = {'visit': frame, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) if len([x for x in detrends if x]): # We need to run at least part of the ISR raws = io.readRaw(dataId) else: io.fileKeys = ['visit', 'ccd'] try: raws = io.read('calexp', dataId) config['do']['calibrate']['repair']['cosmicray'] = False except: raws = io.readRaw(dataId) detrends = None exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=None, psf=psf, sources=None) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) deferredState = DeferredHSCState(dataId, io, matches, matchMeta, sources, brightSources, exposure) return deferredState