def __init__(self, **kwargs): #Define the policy file: policyFile = Policy.defaultPolicyFile(self.packageName, "GotoMapper.yaml", "policy") policy =Policy(policyFile) #Change the policy to point to the sim camera description: policy["camera"] = "../sim/camera" #This creates the camera class by calling CameraMapper (i.e., the parent class): super(GotoMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) #Set the filters: self.filterIdMap = dict(v=0) afwImageUtils.defineFilter(name='R', lambdaEff=635.9, alias=['R']) afwImageUtils.defineFilter(name='G', lambdaEff=534.9, alias=['G']) afwImageUtils.defineFilter(name='B', lambdaEff=446.6, alias=['B']) afwImageUtils.defineFilter(name='L', lambdaEff=535.5, alias=['L']) self.filters = {} self.filters['R'] = afwImage.Filter('R').getCanonicalName() self.filters['G'] = afwImage.Filter('G').getCanonicalName() self.filters['B'] = afwImage.Filter('B').getCanonicalName() self.filters['L'] = afwImage.Filter('L').getCanonicalName() self.defaultFilterName = 'L'
def __init__(self, inputPolicy=None, **kwargs): policyFile = Policy.defaultPolicyFile(self.packageName, "ctio0m9Mapper.yaml", "policy") policy = Policy(policyFile) CameraMapper.__init__(self, policy, os.path.dirname(policyFile), **kwargs) filter_pairings = [ 'NONE+SEMROCK', # list of all filter pairings found in data 'NONE+RONCHI200', 'RONCHI200+SEMROCK', 'NONE+NONE', 'NONE+g', 'NONE+r', 'NONE+i', 'NONE+z', 'RONCHI200+z', 'RONCHI200+g', 'FGB37+RONCHI200', 'NONE+RONCHI400', 'FGC715S+RONCHI400', 'FGC715S+RONCHI200' ] # default no-filter name used for biases and darks - must appear afwImageUtils.defineFilter('NONE', 0.0, alias=[]) for pairing in filter_pairings: afwImageUtils.defineFilter(pairing, 0.0, alias=[])
def __init__(self, inputPolicy=None, **kwargs): policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") policy = Policy(policyFile) super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) DECAM_FILTER_DEFINITIONS.defineFilters() # The data ID key ccdnum is not directly used in the current policy # template of the raw and instcal et al. datasets, so is not in its # keyDict automatically. Add it so the butler know about the data ID key # ccdnum. # Similarly, add "object" for raws. for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"): self.mappings[datasetType].keyDict.update({'ccdnum': int}) self.mappings["raw"].keyDict.update({'object': str}) # The number of bits allocated for fields in object IDs # TODO: This needs to be updated; also see Trac #2797 DecamMapper._nbit_tract = 10 DecamMapper._nbit_patch = 10 DecamMapper._nbit_filter = 4 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract + 2 * DecamMapper._nbit_patch + DecamMapper._nbit_filter)
def __init__(self, inputPolicy=None, **kwargs): policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") policy = Policy(policyFile) super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) afwImageUtils.defineFilter('u', lambdaEff=350, alias=['u DECam c0006 3500.0 1000.0']) afwImageUtils.defineFilter('g', lambdaEff=450, alias=['g DECam SDSS c0001 4720.0 1520.0']) afwImageUtils.defineFilter('r', lambdaEff=600, alias=['r DECam SDSS c0002 6415.0 1480.0']) afwImageUtils.defineFilter('i', lambdaEff=750, alias=['i DECam SDSS c0003 7835.0 1470.0']) afwImageUtils.defineFilter('z', lambdaEff=900, alias=['z DECam SDSS c0004 9260.0 1520.0']) afwImageUtils.defineFilter('y', lambdaEff=1000, alias=['Y DECam c0005 10095.0 1130.0', 'Y']) afwImageUtils.defineFilter('VR', lambdaEff=630, alias=['VR DECam c0007 6300.0 2600.0']) afwImageUtils.defineFilter('N964', lambdaEff=964, alias=['N964 DECam c0008 9645.0 94.0']) afwImageUtils.defineFilter('SOLID', lambdaEff=0, alias=['solid']) # The data ID key ccdnum is not directly used in the current policy # template of the raw and instcal et al. datasets, so is not in its # keyDict automatically. Add it so the butler know about the data ID key # ccdnum. for datasetType in ("raw", "instcal", "dqmask", "wtmap"): self.mappings[datasetType].keyDict.update({'ccdnum': int}) self.mappings["raw"].keyDict.update({'object': str}) # The number of bits allocated for fields in object IDs # TODO: This needs to be updated; also see Trac #2797 DecamMapper._nbit_tract = 10 DecamMapper._nbit_patch = 10 DecamMapper._nbit_filter = 4 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract + 2*DecamMapper._nbit_patch + DecamMapper._nbit_filter)
def __init__(self, inputPolicy=None, **kwargs): # Declare the policy file... policyFile = Policy.defaultPolicyFile(self.packageName, "VircamMapper.yaml", "policy") policy = Policy(policyFile) # ...and add it to the mapper: super(VircamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = { 'visit': int, 'ccd': int, 'ccdnum': int, 'filter': str, 'dataType': str, 'expTime': float, 'dateObs': str, 'taiObs': str, # 'mjd': int, # 'field': str, # 'survey': str } for name in ( "raw", "postISRCCD", #"instcal", #"confmap", # "calexp", "src", "icSrc", "srcMatch", ): self.mappings[name].keyDict.update(keys) self.addFilters() self.filters = {} with warnings.catch_warnings(): # surpress Filter warnings; we already know this is deprecated warnings.simplefilter('ignore', category=FutureWarning) for filt in VIRCAM_FILTER_DEFINITIONS: self.filters[filt.physical_filter] = afwImage.Filter( filt.physical_filter).getCanonicalName() self.defaultFilterName = "unknown" # for filt in VISTA_FILTER_DEFINITIONS: # self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() # ...and set your default filter. self.defaultFilterName = 'Clear' ############################## # The number of bits allocated for fields in object IDs # TODO: Understand how these were set by obs_decam VircamMapper._nbit_tract = 16 VircamMapper._nbit_patch = 5 VircamMapper._nbit_filter = 4 VircamMapper._nbit_id = 64 - (VircamMapper._nbit_tract + 2 * VircamMapper._nbit_patch + VircamMapper._nbit_filter)
def makeNewConfig(oldConfig): newPolicy = Policy( Policy.defaultPolicyFile("importExtData", "importExtData.yaml", "policy")) return RepositoryCfg(root=oldConfig.root, mapper=oldConfig.mapper, mapperArgs=oldConfig.mapperArgs, parents=oldConfig.parents, policy=newPolicy)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") policy = Policy(policyFile) super(MegacamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # The "ccd" provided by the user is translated through the registry # into an extension name for the "raw" template. The template # therefore doesn't include "ccd", so we need to ensure it's # explicitly included so the ArgumentParser can recognise and accept # it. self.exposures['raw'].keyDict['ccd'] = int afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") # define filters? self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) # Ensure each dataset type of interest knows about the full range of # keys available from the registry keys = { 'runId': str, 'object': str, 'visit': int, 'ccd': int, 'extension': int, 'state': str, 'filter': str, 'date': str, 'taiObs': str, 'expTime': float, } for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): self.mappings[name].keyDict.update(keys)
def __init__(self, inputPolicy=None, **kwargs): #Tell the mapper the location of the policy file, which #informs the stack where to read/write and the format of #files. policyFile = Policy.defaultPolicyFile(self.packageName, "SuperbitMapper.yaml", "policy") policy = Policy(policyFile) #Instantiate the parent class (CameraMapper) with the policy file: super(SuperbitMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) #Ensure each dataset type of interest knows about the full range #of keys available from the registry. This means a minimal set of #--id's need to be specified, and the stack will find the rest. keys = {'visit':int, 'filter':str, 'dataType':str, 'expTime':float, 'dateObs':str, 'taiObs':str } for name in ("raw", "postISRCCD", "calexp", "src", "icSrc", "srcMatch"): self.mappings[name].keyDict.update(keys) #Define the filters in the filter registry #obtained from https://sites.physics.utoronto.ca/bit/documentation/camera_lenses/palestine-filter-list afwImageUtils.defineFilter(name='Open', lambdaEff=500, alias=['O']) afwImageUtils.defineFilter(name='Luminance', lambdaEff=500, alias=['L']) afwImageUtils.defineFilter(name='IR', lambdaEff=800, alias=['IR']) afwImageUtils.defineFilter(name='Red', lambdaEff=650, alias=['R']) afwImageUtils.defineFilter(name='Green',lambdaEff=550, alias=['G']) afwImageUtils.defineFilter(name='Blue', lambdaEff=450, alias=['B']) afwImageUtils.defineFilter(name='UV', lambdaEff=375, alias=['UV']) #Allocate the newly-defined filters #Is this ok? self.filters = {} self.filters['O'] = afwImage.Filter('O').getCanonicalName() self.filters['L'] = afwImage.Filter('L').getCanonicalName() self.filters['IR'] = afwImage.Filter('IR').getCanonicalName() self.filters['R'] = afwImage.Filter('R').getCanonicalName() self.filters['G'] = afwImage.Filter('G').getCanonicalName() self.filters['B'] = afwImage.Filter('B').getCanonicalName() self.filters['UV'] = afwImage.Filter('UV').getCanonicalName() #I'm not sure whether this is necessary, but it seems like a #good idea... self.defaultFilterName = 'O'
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_subaru", "SuprimecamMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'SUPA') except: raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): kwargs['calibRoot'] = os.path.join(kwargs['root'], 'CALIB') super(SuprimecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) self.defineFilters()
def testMerge(self): a = Policy() b = Policy() a['a.b.c'] = 1 b['a.b.c'] = 2 b['a.b.d'] = 3 a.merge(b) self.assertEqual(a['a.b.c'], 1) self.assertEqual(a['a.b.d'], 3) # b should remain unchanged: self.assertEqual(b['a.b.c'], 2) self.assertEqual(b['a.b.d'], 3)
def move_files(calib_root, policy_file=POLICY_FILE): """Copy files from a DESC DC2 gen2 origin repo to one with modified names to change parsed filter names to match that expected by the gen3 butler. Parameters ---------- calib_root : str full path to the calib gen2 (POSIX) filestore policy_file : str Policy file from which to load templates """ log = lsst.log.Log.getLogger("convertRepo") policy = Policy(policy_file) for data_type in DATATYPES_TO_CONVERT: if data_type == "SKY": template = policy["calibrations"]["sky"]["template"] template = template.replace("sky", "SKY") else: template = policy["calibrations"][data_type]["template"] for old_filename, new_filename in _transform_pairs( calib_root, template): log.info("Moving %s to %s", old_filename, new_filename) new_dir = os.path.split(new_filename)[0] try: os.makedirs(new_dir) except FileExistsError: pass shutil.move(old_filename, new_filename)
def __init__(self, inputPolicy=None, **kwargs): #Define the policy file: policyFile = Policy.defaultPolicyFile(self.packageName, "GotoMapper.yaml", "policy") policy = Policy(policyFile) #This creates the camera class by calling CameraMapper (i.e., the parent class): super(GotoMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = { 'visit': int, 'ccd': int, 'filter': str, 'dataType': str, 'expTime': float, 'dateObs': str, 'taiObs': str, 'mjd': int, 'field': str, 'survey': str } for name in ( "raw", "postISRCCD", "calexp", "src", "icSrc", "srcMatch", ): self.mappings[name].keyDict.update(keys) #Set the filters: self.filterIdMap = dict(v=0) afwImageUtils.defineFilter(name='R', lambdaEff=635.9, alias=['R']) afwImageUtils.defineFilter(name='G', lambdaEff=534.9, alias=['G']) afwImageUtils.defineFilter(name='B', lambdaEff=446.6, alias=['B']) afwImageUtils.defineFilter(name='L', lambdaEff=535.5, alias=['L']) self.filters = {} self.filters['R'] = afwImage.Filter('R').getCanonicalName() self.filters['G'] = afwImage.Filter('G').getCanonicalName() self.filters['B'] = afwImage.Filter('B').getCanonicalName() self.filters['L'] = afwImage.Filter('L').getCanonicalName() self.defaultFilterName = 'L'
def makeNewConfig(oldConfig): newPolicy = Policy(Policy.defaultPolicyFile("importExtData", "importExtData.yaml", "policy")) return RepositoryCfg(root=oldConfig.root, mapper=oldConfig.mapper, mapperArgs=oldConfig.mapperArgs, parents=oldConfig.parents, policy=newPolicy)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_subaru", "SuprimecamMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'SUPA') except Exception: raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): kwargs['calibRoot'] = os.path.join(kwargs['root'], 'CALIB') super(SuprimecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) self.defineFilters()
def __init__(self, cfgRoot=None, root=None, mapper=None, mapperArgs=None, tags=None, mode=None, policy=None): try: # is cfgRoot a dict? try dict init: self.__init__(**cfgRoot) except TypeError: self._root = Storage.absolutePath(os.getcwd(), root.rstrip(os.sep)) if root else root self._cfgRoot = Storage.absolutePath(os.getcwd(), cfgRoot.rstrip(os.sep)) if cfgRoot else cfgRoot self._mapper = mapper self.mapperArgs = mapperArgs self.tags = set(listify(tags)) self.mode = mode self.policy = Policy(policy) if policy is not None else None
def __init__(self, inputPolicy=None, **kwargs): #Define the policy file: policyFile = Policy.defaultPolicyFile(self.packageName, "GotoMapper.yaml", "policy") policy =Policy(policyFile) #This creates the camera class by calling CameraMapper (i.e., the parent class): super(GotoMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) #Set the filters: self.filterIdMap = dict(v=0) afwImageUtils.defineFilter(name='v', lambdaEff=450, alias=['v']) afwImageUtils.defineFilter(name='m', lambdaEff=610.5, alias=['m']) afwImageUtils.defineFilter(name='Clear', lambdaEff=450, alias=['Clear']) self.filters = {} self.filters['v'] = afwImage.Filter('v').getCanonicalName() self.filters['m'] = afwImage.Filter('m').getCanonicalName() self.filters['Clear'] = afwImage.Filter('Clear').getCanonicalName() self.defaultFilterName = 'm'
def __init__(self, inputPolicy=None, **kwargs): policyFile = Policy.defaultPolicyFile(self.packageName, "CtmoMapper.yaml", "policy") policy = Policy(policyFile) # ...and add it to the mapper: super(CtmoCamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # Define your filter set # Create a dict of filters: self.filters = {} # Define your set of filters; you can have as many filters as you like... afwImageUtils.defineFilter(name="Clear", lambdaEff=535.5, alias=["Clear"]) # ...add them to your filter dict... self.filters["Clear"] = afwImage.Filter("Clear").getCanonicalName() # ...and set your default filter. self.defaultFilterName = "Clear"
def __init__(self, inputPolicy=None, **kwargs): #Declare the policy file... policyFile = Policy.defaultPolicyFile(self.packageName, "NecamMapper.yaml", "policy") policy = Policy(policyFile) #...and add it to the mapper: super(NecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) ###Defining your filter set### #Create a python dict of filters: self.filters = {} #Define your set of filters; you can have as many filters as you like... afwImageUtils.defineFilter(name='Clear', lambdaEff=535.5, alias=['Clear']) #...add them to your filter dict... self.filters['Clear'] = afwImage.Filter('Clear').getCanonicalName() #...and set your default filter. self.defaultFilterName = 'Clear'
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_generic", "GenericMapper.yaml", "policy") #policy = pexPolicy.Policy(policyFile) policy = Policy(policyFile) #super(GenericMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs) super(GenericMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # The "ccd" provided by the user is translated through the registry into an extension name for the "raw" # template. The template therefore doesn't include "ccd", so we need to ensure it's explicitly included # so the ArgumentParser can recognise and accept it. self.exposures['raw'].keyDict['ccd'] = int # define filters? #self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5) self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4) afwImageUtils.defineFilter('u', lambdaEff=380) afwImageUtils.defineFilter('g', lambdaEff=450) afwImageUtils.defineFilter('r', lambdaEff=600) afwImageUtils.defineFilter('i', lambdaEff=770) #afwImageUtils.defineFilter('i2', lambdaEff=750) afwImageUtils.defineFilter('z', lambdaEff=900) afwImageUtils.defineFilter('decam_u', lambdaEff=350, alias=['u DECam c0006 3500.0 1000.0', 'u']) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = {'source': str, 'visit': int, 'ccd': int, #'filter': str, } for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): self.mappings[name].keyDict.update(keys)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") policy = Policy(policyFile) super(MegacamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # The "ccd" provided by the user is translated through the registry # into an extension name for the "raw" template. The template # therefore doesn't include "ccd", so we need to ensure it's # explicitly included so the ArgumentParser can recognise and accept # it. self.exposures['raw'].keyDict['ccd'] = int afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") # define filters? self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) # Ensure each dataset type of interest knows about the full range of # keys available from the registry keys = {'runId': str, 'object': str, 'visit': int, 'ccd': int, 'extension': int, 'state': str, 'filter': str, 'date': str, 'taiObs': str, 'expTime': float, } for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): self.mappings[name].keyDict.update(keys)
def __init__(self, storage, policy): # todo I'm guessing the policy would probably want to come from the default in-package location, and # then be overridden where desired by policy in repository root, and then # have the cfg policy applied self.policy = Policy(policy) self.storage = storage
def __init__(self, inputPolicy=None, **kwargs): policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") policy = Policy(policyFile) super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # lambdaMin and lambda max are chosen to be where the filter rises above 1% # from http://www.ctio.noao.edu/noao/sites/default/files/DECam/DECam_filters_transmission.txt afwImageUtils.defineFilter('u', lambdaEff=350, lambdaMin=305, lambdaMax=403, alias=['u DECam c0006 3500.0 1000.0']) afwImageUtils.defineFilter('g', lambdaEff=450, lambdaMin=394, lambdaMax=555, alias=['g DECam SDSS c0001 4720.0 1520.0']) afwImageUtils.defineFilter('r', lambdaEff=600, lambdaMin=562, lambdaMax=725, alias=['r DECam SDSS c0002 6415.0 1480.0']) afwImageUtils.defineFilter('i', lambdaEff=750, lambdaMin=699, lambdaMax=870, alias=['i DECam SDSS c0003 7835.0 1470.0']) afwImageUtils.defineFilter('z', lambdaEff=900, lambdaMin=837, lambdaMax=1016, alias=['z DECam SDSS c0004 9260.0 1520.0']) afwImageUtils.defineFilter('y', lambdaEff=1000, lambdaMin=941, lambdaMax=1080, alias=['Y DECam c0005 10095.0 1130.0', 'Y']) afwImageUtils.defineFilter('VR', lambdaEff=630, lambdaMin=490, lambdaMax=765, alias=['VR DECam c0007 6300.0 2600.0']) afwImageUtils.defineFilter('N964', lambdaEff=964, alias=['N964 DECam c0008 9645.0 94.0']) afwImageUtils.defineFilter('SOLID', lambdaEff=0, alias=['solid']) # The data ID key ccdnum is not directly used in the current policy # template of the raw and instcal et al. datasets, so is not in its # keyDict automatically. Add it so the butler know about the data ID key # ccdnum. # Similarly, add "object" for raws. for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"): self.mappings[datasetType].keyDict.update({'ccdnum': int}) self.mappings["raw"].keyDict.update({'object': str}) # The number of bits allocated for fields in object IDs # TODO: This needs to be updated; also see Trac #2797 DecamMapper._nbit_tract = 10 DecamMapper._nbit_patch = 10 DecamMapper._nbit_filter = 4 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract + 2 * DecamMapper._nbit_patch + DecamMapper._nbit_filter)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join( os.environ.get('SUPRIME_DATA_DIR'), 'HSC') except: raise RuntimeError( "Either $SUPRIME_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): calibSearch = [os.path.join(kwargs['root'], 'CALIB')] if "repositoryCfg" in kwargs: calibSearch += [ os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root") ] calibSearch += [ cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root") ] for calibRoot in calibSearch: if os.path.exists( os.path.join(calibRoot, "calibRegistry.sqlite3")): kwargs['calibRoot'] = calibRoot break if not kwargs.get('calibRoot', None): lsst.log.Log.getLogger("HscMapper").warn( "Unable to find calib root directory") super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) self._linearize = LinearizeSquared() # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = { 'field': str, 'visit': int, 'filter': str, 'ccd': int, 'dateObs': str, 'taiObs': str, 'expTime': float, 'pointing': int, } for name in ( "raw", # processCcd outputs "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", "srcMatch", # mosaic outputs "wcs", "fcr", # processCcd QA "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", # forcedPhot outputs "forced_src", ): self.mappings[name].keyDict.update(keys) # SDSS g': http://www.naoj.org/Observing/Instruments/SCam/txt/g.txt # SDSS r': http://www.naoj.org/Observing/Instruments/SCam/txt/r.txt # SDSS i': http://www.naoj.org/Observing/Instruments/SCam/txt/i.txt # SDSS z': http://www.naoj.org/Observing/Instruments/SCam/txt/z.txt # y-band: Shimasaku et al., 2005, PASJ, 57, 447 # The order of these defineFilter commands matters as their IDs are used to generate at least some # object IDs (e.g. on coadds) and changing the order will invalidate old objIDs afwImageUtils.resetFilters() afwImageUtils.defineFilter(name="UNRECOGNISED", lambdaEff=0, alias=[ "NONE", "None", "Unrecognised", "UNRECOGNISED", "Unrecognized", "UNRECOGNIZED", "NOTSET", ]) afwImageUtils.defineFilter(name='g', lambdaEff=477, alias=['W-S-G+', 'HSC-G']) afwImageUtils.defineFilter(name='r', lambdaEff=623, alias=['W-S-R+', 'HSC-R']) afwImageUtils.defineFilter(name='r1', lambdaEff=623, alias=['109', 'ENG-R1']) afwImageUtils.defineFilter(name='i', lambdaEff=775, alias=['W-S-I+', 'HSC-I']) afwImageUtils.defineFilter(name='z', lambdaEff=925, alias=['W-S-Z+', 'HSC-Z']) afwImageUtils.defineFilter(name='y', lambdaEff=990, alias=['W-S-ZR', 'HSC-Y']) afwImageUtils.defineFilter(name='N387', lambdaEff=387, alias=['NB0387']) afwImageUtils.defineFilter(name='N515', lambdaEff=515, alias=['NB0515']) afwImageUtils.defineFilter(name='N656', lambdaEff=656, alias=['NB0656']) afwImageUtils.defineFilter(name='N816', lambdaEff=816, alias=['NB0816']) afwImageUtils.defineFilter(name='N921', lambdaEff=921, alias=['NB0921']) afwImageUtils.defineFilter(name='N1010', lambdaEff=1010, alias=['NB1010']) afwImageUtils.defineFilter(name='SH', lambdaEff=0, alias=[ 'SH', ]) afwImageUtils.defineFilter(name='PH', lambdaEff=0, alias=[ 'PH', ]) afwImageUtils.defineFilter(name='N527', lambdaEff=527, alias=['NB0527']) afwImageUtils.defineFilter(name='N718', lambdaEff=718, alias=['NB0718']) afwImageUtils.defineFilter(name='I945', lambdaEff=945, alias=['IB0945']) afwImageUtils.defineFilter(name='N973', lambdaEff=973, alias=['NB0973']) afwImageUtils.defineFilter(name='i2', lambdaEff=775, alias=['HSC-I2']) afwImageUtils.defineFilter(name='r2', lambdaEff=623, alias=['HSC-R2']) afwImageUtils.defineFilter(name='N468', lambdaEff=468, alias=['NB0468']) afwImageUtils.defineFilter(name='N926', lambdaEff=926, alias=['NB0926']) # # self.filters is used elsewhere, and for now we'll set it # # It's a bit hard to initialise self.filters properly until #2113 is resolved, # including the part that makes it possible to get all aliases # self.filters = {} for f in [ "HSC-G", "HSC-R", "HSC-R2", "HSC-I", "HSC-I2", "HSC-Z", "HSC-Y", "ENG-R1", "NB0387", "NB0468", "NB0515", "NB0527", "NB0656", "NB0718", "NB0816", "NB0921", "NB0926", "IB0945", "NB0973", "NB1010", "SH", "PH", "NONE", "UNRECOGNISED" ]: self.filters[f] = afwImage.Filter(f).getCanonicalName() self.defaultFilterName = "UNRECOGNISED" # # The number of bits allocated for fields in object IDs, appropriate for # the default-configured Rings skymap. # # This shouldn't be the mapper's job at all; see #2797. HscMapper._nbit_tract = 16 HscMapper._nbit_patch = 5 HscMapper._nbit_filter = 6 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2 * HscMapper._nbit_patch + HscMapper._nbit_filter) if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: raise RuntimeError( "You have more filters defined than fit into the %d bits allocated" % HscMapper._nbit_filter)
def __init__(self, root=None, registry=None, calibRoot=None, calibRegistry=None, provided=None, parentRegistry=None, repositoryCfg=None): """Initialize the CameraMapper. Parameters ---------- policy : daf_persistence.Policy, Can also be pexPolicy.Policy, only for backward compatibility. Policy with per-camera defaults already merged. repositoryDir : string Policy repository for the subclassing module (obtained with getRepositoryPath() on the per-camera default dictionary). root : string, optional Path to the root directory for data. registry : string, optional Path to registry with data's metadata. calibRoot : string, optional Root directory for calibrations. calibRegistry : string, optional Path to registry with calibrations' metadata. provided : list of string, optional Keys provided by the mapper. parentRegistry : Registry subclass, optional Registry from a parent repository that may be used to look up data's metadata. repositoryCfg : daf_persistence.RepositoryCfg or None, optional The configuration information for the repository this mapper is being used with. """ policyFile = Policy.defaultPolicyFile("obs_hsc_sims", "HscSimsMapper.yaml", "policy") policy = Policy(policyFile) dafPersist.Mapper.__init__(self) self.log = lsstLog.Log.getLogger("HscSimsMapper") if root: self.root = root elif repositoryCfg: self.root = repositoryCfg.root else: self.root = None if isinstance(policy, pexPolicy.Policy): policy = dafPersist.Policy(policy) repoPolicy = repositoryCfg.policy if repositoryCfg else None if repoPolicy is not None: policy.update(repoPolicy) # Don't load the default policy from obs_base # defaultPolicyFile = dafPersist.Policy.defaultPolicyFile("obs_base", # "MapperDictionary.paf", # "policy") # dictPolicy = dafPersist.Policy(defaultPolicyFile) # policy.merge(dictPolicy) # Levels self.levels = dict() if 'levels' in policy: levelsPolicy = policy['levels'] for key in levelsPolicy.names(True): self.levels[key] = set(levelsPolicy.asArray(key)) self.defaultLevel = policy['defaultLevel'] self.defaultSubLevels = dict() if 'defaultSubLevels' in policy: self.defaultSubLevels = policy['defaultSubLevels'] # Root directories if root is None: root = "." root = dafPersist.LogicalLocation(root).locString() self.rootStorage = dafPersist.Storage.makeFromURI(uri=root) # If the calibRoot is passed in, use that. If not and it's indicated in # the policy, use that. And otherwise, the calibs are in the regular # root. # If the location indicated by the calib root does not exist, do not # create it. calibStorage = None if calibRoot is not None: calibRoot = dafPersist.Storage.absolutePath(root, calibRoot) calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot, create=False) else: calibRoot = policy.get('calibRoot', None) if calibRoot: calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot, create=False) if calibStorage is None: calibStorage = self.rootStorage self.root = root # Registries self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath", self.rootStorage, searchParents=False, posixIfNoSql=True) if not self.registry: self.registry = parentRegistry needCalibRegistry = policy.get('needCalibRegistry', None) if needCalibRegistry: if calibStorage: self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy, "calibRegistryPath", calibStorage, posixIfNoSql=False) # NB never use posix for calibs else: raise RuntimeError( "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " + "calibRoot ivar:%s or policy['calibRoot']:%s" % (calibRoot, policy.get('calibRoot', None))) else: self.calibRegistry = None # Dict of valid keys and their value types self.keyDict = dict() self._initMappings(policy, self.rootStorage, calibStorage, provided=None) self._initWriteRecipes() # Camera geometry # #self.cameraDataLocation = None # path to camera geometry config file # #self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir) # Defect registry and root. Defects are stored with the camera and the registry is loaded from the # camera package, which is on the local filesystem. # #self.defectRegistry = None # #if 'defects' in policy: # # self.defectPath = os.path.join(repositoryDir, policy['defects']) # # defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") # # self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) # Filter translation table self.filters = None
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') except Exception: raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): calibSearch = [os.path.join(kwargs['root'], 'CALIB')] if "repositoryCfg" in kwargs: calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] for calibRoot in calibSearch: if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): kwargs['calibRoot'] = calibRoot break if not kwargs.get('calibRoot', None): lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) self._linearize = LinearizeSquared() # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = {'field': str, 'visit': int, 'filter': str, 'ccd': int, 'dateObs': str, 'taiObs': str, 'expTime': float, 'pointing': int, } for name in ("raw", # processCcd outputs "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", "srcMatch", # mosaic outputs "wcs", "fcr", # processCcd QA "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", # forcedPhot outputs "forced_src", ): self.mappings[name].keyDict.update(keys) self.addFilters() self.filters = {} for filt in HSC_FILTER_DEFINITIONS: self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() self.defaultFilterName = "UNRECOGNISED" # # The number of bits allocated for fields in object IDs, appropriate for # the default-configured Rings skymap. # # This shouldn't be the mapper's job at all; see #2797. HscMapper._nbit_tract = 16 HscMapper._nbit_patch = 5 HscMapper._nbit_filter = 6 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % HscMapper._nbit_filter)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") policy = Policy(policyFile) repositoryDir = os.path.dirname(policyFile) super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs) # Defect registry and root. Defects are stored with the camera and the # registry is loaded from the camera package, which is on the local # filesystem. self.defectRegistry = None if 'defects' in policy: self.defectPath = os.path.join(repositoryDir, policy['defects']) defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) # The "ccd" provided by the user is translated through the registry # into an extension name for the "raw" template. The template # therefore doesn't include "ccd", so we need to ensure it's # explicitly included so the ArgumentParser can recognise and accept # it. self.exposures['raw'].keyDict['ccd'] = int afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") # define filters? self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) # Ensure each dataset type of interest knows about the full range of # keys available from the registry keys = {'runId': str, 'object': str, 'visit': int, 'ccd': int, 'extension': int, 'state': str, 'filter': str, 'date': str, 'taiObs': str, 'expTime': float, } for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): self.mappings[name].keyDict.update(keys) # # The number of bits allocated for fields in object IDs, appropriate # for the default-configured Rings skymap. # MegacamMapper._nbit_tract = 16 MegacamMapper._nbit_patch = 5 MegacamMapper._nbit_filter = 6 MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch + MegacamMapper._nbit_filter) if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter: raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % MegacamMapper._nbit_filter)
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_pfs", "PfsMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): pfsDataDir = os.environ.get('PFS_DATA_DIR') if pfsDataDir is None: raise RuntimeError( "Either $PFS_DATA_DIR or root= must be specified") kwargs['root'] = os.path.join(pfsDataDir, 'PFS') if not kwargs.get('calibRoot', None): kwargs['calibRoot'] = os.path.join(kwargs['root'], 'CALIB') super(PfsMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = { 'site': str, 'category': str, 'field': str, 'visit': int, 'ccd': int, 'filter': str, 'arm': str, 'spectrograph': int, 'dateObs': str, 'expTime': float, 'dataType': str, 'taiObs': str, 'pfsDesignId': int, 'slitOffset': float, } for name in ("raw", "pfsArm", "wlFitData", "arcLines"): self.mappings[name].keyDict.update(keys) # The order of these defineFilter commands matters as their IDs are used to generate at least some # object IDs (e.g. on coadds) and changing the order will invalidate old objIDs afwImageUtils.resetFilters() afwImageUtils.defineFilter(name="UNRECOGNISED", lambdaEff=0, alias=[ "NONE", "None", "Unrecognised", "UNRECOGNISED", "Unrecognized", "UNRECOGNIZED", "NOTSET", ]) afwImageUtils.defineFilter(name='b', lambdaEff=477, alias=['blue', 'PFS-B']) afwImageUtils.defineFilter(name='r', lambdaEff=623, alias=['red', 'PFS-R']) afwImageUtils.defineFilter(name='n', lambdaEff=623, alias=['nearInfraRed', 'PFS-N']) afwImageUtils.defineFilter(name='m', lambdaEff=775, alias=['mediumResolutionRed', 'PFS-M']) # # self.filters is used elsewhere, and for now we'll set it # self.filters = {} for f in ["b", "r", "n", "m", "NONE", "UNRECOGNISED"]: # Get the canonical name -- see #2113 self.filters[f] = afwImage.Filter( afwImage.Filter(f).getId()).getName() self.defaultFilterName = "UNRECOGNISED" # # The number of bits allocated for fields in object IDs, appropriate for # the default-configured Rings skymap. # # This shouldn't be the mapper's job at all; see #2797. PfsMapper._nbit_id = 64
def setUp(self): self.policy = Policy( data={'body': {'job': {'position': 'Developer', 'company': 'Microsoft'}, 'name': 'John'}, 'error': False})
def testBasic(self): p = Policy() p['a'] = {1:2} self.assertEqual(p, {'a':{1:2}}) p.update({'a':{3:4}}) self.assertEqual(p, {'a':{1:2, 3:4}})
def read(self, butlerLocation): """Read from a butlerLocation. :param butlerLocation: :return: a list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ additionalData = butlerLocation.getAdditionalData() # Create a list of Storages for the item. storageName = butlerLocation.getStorageName() results = [] locations = butlerLocation.getLocations() pythonType = butlerLocation.getPythonType() if pythonType is not None: if isinstance(pythonType, basestring): # import this pythonType dynamically pythonTypeTokenList = pythonType.split('.') importClassString = pythonTypeTokenList.pop() importClassString = importClassString.strip() importPackage = ".".join(pythonTypeTokenList) importType = __import__(importPackage, globals(), locals(), [importClassString], -1) pythonType = getattr(importType, importClassString) # see note re. discomfort with the name 'butlerWrite' in the write method, above. Same applies to butlerRead. if hasattr(pythonType, 'butlerRead'): results = pythonType.butlerRead(butlerLocation=butlerLocation) return results for locationString in locations: logLoc = LogicalLocation(locationString, additionalData) if storageName == "PafStorage": finalItem = pexPolicy.Policy.createPolicy(logLoc.locString()) elif storageName == "YamlStorage": finalItem = Policy(filePath=logLoc.locString()) elif storageName == "PickleStorage": if not os.path.exists(logLoc.locString()): raise RuntimeError, "No such pickle file: " + logLoc.locString() with open(logLoc.locString(), "rb") as infile: finalItem = cPickle.load(infile) elif storageName == "FitsCatalogStorage": if not os.path.exists(logLoc.locString()): raise RuntimeError, "No such FITS catalog file: " + logLoc.locString() hdu = additionalData.getInt("hdu", 0) flags = additionalData.getInt("flags", 0) finalItem = pythonType.readFits(logLoc.locString(), hdu, flags) elif storageName == "ConfigStorage": if not os.path.exists(logLoc.locString()): raise RuntimeError, "No such config file: " + logLoc.locString() finalItem = pythonType() finalItem.load(logLoc.locString()) else: storageList = StorageList() storage = self.persistence.getRetrieveStorage(storageName, logLoc) storageList.append(storage) itemData = self.persistence.unsafeRetrieve( butlerLocation.getCppType(), storageList, additionalData) finalItem = pythonType.swigConvert(itemData) results.append(finalItem) return results
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') except Exception: raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): calibSearch = [os.path.join(kwargs['root'], 'CALIB')] if "repositoryCfg" in kwargs: calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] for calibRoot in calibSearch: if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): kwargs['calibRoot'] = calibRoot break if not kwargs.get('calibRoot', None): lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) self._linearize = LinearizeSquared() # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = {'field': str, 'visit': int, 'filter': str, 'ccd': int, 'dateObs': str, 'taiObs': str, 'expTime': float, 'pointing': int, } for name in ("raw", # processCcd outputs "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", "srcMatch", # mosaic outputs "wcs", "fcr", # processCcd QA "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", # forcedPhot outputs "forced_src", ): self.mappings[name].keyDict.update(keys) self.addFilters() # # self.filters is used elsewhere, and for now we'll set it # # It's a bit hard to initialise self.filters properly until #2113 is resolved, # including the part that makes it possible to get all aliases # self.filters = {} for f in HSC_FILTER_NAMES: self.filters[f] = afwImage.Filter(f).getCanonicalName() self.defaultFilterName = "UNRECOGNISED" # # The number of bits allocated for fields in object IDs, appropriate for # the default-configured Rings skymap. # # This shouldn't be the mapper's job at all; see #2797. HscMapper._nbit_tract = 16 HscMapper._nbit_patch = 5 HscMapper._nbit_filter = 6 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % HscMapper._nbit_filter)
import sqlite3 import pathlib import tempfile import yaml from contextlib import closing import lsst.log import lsst.log.utils from lsst.daf.persistence import Policy from lsst.obs.base.gen2to3.repoWalker import PathElementParser # constants DATATYPES_TO_CONVERT = ("flat", "sky", "SKY") POLICY_FILE = Policy.defaultPolicyFile("obs_lsst", "lsstCamMapper.yaml", "policy") # exception classes # interface functions def move_files(calib_root, policy_file=POLICY_FILE): """Copy files from a DESC DC2 gen2 origin repo to one with modified names to change parsed filter names to match that expected by the gen3 butler. Parameters ---------- calib_root : str full path to the calib gen2 (POSIX) filestore policy_file : str
def __init__(self, **kwargs): policyFile = Policy.defaultPolicyFile("obs_pfs", "PfsMapper.yaml", "policy") policy = Policy(policyFile) if not kwargs.get('root', None): try: kwargs['root'] = os.path.join(os.environ.get('PFS_DATA_DIR'), 'PFS') except: raise RuntimeError("Either $PFS_DATA_DIR or root= must be specified") if not kwargs.get('calibRoot', None): kwargs['calibRoot'] = os.path.join(kwargs['root'], 'CALIB') super(PfsMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = {'site': str, 'category': str, 'field': str, 'expId': int, 'visit': int, 'ccd': int, 'filter': str, 'arm': str, 'spectrograph': int, 'dateObs': str, 'expTime': float, 'dataType': str, 'taiObs': str, 'pfiDesignId': int, 'slitOffset': float, } for name in ("raw", "pfsArm", "pfsMerged"): self.mappings[name].keyDict.update(keys) # The order of these defineFilter commands matters as their IDs are used to generate at least some # object IDs (e.g. on coadds) and changing the order will invalidate old objIDs afwImageUtils.resetFilters() afwImageUtils.defineFilter(name="UNRECOGNISED", lambdaEff=0, alias=["NONE", "None", "Unrecognised", "UNRECOGNISED", "Unrecognized", "UNRECOGNIZED", "NOTSET",]) afwImageUtils.defineFilter(name='b', lambdaEff=477, alias=['blue', 'PFS-B']) afwImageUtils.defineFilter(name='r', lambdaEff=623, alias=['red', 'PFS-R']) afwImageUtils.defineFilter(name='n', lambdaEff=623, alias=['nearInfraRed', 'PFS-N']) afwImageUtils.defineFilter(name='m', lambdaEff=775, alias=['mediumResolutionRed', 'PFS-M']) # # self.filters is used elsewhere, and for now we'll set it # self.filters = {} for f in ["b", "r", "n", "m", "NONE", "UNRECOGNISED" ]: # Get the canonical name -- see #2113 self.filters[f] = afwImage.Filter(afwImage.Filter(f).getId()).getName() self.defaultFilterName = "UNRECOGNISED" # # The number of bits allocated for fields in object IDs, appropriate for # the default-configured Rings skymap. # # This shouldn't be the mapper's job at all; see #2797. PfsMapper._nbit_id = 64
def __init__(self): path = Policy.defaultPolicyFile('ap_verify', 'dataset_config.yaml', 'config') self._allInfo = Policy(path) self._validate()
class PolicyTestCase(unittest.TestCase): """A test case for the butler policy""" def setUp(self): self.policy = Policy( data={'body': {'job': {'position': 'Developer', 'company': 'Microsoft'}, 'name': 'John'}, 'error': False}) def testBasic(self): p = Policy() p['a'] = {1:2} self.assertEqual(p, {'a':{1:2}}) p.update({'a':{3:4}}) self.assertEqual(p, {'a':{1:2, 3:4}}) def testUpdateWithDict(self): self.policy.update({'body': {'job': {'position': 'Manager'}}}) self.assertEqual(self.policy['body'], {'job': {'position': 'Manager', 'company': 'Microsoft'}, 'name': 'John'}) self.policy.update({'body': {'name': {'first':'John', 'last':'Smith'}}}) self.assertEqual(self.policy['body'], {'job': {'position': 'Manager', 'company': 'Microsoft'}, 'name': {'first':'John', 'last':'Smith'}}) def testUpdateWithPolicy(self): p1 = Policy(data={'body': {'job': {'position': 'Manager'}}}) self.policy.update(p1) self.assertEqual(self.policy['body'], {'job': {'position': 'Manager', 'company': 'Microsoft'}, 'name': 'John'}) def testGet(self): self.assertEqual(self.policy['body'], {'job': {'position': 'Developer', 'company': 'Microsoft'}, 'name': 'John'}) self.assertEqual(self.policy['body.job'], {'position': 'Developer', 'company': 'Microsoft'}) self.assertEqual(self.policy['body.job.position'], 'Developer') def testSet(self): # change an item self.policy['body.job.company'] = 'SLAC' self.assertEqual(self.policy['body'], {'job': {'position': 'Developer', 'company': 'SLAC'}, 'name': 'John'}) # add an item self.policy['body.job.salary'] = 100000 self.assertEqual(self.policy['body'], {'job': {'position': 'Developer', 'company': 'SLAC', 'salary': 100000}, 'name': 'John'}) def testNames(self): names = self.policy.names() names.sort() expectedNames = ['body', 'body.job', 'body.job.position', 'body.job.company', 'body.name', 'error'] expectedNames.sort() self.assertEqual(names, expectedNames) def testCopyPolicy(self): pol = Policy(policy=self.policy) self.assertEqual(pol, self.policy) def testGetPolicy(self): policy = self.policy['body'] self.assertEqual(policy, {'job': {'position': 'Developer', 'company': 'Microsoft'}, 'name': 'John'}) self.assertEqual(policy['job.position'], 'Developer') # note: verifies dot naming self.assertTrue(isinstance(policy, Policy)) def testDotsInBraces(self): self.assertEqual(self.policy['body.job.company'], 'Microsoft') def testMerge(self): a = Policy() b = Policy() a['a.b.c'] = 1 b['a.b.c'] = 2 b['a.b.d'] = 3 a.merge(b) self.assertEqual(a['a.b.c'], 1) self.assertEqual(a['a.b.d'], 3) # b should remain unchanged: self.assertEqual(b['a.b.c'], 2) self.assertEqual(b['a.b.d'], 3) def testOpenDefaultPolicy(self): policy = Policy(defaultInitData=('daf_persistence', 'testPolicy.yaml', 'tests')) self.assertEqual(policy['exposures.raw.template'], 'foo/bar/baz.fits.gz') def testDumpLoad(self): self.policy.dumpToFile('testDumpFile.yaml') loadedPolicy = Policy(filePath='testDumpFile.yaml') self.assertEqual(self.policy, loadedPolicy) os.remove('testDumpFile.yaml') def testNonExistantPolicyAtPath(self): self.assertRaises(IOError, Policy, filePath="c:/policy.yaml") self.assertRaises(IOError, Policy, filePath="c:/policy.paf") self.assertRaises(IOError, Policy, filePath="c:/policy")