def _getFullPath(self, pathType=None, url=None, **pathParams):
"""Returns the full path of the file in the tree.
This method must be overridden by each subclass.
"""
# # check for public release
# is_public = 'DR' in self._release
# ismpl = 'MPL' in self._release
# path_release = self._release.lower() if is_public or ismpl else None
if not Path:
raise MarvinMissingDependency('sdss_access is not installed')
else:
path = Path(release=self._release)
try:
if url:
fullpath = path.url(pathType, **pathParams)
else:
fullpath = path.full(pathType, **pathParams)
except Exception as ee:
warnings.warn(
'sdss_access was not able to retrieve the full path of the file. '
'Error message is: {0}'.format(str(ee)), MarvinUserWarning)
fullpath = None
return fullpath
def getDapRedux(release=None):
''' Retrieve SAS url link to the DAP redux directory
Parameters:
release (str):
The release version of the data to download.
Defaults to Marvin config.release.
Returns:
dapredux (str):
The full redux path to the DAP MAPS
'''
if not Path:
raise MarvinError('sdss_access is not installed')
else:
sdss_path = Path()
release = release or marvin.config.release
drpver, dapver = marvin.config.lookUpVersions(release=release)
# hack a url version of MANGA_SPECTRO_ANALYSIS
dapdefault = sdss_path.dir('mangadefault',
drpver=drpver,
dapver=dapver,
plate=None,
ifu=None)
dappath = dapdefault.rsplit('/', 2)[0]
dapredux = sdss_path.url('', full=dappath)
return dapredux
def _initCubes(self):
''' Initialize a list of Marvin Cube objects '''
_cubes = [None]
if self.data_origin == 'file':
sdss_path = Path(release=self.release)
if self.dir3d == 'stack':
cubes = sdss_path.expand('mangacube', drpver=self._drpver,
plate=self.plateid, ifu='*', wave='LOG')
else:
cubes = sdss_path.expand('mangamastar', drpver=self._drpver,
plate=self.plateid, ifu='*', wave='LOG')
_cubes = [Cube(filename=cube, mode=self.mode, release=self.release) for cube in cubes]
elif self.data_origin == 'db':
_cubes = [Cube(plateifu=cube.plateifu, mode=self.mode, release=self.release)
for cube in self._plate.cubes]
elif self.data_origin == 'api':
routeparams = {'plateid': self.plateid}
url = config.urlmap['api']['getPlateCubes']['url'].format(**routeparams)
# Make the API call
response = self._toolInteraction(url)
data = response.getData()
plateifus = data['plateifus']
_cubes = [Cube(plateifu=pifu, mode=self.mode, release=self.release) for pifu in plateifus]
FuzzyList.__init__(self, _cubes)
self.mapper = (lambda e: e.plateifu)
def getDefaultMapPath(**kwargs):
''' Retrieve the default Maps path
Uses sdss_access Path to generate a url download link to the
default MAPS file for a given MPL.
Parameters:
release (str):
The release version of the data to download.
Defaults to Marvin config.release.
plate (int):
The plate id
ifu (int):
The ifu number
bintype (str):
The bintype of the default file to grab. Defaults to MAPS
daptype (str):
The daptype of the default map to grab. Defaults to SPX-MILESHC
Returns:
maplink (str):
The sas url to download the default maps file
'''
if not Path:
raise MarvinError('sdss_access is not installed')
else:
sdss_path = Path()
# Get kwargs
release = kwargs.get('release', marvin.config.release)
drpver, dapver = marvin.config.lookUpVersions(release=release)
plate = kwargs.get('plate', None)
ifu = kwargs.get('ifu', None)
daptype = kwargs.get('daptype', 'SPX-GAU-MILESHC')
bintype = kwargs.get('bintype', 'MAPS')
# get the sdss_path name by MPL
# TODO: this is likely to break in future MPL/DRs. Just a heads up.
if '4' in release:
name = 'mangadefault'
elif '5' in release:
name = 'mangadap5'
else:
return None
# construct the url link to default maps file
maplink = sdss_path.url(name,
drpver=drpver,
dapver=dapver,
mpl=release,
plate=plate,
ifu=ifu,
daptype=daptype,
mode=bintype)
return maplink
def initDynamic(self):
''' Route to run when the dynamic toggle is initialized
This creates the web spectrum and dap heatmaps
'''
# get the form parameters
args = av.manual_parse(self, request, use_params='galaxy', required='plateifu')
# turning toggle on
current_session['toggleon'] = args.get('toggleon')
# get the cube
cubeinputs = {'plateifu': args.get('plateifu'), 'release': self._release}
cube = Cube(**cubeinputs)
output = {'specstatus': -1, 'mapstatus': -1}
# get web spectrum
webspec, specmsg = getWebSpectrum(cube, cube.ra, cube.dec, byradec=True)
daplist = get_dap_maplist(self._dapver, web=True)
dapdefaults = get_default_mapset(self._dapver)
# build the uber map dictionary
try:
mapdict = buildMapDict(cube, dapdefaults, self._dapver)
mapmsg = None
except Exception as e:
mapdict = [{'data': None, 'msg': 'Error', 'plotparams': None} for m in dapdefaults]
mapmsg = 'Error getting maps: {0}'.format(e)
else:
output['mapstatus'] = 1
if not webspec:
output['error'] = 'Error: {0}'.format(specmsg)
else:
output['specstatus'] = 1
sdss_path = Path()
output['image'] = sdss_path.url('mangaimage', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu, dir3d=cube.dir3d)
output['spectra'] = webspec
output['specmsg'] = specmsg
output['maps'] = mapdict
output['mapmsg'] = mapmsg
output['dapmaps'] = daplist
output['dapmapselect'] = dapdefaults
dm = datamodel[self._dapver]
output['dapbintemps'] = dm.get_bintemps(db_only=True)
current_session['bintemp'] = '{0}-{1}'.format(dm.get_bintype(), dm.get_template())
# try to jsonify the result
try:
jsonout = jsonify(result=output)
except Exception as e:
jsonout = jsonify(result={'specstatus': -1, 'mapstatus': -1, 'error': '{0}'.format(e)})
return jsonout
def _checkFilename(self):
''' Checks the filename for a proper FITS file '''
# if filename is not FITS, then try to load one
if 'fits' not in self.filename.lower():
if not Path:
raise MarvinError('sdss_access is not installed')
else:
# is_public = 'DR' in self._release
# path_release = self._release.lower() if is_public else None
sdss_path = Path(release=self._release)
# try a cube
full = sdss_path.full('mangacube', drpver=self._drpver, plate=self.plateid, ifu='*', wave='LOG')
cubeexists = sdss_path.any('', full=full)
if cubeexists:
file = sdss_path.one('', full=full)
else:
# try an rss
full = sdss_path.full('mangarss', drpver=self._drpver, plate=self.plateid, ifu='*', wave='LOG')
rssexists = sdss_path.any('', full=full)
if rssexists:
file = sdss_path.one('', full=full)
else:
file = None
# load the file
if file:
self.filename = file
else:
self.filename = None
async def get_path_kwargs(path: Type[PathModel] = Depends(extract_path)):
""" Get a list of input keyword arguments
Given an sdss_access path name, get the list of input keywords needed
to construct the full path.
Parameters
----------
name : str
a sdss_access path name
Returns
-------
A dict of path name and list of string keywords
"""
p = Path()
return {'name': path.name, 'kwargs': p.lookup_keys(path.name)}
def _getFullPath(self, pathType=None, url=None, **pathParams):
"""Returns the full path of the file in the tree."""
if not Path:
raise MarvinMissingDependency('sdss_access is not installed')
else:
try:
if url:
fullpath = Path().url(pathType, **pathParams)
else:
fullpath = Path().full(pathType, **pathParams)
except Exception as ee:
warnings.warn(
'sdss_access was not able to retrieve the full path of the file. '
'Error message is: {0}'.format(str(ee)), MarvinUserWarning)
fullpath = None
return fullpath
def set_filepaths(self, pathtype='full'):
"""Set the paths for cube, maps, etc."""
self.path = Path()
if check_versions(self.drpver, 'v2_5_3'):
self.imgpath = self.path.__getattribute__(pathtype)('mangaimagenew', **self.access_kwargs)
else:
self.imgpath = self.path.__getattribute__(pathtype)('mangaimage', **self.access_kwargs)
self.cubepath = self.path.__getattribute__(pathtype)('mangacube', **self.access_kwargs)
self.rsspath = self.path.__getattribute__(pathtype)('mangarss', **self.access_kwargs)
if self.release == 'MPL-4':
self.mapspath = self.path.__getattribute__(pathtype)('mangamap', **self.access_kwargs)
self.modelpath = None
else:
self.access_kwargs.pop('mode')
self.mapspath = self.path.__getattribute__(pathtype)('mangadap5', mode='MAPS',
**self.access_kwargs)
self.modelpath = self.path.__getattribute__(pathtype)('mangadap5', mode='LOGCUBE',
**self.access_kwargs)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
p = Path()
self.template = p.templates[self.name]
if self.kwargs:
self.full = p.full(self.name, **self.kwargs)
self.url = p.url(self.name, **self.kwargs)
self.file = p.name(self.name, **self.kwargs)
self.location = p.location(self.name, **self.kwargs)
self.exists = p.exists(self.name, **self.kwargs)
def good_kwargs(cls, v, values):
name = values.get('name')
keys = set(Path().lookup_keys(name))
valid = set(v) & set(keys)
if not valid:
return None
missing = set(keys) - set(v)
if missing:
mstr = ', '.join(missing)
raise ValueError(
f'Validation error: Missing kwargs {mstr} for name: {name}')
return v
def _getCube(name, use_file=False, release=None, **kwargs):
''' Retrieve a cube using marvin tools '''
drpver, __ = config.lookUpVersions(release)
cube = None
results = {}
# parse name into either mangaid or plateifu
try:
idtype = parseIdentifier(name)
except Exception as ee:
results['error'] = 'Failed to parse input name {0}: {1}'.format(name, str(ee))
return cube, results
filename = None
plateifu = None
mangaid = None
try:
if use_file:
if idtype == 'mangaid':
plate, ifu = mangaid2plateifu(name, drpver=drpver)
elif idtype == 'plateifu':
plate, ifu = name.split('-')
if Path is not None:
filename = Path(release=release).full('mangacube', ifu=ifu, plate=plate, drpver=drpver, wave='LOG')
assert os.path.exists(filename), 'file not found.'
else:
raise MarvinError('cannot create path for MaNGA cube.')
else:
if idtype == 'plateifu':
plateifu = name
elif idtype == 'mangaid':
mangaid = name
else:
raise MarvinError('invalid plateifu or mangaid: {0}'.format(idtype))
cube = Cube(filename=filename, mangaid=mangaid, plateifu=plateifu,
mode='local', release=release)
results['status'] = 1
except Exception as ee:
results['error'] = 'Failed to retrieve cube {0}: {1}'.format(name, str(ee))
return cube, results
def _getRSS(name, use_file=True, release=None, **kwargs):
"""Retrieves a RSS Marvin object."""
drpver, __ = marvin.config.lookUpVersions(release)
rss = None
results = {}
# parse name into either mangaid or plateifu
try:
idtype = parseIdentifier(name)
except Exception as ee:
results['error'] = 'Failed to parse input name {0}: {1}'.format(name, str(ee))
return rss, results
filename = None
plateifu = None
mangaid = None
try:
if use_file:
if idtype == 'mangaid':
plate, ifu = mangaid2plateifu(name, drpver=drpver)
elif idtype == 'plateifu':
plate, ifu = name.split('-')
if Path is not None:
filename = Path().full('mangarss', ifu=ifu, plate=plate, drpver=drpver)
assert os.path.exists(filename), 'file not found.'
else:
raise MarvinError('cannot create path for MaNGA rss.')
else:
if idtype == 'plateifu':
plateifu = name
elif idtype == 'mangaid':
mangaid = name
else:
raise MarvinError('invalid plateifu or mangaid: {0}'.format(idtype))
rss = marvin.tools.RSS(filename=filename, mangaid=mangaid, plateifu=plateifu,
mode='local', release=release)
results['status'] = 1
except Exception as ee:
results['error'] = 'Failed to retrieve RSS {0}: {1}'.format(name, str(ee))
return rss, results
def test_public(self):
path = Path()
url = path.url('mangacube', drpver='v2_3_1', plate=8485, ifu='1901')
assert 'mangawork' in url
release = 'dr14'
path = Path(public=True, release=release)
url = path.url('mangacube', drpver='v2_3_1', plate=8485, ifu='1901')
assert release in url
def _get_model_cube(name, use_file=False, release=None, **kwargs):
"""Retrieves a Marvin ModelCube object."""
model_cube = None
results = {}
drpver, dapver = config.lookUpVersions(release)
# parse name into either mangaid or plateifu
try:
idtype = parseIdentifier(name)
except Exception as err:
results['error'] = 'Failed to parse input name {0}: {1}'.format(
name, str(err))
return model_cube, results
filename = None
plateifu = None
mangaid = None
bintype = kwargs.pop('bintype')
template = kwargs.pop('template')
try:
if use_file:
if idtype == 'mangaid':
plate, ifu = mangaid2plateifu(name, drpver=drpver)
elif idtype == 'plateifu':
plate, ifu = name.split('-')
if Path is not None:
daptype = '{0}-{1}'.format(bintype, template)
filename = Path().full('mangadap5',
ifu=ifu,
drpver=drpver,
dapver=dapver,
plate=plate,
mode='LOGCUBE',
daptype=daptype)
assert os.path.exists(filename), 'file not found.'
else:
raise MarvinError('cannot create path for MaNGA cube.')
else:
if idtype == 'plateifu':
plateifu = name
elif idtype == 'mangaid':
mangaid = name
else:
raise MarvinError(
'invalid plateifu or mangaid: {0}'.format(idtype))
model_cube = ModelCube(filename=filename,
mangaid=mangaid,
plateifu=plateifu,
release=release,
template=template,
bintype=bintype,
**kwargs)
results['status'] = 1
except Exception as err:
results['error'] = 'Failed to retrieve ModelCube {0}: {1}'.format(
name, str(err))
return model_cube, results
class Galaxy(object):
"""An example galaxy for Marvin-tools testing."""
sasbasedir = os.getenv('SAS_BASE_DIR')
mangaredux = os.getenv('MANGA_SPECTRO_REDUX')
mangaanalysis = os.getenv('MANGA_SPECTRO_ANALYSIS')
dir3d = 'stack'
def __init__(self, plateifu):
"""Initialize plate and ifu."""
self.plateifu = plateifu
self.plate, self.ifu = self.plateifu.split('-')
self.plate = int(self.plate)
def set_galaxy_data(self, data_origin=None):
"""Set galaxy properties from the configuration file."""
if self.plateifu not in galaxy_data:
return
data = copy.deepcopy(galaxy_data[self.plateifu])
for key in data.keys():
setattr(self, key, data[key])
# sets specfic data per release
releasedata = self.releasedata[self.release]
for key in releasedata.keys():
setattr(self, key, releasedata[key])
# remap NSA drpall names for MPL-4 vs 5+
drpcopy = self.nsa_data['drpall'].copy()
for key, val in self.nsa_data['drpall'].items():
if isinstance(val, list):
newval, newkey = drpcopy.pop(key)
if self.release == 'MPL-4':
drpcopy[newkey] = newval
else:
drpcopy[key] = newval
self.nsa_data['drpall'] = drpcopy
def set_params(self, bintype=None, template=None, release=None):
"""Set bintype, template, etc."""
self.release = release
self.drpver, self.dapver = config.lookUpVersions(self.release)
self.drpall = 'drpall-{0}.fits'.format(self.drpver)
self.bintype = datamodel[self.dapver].get_bintype(bintype)
self.template = datamodel[self.dapver].get_template(template)
self.bintemp = '{0}-{1}'.format(self.bintype.name, self.template.name)
if release == 'MPL-4':
self.niter = int('{0}{1}'.format(self.template.n, self.bintype.n))
else:
self.niter = '*'
self.access_kwargs = {
'plate': self.plate,
'ifu': self.ifu,
'drpver': self.drpver,
'dapver': self.dapver,
'dir3d': self.dir3d,
'mpl': self.release,
'bintype': self.bintype.name,
'n': self.niter,
'mode': '*',
'daptype': self.bintemp
}
def set_filepaths(self, pathtype='full'):
"""Set the paths for cube, maps, etc."""
self.path = Path()
self.imgpath = self.path.__getattribute__(pathtype)(
'mangaimage', **self.access_kwargs)
self.cubepath = self.path.__getattribute__(pathtype)(
'mangacube', **self.access_kwargs)
self.rsspath = self.path.__getattribute__(pathtype)(
'mangarss', **self.access_kwargs)
if self.release == 'MPL-4':
self.mapspath = self.path.__getattribute__(pathtype)(
'mangamap', **self.access_kwargs)
self.modelpath = None
else:
self.access_kwargs.pop('mode')
self.mapspath = self.path.__getattribute__(pathtype)(
'mangadap5', mode='MAPS', **self.access_kwargs)
self.modelpath = self.path.__getattribute__(pathtype)(
'mangadap5', mode='LOGCUBE', **self.access_kwargs)
def get_location(self, path):
"""Extract the location from the input path."""
return self.path.location("", full=path)
def partition_path(self, path):
"""Partition the path into non-redux/analysis parts."""
endredux = path.partition(self.mangaredux)[-1]
endanalysis = path.partition(self.mangaanalysis)[-1]
end = (endredux or endanalysis)
return end
def new_path(self, name, newvar):
''' Sets a new path with the subsituted name '''
access_copy = self.access_kwargs.copy()
access_copy['mode'] = '*'
access_copy.update(**newvar)
if name == 'maps':
access_copy['mode'] = 'MAPS'
name = 'mangamap' if self.release == 'MPL-4' else 'mangadap5'
elif name == 'modelcube':
access_copy['mode'] = 'LOGCUBE'
name = None if self.release == 'MPL-4' else 'mangadap5'
path = self.path.full(name, **access_copy) if name else None
return path
def get_path(path_id, **kwargs):
"""A wrapper around sdss_access.path.Path."""
return Path().full(path_id, **kwargs)
def get(self, galid):
''' Retrieve info for a given cube '''
# determine type of galid
args = av.manual_parse(self, request, use_params='galaxy')
self.galaxy['id'] = args['galid']
idtype = parseIdentifier(galid)
if idtype in ['plateifu', 'mangaid']:
# set plateifu or mangaid
self.galaxy['idtype'] = idtype
galaxyid = {self.galaxy['idtype']: galid, 'release': self._release}
# Get cube
try:
cube = Cube(**galaxyid)
except MarvinError as e:
self.galaxy['cube'] = None
self.galaxy['error'] = 'MarvinError: {0}'.format(e)
return render_template("galaxy.html", **self.galaxy)
else:
self.galaxy['cube'] = cube
self.galaxy['daplink'] = getDapRedux(release=self._release)
# get SAS url links to cube, rss, maps, image
if Path:
sdss_path = Path()
self.galaxy['image'] = sdss_path.url('mangaimage', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu, dir3d=cube.dir3d)
cubelink = sdss_path.url('mangacube', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu)
rsslink = sdss_path.url('mangarss', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu)
maplink = getDefaultMapPath(release=self._release, plate=cube.plate, ifu=cube.ifu, daptype='SPX-GAU-MILESHC', mode='MAPS')
self.galaxy['links'] = {'cube': cubelink, 'rss': rsslink, 'map': maplink}
else:
self.galaxy['image'] = cube.data.image
# Get the initial spectrum
if cube:
daplist = get_dap_maplist(self._dapver, web=True)
dapdefaults = get_default_mapset(self._dapver)
self.galaxy['cube'] = cube
self.galaxy['toggleon'] = current_session.get('toggleon', 'false')
self.galaxy['cubehdr'] = cube.header
self.galaxy['quality'] = ('DRP3QUAL', cube.quality_flag.mask, cube.quality_flag.labels)
self.galaxy['mngtarget'] = {'bits': [it.mask for it in cube.target_flags if it.mask != 0],
'labels': [it.labels for it in cube.target_flags if len(it.labels) > 0],
'names': [''.join(('MNGTARG', it.name[-1])) for it in cube.target_flags if it.mask != 0]}
# make the nsa dictionary
hasnsa = cube.nsa is not None
self.galaxy['hasnsa'] = hasnsa
if hasnsa:
cols = self.galaxy.get('nsaplotcols')
nsadict, nsacols = make_nsa_dict(cube.nsa)
nsatmp = [nsacols.pop(nsacols.index(i)) for i in cols]
nsatmp.extend(nsacols)
self.galaxy['nsacols'] = nsatmp
self.galaxy['nsadict'] = nsadict
self.galaxy['dapmaps'] = daplist
self.galaxy['dapmapselect'] = dapdefaults
dm = datamodel[self._dapver]
self.galaxy['dapbintemps'] = dm.get_bintemps(db_only=True)
current_session['bintemp'] = '{0}-{1}'.format(dm.get_bintype(), dm.get_template())
# TODO - make this general - see also search.py for querystr
self.galaxy['cubestr'] = ("<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># access the header<br>cube.header<br># get NSA data<br>\
cube.nsa<br></samp></html>".format(cube.plateifu))
self.galaxy['spaxelstr'] = ("<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># get a spaxel<br>spaxel=cube[16, 16]<br>flux = \
spaxel.flux<br>wave = flux.wavelength<br>ivar = flux.ivar<br>mask = \
flux.mask<br>flux.plot()<br></samp></html>".format(cube.plateifu))
self.galaxy['mapstr'] = ("<html><samp>from marvin.tools.maps import Maps<br>maps = \
Maps(plateifu='{0}')<br>print(maps)<br># get an emission \
line map<br>haflux = maps.emline_gflux_ha_6564<br>values = \
haflux.value<br>ivar = haflux.ivar<br>mask = haflux.mask<br>haflux.plot()<br>\
</samp></html>".format(cube.plateifu))
else:
self.galaxy['error'] = 'Error: Galaxy ID {0} must either be a Plate-IFU, or MaNGA-Id designation.'.format(galid)
return render_template("galaxy.html", **self.galaxy)
return render_template("galaxy.html", **self.galaxy)
def path():
''' Fixture to create a generic Path object '''
path = Path()
yield path
path = None
def is_name(cls, v):
if v not in Path().lookup_names():
raise ValueError(
f'Validation error: path name {v} not a valid sdss_access name'
)
return v
def get(self, galid):
''' Retrieve info for a given cube '''
# determine type of galid
args = av.manual_parse(self, request, use_params='galaxy')
self.galaxy['id'] = args['galid']
self.galaxy['latest_dr'] = self._release.lower(
) if 'DR' in self._release else marvin.config._get_latest_release(
dr_only=True).lower()
idtype = parseIdentifier(galid)
if idtype in ['plateifu', 'mangaid']:
# set plateifu or mangaid
self.galaxy['idtype'] = idtype
galaxyid = {self.galaxy['idtype']: galid, 'release': self._release}
# Get cube
try:
cube = Cube(**galaxyid)
except MarvinError as e:
self.galaxy['cube'] = None
errmsg = 'MarvinError: {0}'.format(e)
# check target status and fine-tune the error message
if idtype == 'mangaid':
status = target_status(galid, drpver=self._drpver)
if status == 'not yet observed':
errmsg = '{0} is a valid target but has not yet been observed'.format(
galid)
elif status == 'not valid target':
errmsg = '{0} is not valid MaNGA target. Check your syntax'.format(
galid)
self.galaxy['error'] = errmsg
return render_template("galaxy.html", **self.galaxy)
else:
dm = datamodel[self._dapver]
self.galaxy['cube'] = cube
self.galaxy['daplink'] = getDapRedux(release=self._release)
# get SAS url links to cube, rss, maps, image
if Path:
is_public = 'DR' in self._release
path_release = self._release.lower() if is_public else None
sdss_path = Path(public=is_public, release=path_release)
self.galaxy['image'] = cube.getImage().url
cubelink = sdss_path.url('mangacube',
drpver=cube._drpver,
plate=cube.plate,
ifu=cube.ifu)
rsslink = sdss_path.url('mangarss',
drpver=cube._drpver,
plate=cube.plate,
ifu=cube.ifu)
daptype = "{0}-{1}".format(dm.default_bintype,
dm.default_template)
maplink = getDefaultMapPath(release=self._release,
plate=cube.plate,
ifu=cube.ifu,
daptype=daptype,
mode='MAPS')
mclink = getDefaultMapPath(release=self._release,
plate=cube.plate,
ifu=cube.ifu,
daptype=daptype,
mode='LOGCUBE')
self.galaxy['links'] = {
'cube': cubelink,
'rss': rsslink,
'map': maplink,
'mc': mclink
}
else:
self.galaxy['image'] = cube.data.image
# Get the initial spectrum
if cube:
dm = datamodel[self._dapver]
daplist = [p.full(web=True) for p in dm.properties]
dapdefaults = dm.get_default_mapset()
self.galaxy['cube'] = cube
self.galaxy['toggleon'] = current_session.get(
'toggleon', 'false')
self.galaxy['cubehdr'] = cube.header
self.galaxy['quality'] = ('DRP3QUAL', cube.quality_flag.mask,
cube.quality_flag.labels)
self.galaxy['mngtarget'] = {
'bits':
[it.mask for it in cube.target_flags if it.mask != 0],
'labels': [
it.labels for it in cube.target_flags
if len(it.labels) > 0
],
'names': [
''.join(('MNGTARG', it.name[-1]))
for it in cube.target_flags if it.mask != 0
]
}
# make the nsa dictionary
hasnsa = cube.nsa is not None
self.galaxy['hasnsa'] = hasnsa
if hasnsa:
cols = self.galaxy.get('nsaplotcols')
nsadict, nsacols = make_nsa_dict(cube.nsa)
nsatmp = [
nsacols.pop(nsacols.index(i)) for i in cols
if i in nsacols
]
nsatmp.extend(nsacols)
self.galaxy['nsacols'] = nsatmp
self.galaxy['nsadict'] = nsadict
self.galaxy['dapmaps'] = daplist
self.galaxy['dapmapselect'] = current_session.get(
'selected_dapmaps', dapdefaults)
dm = datamodel[self._dapver]
self.galaxy['dapbintemps'] = dm.get_bintemps(db_only=True)
if 'bintemp' not in current_session or current_session[
'bintemp'] not in self.galaxy['dapbintemps']:
current_session['bintemp'] = '{0}-{1}'.format(
dm.get_bintype(), dm.get_template())
# get default map quality
maps = cube.getMaps(
plateifu=cube.plateifu,
mode='local',
bintype=current_session['bintemp'].split('-')[0])
mapqual = ('DAPQUAL', maps.quality_flag.mask,
maps.quality_flag.labels)
self.galaxy['mapquality'] = mapqual
# TODO - make this general - see also search.py for querystr
self.galaxy['cubestr'] = (
"<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># access the header<br>cube.header<br># get NSA data<br>\
cube.nsa<br></samp></html>".format(cube.plateifu))
self.galaxy['spaxelstr'] = (
"<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># get a spaxel by slicing cube[i,j]<br>spaxel=cube[16, 16]<br>flux = \
spaxel.flux<br>wave = flux.wavelength<br>ivar = flux.ivar<br>mask = \
flux.mask<br>flux.plot()<br></samp></html>".format(
cube.plateifu))
self.galaxy['mapstr'] = (
"<html><samp>from marvin.tools.maps import Maps<br>maps = \
Maps(plateifu='{0}')<br>print(maps)<br># get an emission \
line map<br>haflux = maps.emline_gflux_ha_6564<br>values = \
haflux.value<br>ivar = haflux.ivar<br>mask = haflux.mask<br>haflux.plot()<br>\
</samp></html>".format(cube.plateifu))
else:
self.galaxy[
'error'] = 'Error: Galaxy ID {0} must either be a Plate-IFU, or MaNGA-Id designation.'.format(
galid)
return render_template("galaxy.html", **self.galaxy)
return render_template("galaxy.html", **self.galaxy)
async def get_paths():
""" Get a list of sdss_access path names """
p = Path()
return {'names': list(p.lookup_names())}
class FileObject(BaseObject):
path = Path()
def __init__(self, inputs=None, filename=None, **kwargs):
product = kwargs.pop('product', None)
version = kwargs.pop('version', None)
super(FileObject, self).__init__(product=product, version=version)
self.path.replant_tree(str(version))
self.filename = filename
self.path_name = kwargs.pop('path_name', None)
self.parent = kwargs.pop('parent', None)
self._info = None
self._stats = None
self._changes = None
self.loaded = False
# check inputs and produce filename
self._determine_inputs(inputs, **kwargs)
self._parse_filename()
def _determine_inputs(self, inputs, **kwargs):
''' Determine the input '''
# use filename if found instead of inputs
if inputs is None and self.filename:
return
# no input at all
if not inputs and self.path_name is None:
raise ValueError(
'No input has been provided to determine a valid file')
# check if inputs is a pathlib.Path
if isinstance(inputs, pathlib.Path):
inputs = str(inputs)
assert isinstance(inputs, six.string_types), 'input must be a string'
# check if input is an sdss_access Path
if inputs in self.path.lookup_names():
self.path_name = inputs
keys = self.path.lookup_keys(inputs)
missing = set(keys) - set(kwargs.keys())
if missing:
raise KeyError('Input sdss_access name missing '
'necessary kwargs: {0}'.format(
', '.join(missing)))
# get the full path
self.filename = self.path.full(inputs, **kwargs)
else:
# assume it is a filename
self.filename = inputs
def _parse_filename(self):
''' Parse a filename into components '''
path = pathlib.Path(self.filename)
self.filepath = path.parent
self.filename = path.name
self.fullpath = self.filepath / self.filename
self.file_exists = self.path.exists('', full=self.fullpath)
# if not self.file_exists:
# raise NameError('{0} not a valid file'.format(self.fullpath))
@staticmethod
def _get_example(example, replace=None):
''' set the example '''
ex_regex = r'^(\w+work|dr\d{1,2})'
assert re.match(ex_regex,
example), 'example must start work xxxwork or drxx'
if replace and 'dr' in str(replace).lower():
example = re.sub(ex_regex, str(replace).lower(), example)
path = pathlib.Path(os.path.expandvars('$SAS_BASE_DIR')) / example
return path
@classmethod
def from_example(cls, example, **kwargs):
''' Instantiates a file from an example filepath'''
version = kwargs.get('version', None)
path = cls._get_example(example, replace=version)
# if not path.is_file():
# raise ValueError(f'Example provided does seem to exist! Check again.')
return cls(path, **kwargs)
@classmethod
def from_path(cls, path_name, example=None, **kwargs):
''' Instantiates a file from an sdss_access path definition '''
path_kwargs = kwargs.pop('path_kwargs', None)
version = kwargs.get('version', None)
cls.path.replant_tree(str(version) if version else None)
if example:
path = cls._get_example(example, replace=version)
args = cls.path.extract(path_name, path)
kwargs.update(args)
elif path_kwargs:
# to handle sdss_access path kwargs and versioning issues
# TODO cleanup version handling to better handle path_kwarg inputs
args = path_kwargs.copy()
missing = set(cls.path.lookup_keys(path_name)) - set(args.keys())
if version:
if isinstance(version, six.string_types):
version_kwarg = dict.fromkeys(missing, version)
else:
version_kwarg = version.__dict__
args.update(version_kwarg)
kwargs.update(args)
else:
raise ValueError(
'no example string or sdss_access path kwargs found. Cannot construct object.'
)
return cls(path_name, **kwargs)
def compute_changelog(self, otherfile=None):
assert otherfile is not None, (
'You must specify another file to view '
'the changlog with this file. Otherwise '
'use the changlog from the datamodel to '
'see the full changelog')
if not self._changes:
name = str(self.fullpath)
self._changes = compute_diff(name, otherfile, change='fits')
return self._changes
# -*- coding: utf-8 -*-
#
# Filename: access.py
# Project: io
# Author: Brian Cherinka
# Created: Monday, 7th September 2020 1:41:59 pm
# License: BSD 3-clause "New" or "Revised" License
# Copyright (c) 2020 Brian Cherinka
# Last Modified: Monday, 7th September 2020 1:42:00 pm
# Modified By: Brian Cherinka
from __future__ import print_function, division, absolute_import
from sdss_access.path import Path
from astropy.io import fits
path = Path(public=True, release='DR15')
def get_path(name, ptype='full', **kwargs):
''' get an sdss_access local filepath '''
if ptype == 'full':
return path.full(name, **kwargs)
elif ptype == 'name':
return path.name(name, **kwargs)
elif ptype == 'url':
return path.url(name, **kwargs)
else:
return path.full(name, **kwargs)
def get_cube(plate=8485, ifu=1901, drpver='v2_4_3'):
def loadFibres(self, pM, replace=False):
"""Loads fibres from the plateHoles file for this plPlugMapM."""
if self.verbose:
print('Populating the fiber table for APOGEE plate ... ')
plateHolesName = Path().name('plateHoles', plateid=self._getPlateID())
# Make sure the plateholes file exists in the plateholes table
# It needs to be added before the fiber table can be populated
session = Session()
with session.begin(subtransactions=True):
try:
session.query(plateDB.PlateHolesFile).filter(
plateDB.PlateHolesFile.filename == plateHolesName).one()
except NoResultFound:
raise PlPlugMapMFileException(
'Error: The plateHoles file \'{0}\' was not found in the '
'database for the plPlugMapM file \'{1}\' '
'Consequently, the \'fiber\' table cannot be populated. '
'(Was the script to load the plate holes table run?)'
.format(plateHolesName, pM.filename))
except MultipleResultsFound:
# This should really never happen. If it does, fix by adding a
# unique contraint to the filename field of 'plate_holes_file'.
raise PlPlugMapMFileException(
'Error: More than one entry in the plate_holes_file table '
'was found for the filename: \'{0}\''.format(pM.filename))
# The plPlugMapM records should not have any fibre associated. We check
# it. If there are, we raise an error except if replace=True.
if len(pM.fibers) > 0:
if replace is False:
raise PlPlugMapMFileException(
'The PlPlugMapM record for filename={0} has fibres '
'associated. This should not happen unless you are '
'overwriting the records. You can try again with '
'replace=True'.format(pM.filename))
else:
warnings.warn('removing {0} fibres for PlPlugMapM with '
'filename={1}'.format(len(pM.fibers),
pM.filename), UserWarning)
with session.begin(subtransactions=True):
fibers = session.query(plateDB.Fiber).filter(
plateDB.Fiber.pl_plugmap_m_pk == pM.pk)
fibers.delete()
session.flush()
# When data is stored in postgres database, a number like 25.9983
# might be stored as 25.998300001 or 25.99829999 so the (rounded)
# values queried from the DB might not match exactly the (precise)
# values from the pl_plugmap_m files. This method searches the database
# for the matching value within a +/- sigma window for both x-focal
# and y-focal. If sigma is very small, much smaller than the limit of
# how close two fibers can be together, only one match will be found.
plugMapObjs = self.data['PLUGMAPOBJ']
sigmaX = .00001
sigmaY = .00001
fibreData = []
# Creates and array of plateHoles
with session.begin(subtransactions=True):
ph_dbo = session.query(plateDB.PlateHole).join(
plateDB.PlateHolesFile).filter(
plateDB.PlateHolesFile.filename ==
plateHolesName).all()
plateHolesList = [(int(ph_dbo[ii].pk), float(ph_dbo[ii].xfocal),
float(ph_dbo[ii].yfocal))
for ii in range(len(ph_dbo))]
plateHolesArray = np.array(plateHolesList)
for (holeType, fiberId, spectrographId,
xFocal, yFocal) in zip(plugMapObjs['holeType'],
plugMapObjs['fiberId'],
plugMapObjs['spectrographId'],
plugMapObjs['xFocal'],
plugMapObjs['yFocal']):
if (holeType == 'OBJECT' and fiberId != -1 and
spectrographId == 2):
# Finds the appropriate hole
indices = np.where(
(plateHolesArray[:, 1] > (xFocal - sigmaX)) &
(plateHolesArray[:, 1] < (xFocal + sigmaX)) &
(plateHolesArray[:, 2] > (yFocal - sigmaY)) &
(plateHolesArray[:, 2] < (yFocal + sigmaY))
)
if len(indices) == 0:
raise PlPlugMapMFileException(
'Error: Problem filling fiber table for PlPlugMap {0}.'
' No result found for plate hole at {1}, {2} ... '
'exiting'.format(pM.filename, xFocal, yFocal))
elif len(indices) > 1:
raise PlPlugMapMFileException(
'Error: Problem filling fiber table for PlPlugMap {0}.'
' Multiple results found for plate hole at '
'{1}, {2} ... exiting'.format(pM.filename, xFocal,
yFocal))
plateHole = plateHolesArray[indices][0]
# We are going to use bulk insert, so for now we just add the
# data to a list.
fibreData.append({'fiber_id': int(fiberId),
'plate_hole_pk': plateHole[0],
'pl_plugmap_m_pk': pM.pk})
# Now we insert the data
if len(fibreData) > 0:
plateDB.db.engine.execute(
plateDB.Fiber.__table__.insert(), fibreData)