def setGlobalSession():
''' Sets the global session for Flask '''
mpls = list(config._mpldict.keys())
versions = [{
'name': mpl,
'subtext': str(config.lookUpVersions(release=mpl))
} for mpl in mpls]
current_session['versions'] = versions
if 'release' not in current_session:
current_session['release'] = config.release
drpver, dapver = config.lookUpVersions(release=config.release)
current_session['drpver'] = drpver
current_session['dapver'] = dapver
def _query_versions(self, mode='local', release=None):
release = config.release
drpver, dapver = config.lookUpVersions(release=release)
p = 'haflux > 25'
if '4' in release:
name = 'CleanSpaxelProp'
else:
name = 'CleanSpaxelProp{0}'.format(release.split('-')[1])
if release:
q = Query(searchfilter=p, mode=mode, release=release)
else:
q = Query(searchfilter=p, mode=mode)
self.assertEqual(q._drpver, drpver)
self.assertEqual(q._dapver, dapver)
self.assertEqual(q._release, release)
self.assertEqual(q.marvinform._release, release)
self.assertEqual(q.marvinform._param_form_lookup._release, release)
self.assertEqual(
q.marvinform._param_form_lookup['spaxelprop.file'].Meta.model.
__name__, name)
def setUp(self):
self._reset_the_config()
self.set_sasurl('local')
self.mode = self.init_mode
config.setMPL('MPL-4')
config.forceDbOn()
self.drpver, __ = config.lookUpVersions(release=config.release)
def update_allowed():
''' Update the allowed versions shown in the header dropdown '''
logged_in = current_session.get('loginready', None)
mpls = list(config._allowed_releases.keys())
# this is to remove MPLs from the list of web versions when not logged in to the collaboration site
if not logged_in:
# select only DR allowed releases
mpls = [mpl for mpl in mpls if 'DR' in mpl]
# select the release to switch to; prioritize the session release
if 'release' not in current_session:
release = config.release if 'DR' in config.release else max(mpls)
else:
release = current_session['release'] if 'DR' in current_session[
'release'] else max(mpls)
# update the session with the new release
set_session_versions(release)
versions = [{
'name': mpl,
'subtext': str(config.lookUpVersions(release=mpl))
} for mpl in mpls]
return versions
def test_summary_files(self):
config.setDefaultDrpAll() # need to reset the config
drp, dap = config.lookUpVersions()
assert config.drpall is not None
assert config.dapall is not None
assert 'drpall-{0}'.format(drp) in config.drpall
assert 'dapall-{0}-{1}'.format(drp, dap) in config.dapall
def set_params(self, bintype=None, template=None, release=None):
"""Set bintype, template, etc."""
self.release = release
self.drpver, self.dapver = config.lookUpVersions(self.release)
self.drpall = 'drpall-{0}.fits'.format(self.drpver)
self.bintype = datamodel[self.dapver].get_bintype(bintype)
self.template = datamodel[self.dapver].get_template(template)
self.bintemp = '{0}-{1}'.format(self.bintype.name, self.template.name)
if release == 'MPL-4':
self.niter = int('{0}{1}'.format(self.template.n, self.bintype.n))
else:
self.niter = '*'
self.access_kwargs = {
'plate': self.plate,
'ifu': self.ifu,
'drpver': self.drpver,
'dapver': self.dapver,
'dir3d': self.dir3d,
'mpl': self.release,
'bintype': self.bintype.name,
'n': self.niter,
'mode': '*',
'daptype': self.bintemp
}
def get_drpver(release=None):
''' Get the drpver from the input release from the request '''
if not release:
release = config.release
drpver, dapver = config.lookUpVersions(release)
return drpver
def update_allowed():
''' Update the allowed versions '''
mpls = list(config._allowed_releases.keys())
versions = [{
'name': mpl,
'subtext': str(config.lookUpVersions(release=mpl))
} for mpl in mpls]
return versions
def __init__(self,
input=None,
filename=None,
mangaid=None,
plateifu=None,
mode=None,
data=None,
release=None,
drpall=None,
download=None,
ignore_db=False):
self.data = data
self.data_origin = None
self._ignore_db = ignore_db
self.filename = filename
self.mangaid = mangaid
self.plateifu = plateifu
self.mode = mode if mode is not None else config.mode
self._release = release if release is not None else config.release
self._drpver, self._dapver = config.lookUpVersions(
release=self._release)
self._drpall = config._getDrpAllPath(
self._drpver) if drpall is None else drpall
self._forcedownload = download if download is not None else config.download
self._determine_inputs(input)
assert self.mode in ['auto', 'local', 'remote']
assert self.filename is not None or self.plateifu is not None, 'no inputs set.'
self.datamodel = None
self._set_datamodel()
if self.mode == 'local':
self._doLocal()
elif self.mode == 'remote':
self._doRemote()
elif self.mode == 'auto':
try:
self._doLocal()
except Exception as ee:
if self.filename:
# If the input contains a filename we don't want to go into remote mode.
raise (ee)
else:
warnings.warn('local mode failed. Trying remote now.',
MarvinUserWarning)
self._doRemote()
# Sanity check to make sure data_origin has been properly set.
assert self.data_origin in ['file', 'db',
'api'], 'data_origin is not properly set.'
def test_masked(self, maps_release_only):
__, dapver = config.lookUpVersions(maps_release_only.release)
params = get_default_plot_params(dapver)
ha = maps_release_only['emline_gflux_ha_6564']
expected = ha.pixmask.get_mask(params['default']['bitmasks'],
dtype=bool)
assert ha.masked.data == pytest.approx(ha.value)
assert (ha.masked.mask == expected).all()
def set_summary_file(self, release):
''' Sets the path to the Firefly summary file '''
# define the variables to build a unique path to your VAC file
# look up the MaNGA drpver from the release
drpver, dapver = config.lookUpVersions(release)
self.path_params = {'ver': self.version[release], 'drpver': drpver}
# get_path returns False if the files do not exist locally
self.summary_file = self.get_path('mangaffly',
path_params=self.path_params)
def test_cube_remote_drpver_differ_from_global(self, galaxy, monkeyconfig):
if galaxy.release == 'MPL-5':
pytest.skip('Skipping release for forced global MPL-5')
drpver, dapver = config.lookUpVersions(config.release)
assert config.release == 'MPL-5'
cube = Cube(plateifu=galaxy.plateifu,
mode='remote',
release=galaxy.release)
assert cube.release != config.release
assert cube._drpver != drpver
def selectmpl(self):
''' Global selection of the MPL/DR versions '''
args = av.manual_parse(self, request, use_params='index')
version = args.get('release', None)
current_session['release'] = version
drpver, dapver = config.lookUpVersions(release=version)
current_session['drpver'] = drpver
current_session['dapver'] = dapver
out = {'status': 1, 'msg': 'Success', 'current_release': version,
'current_drpver': drpver, 'current_dapver': dapver}
return jsonify(result=out)
def _getCube(name, use_file=False, release=None, **kwargs):
''' Retrieve a cube using marvin tools '''
drpver, __ = config.lookUpVersions(release)
cube = None
results = {}
# parse name into either mangaid or plateifu
try:
idtype = parseIdentifier(name)
except Exception as ee:
results['error'] = 'Failed to parse input name {0}: {1}'.format(name, str(ee))
return cube, results
filename = None
plateifu = None
mangaid = None
try:
if use_file:
if idtype == 'mangaid':
plate, ifu = mangaid2plateifu(name, drpver=drpver)
elif idtype == 'plateifu':
plate, ifu = name.split('-')
if Path is not None:
filename = Path().full('mangacube', ifu=ifu, plate=plate, drpver=drpver)
assert os.path.exists(filename), 'file not found.'
else:
raise MarvinError('cannot create path for MaNGA cube.')
else:
if idtype == 'plateifu':
plateifu = name
elif idtype == 'mangaid':
mangaid = name
else:
raise MarvinError('invalid plateifu or mangaid: {0}'.format(idtype))
cube = Cube(filename=filename, mangaid=mangaid, plateifu=plateifu,
mode='local', release=release)
results['status'] = 1
except Exception as ee:
results['error'] = 'Failed to retrieve cube {0}: {1}'.format(name, str(ee))
return cube, results
def test_format_use_masks(self, use_masks, mask, expected, set_release):
if config.release == 'MPL-4':
pytest.skip('Skip MPL-4.')
for prop in [
'stellar_vel', 'stellar_sigma', 'emline_gflux', 'specindex'
]:
params = datamodel[config.lookUpVersions()[1]].get_plot_params(
prop)
actual = mapplot._format_use_masks(
use_masks, mask, dapmap=None, default_masks=params['bitmasks'])
assert actual == expected
def updateGlobalSession():
''' updates the Marvin config with the global Flask session '''
# check if mpl versions in session
if 'versions' not in current_session:
setGlobalSession()
elif 'drpver' not in current_session or \
'dapver' not in current_session:
drpver, dapver = config.lookUpVersions(release=config.release)
current_session['drpver'] = drpver
current_session['dapver'] = dapver
elif 'release' not in current_session:
current_session['release'] = config.release
def test_format_use_masks_mpl4(self, use_masks, mask, expected,
set_release):
if config.release != 'MPL-4':
pytest.skip('Only include MPL-4.')
for prop in [
'stellar_vel', 'stellar_sigma', 'emline_gflux', 'specindex'
]:
params = get_plot_params(dapver=config.lookUpVersions()[1],
prop=prop)
actual = mapplot._format_use_masks(
use_masks, mask, dapmap=None, default_masks=params['bitmasks'])
assert actual == expected
def update_allowed():
''' Update the allowed versions '''
logged_in = current_session.get('loginready', None)
mpls = list(config._allowed_releases.keys())
# this is to remove MPLs from the list of web versions when not logged in to the collaboration site
if not logged_in:
mpls = [mpl for mpl in mpls if 'DR' in mpl]
set_session_versions(max(mpls))
versions = [{
'name': mpl,
'subtext': str(config.lookUpVersions(release=mpl))
} for mpl in mpls]
return versions
def create_args(self):
''' Build the final argument list for webargs validation '''
# get the dapver
drpver, self.dapver = config.lookUpVersions(self.release)
# reset the final args
self._reset_final_args()
# add view args to the list
self._add_view_args()
# add form param args to the list
if self.use_params:
self._add_param_args()
def add_data(rsync, release=None, plate=None, ifu=None, exclude=[]):
drpver, dapver = config.lookUpVersions(release)
if 'drpall' not in exclude:
rsync.add('drpall', drpver=drpver)
if 'mangacube' not in exclude:
rsync.add('mangacube', plate=plate, ifu=ifu, drpver=drpver)
if 'mangarss' not in exclude:
rsync.add('mangarss', plate=plate, ifu=ifu, drpver=drpver)
if 'mangaimage' not in exclude:
rsync.add('mangaimage',
plate=plate,
drpver=drpver,
dir3d='stack',
ifu='*')
if release in ['MPL-5', 'MPL-6']:
if 'mangadap5' not in exclude:
rsync.add('mangadap5',
plate=plate,
drpver=drpver,
dapver=dapver,
ifu=ifu,
daptype='*',
mode='*')
elif release == 'MPL-4':
if 'mangamap' not in exclude:
rsync.add('mangamap',
plate=plate,
drpver=drpver,
dapver=dapver,
ifu=ifu,
bintype='*',
mode='*',
n='**')
if 'mangadefault' not in exclude:
rsync.add('mangadefault',
plate=plate,
drpver=drpver,
dapver=dapver,
ifu=ifu)
return rsync
def test_pickling_data(self, temp_scratch, galaxy):
drpver, __ = config.lookUpVersions()
maps = Maps(filename=galaxy.mapspath)
modelcube = ModelCube(filename=galaxy.modelpath)
spaxel = maps.getSpaxel(25, 15, xyorig='lower', drp=False, models=modelcube)
file = temp_scratch.join('test_spaxel.mpf')
path_saved = spaxel.save(str(file), overwrite=True)
assert file.check() is True
assert os.path.exists(path_saved)
del spaxel
spaxel_restored = Spaxel.restore(str(file))
assert spaxel_restored.stellar_vel.value is not None
assert spaxel_restored.stellar_vel.bin.binid is not None
def _test_getspaxel(self, spaxel, galaxy):
spaxel_drpver, spaxel_dapver = config.lookUpVersions(spaxel.release)
assert spaxel_drpver == galaxy.drpver
assert spaxel_dapver == galaxy.dapver
assert spaxel.plateifu == galaxy.plateifu
assert spaxel.mangaid == galaxy.mangaid
assert spaxel.getModelCube() is not None
assert spaxel.getModelCube().bintype == galaxy.bintype
assert spaxel.getModelCube().template == galaxy.template
assert spaxel.template == galaxy.template
assert spaxel.template == galaxy.template
assert spaxel._parent_shape == tuple(galaxy.shape)
assert isinstance(spaxel.binned_flux, Spectrum)
assert isinstance(spaxel.full_fit, Spectrum)
assert isinstance(spaxel.emline_fit, Spectrum)
def test_pickling_all_api(self, temp_scratch, galaxy):
drpver, __ = config.lookUpVersions()
cube = Cube(plateifu=galaxy.plateifu, mode='remote')
maps = Maps(plateifu=galaxy.plateifu, mode='remote')
modelcube = ModelCube(plateifu=galaxy.plateifu, mode='remote')
spaxel = cube.getSpaxel(1, 3, maps=maps, modelcube=modelcube)
assert spaxel._cube.data_origin == 'api'
assert spaxel._maps.data_origin == 'api'
assert spaxel._modelcube.data_origin == 'api'
file = temp_scratch.join('test_spaxel_api.mpf')
path_saved = spaxel.save(str(file), overwrite=True)
assert file.check() is True
assert os.path.exists(path_saved)
del spaxel
spaxel_restored = Spaxel.restore(str(file))
assert spaxel_restored is not None
assert isinstance(spaxel_restored, Spaxel)
assert spaxel_restored._cube is not None
assert isinstance(spaxel_restored._cube, Cube)
assert spaxel_restored._cube.data_origin == 'api'
assert spaxel_restored._cube.data is None
assert spaxel_restored._cube.header['VERSDRP3'] == drpver
assert spaxel_restored._maps is not None
assert isinstance(spaxel_restored._maps, Maps)
assert spaxel_restored._maps.data_origin == 'api'
assert spaxel_restored._maps.data is None
assert spaxel_restored._modelcube is not None
assert isinstance(spaxel_restored._modelcube, ModelCube)
assert spaxel_restored._modelcube.data_origin == 'api'
assert spaxel_restored._modelcube.data is None
def get_drpall_row(plateifu, drpver=None, drpall=None):
"""Returns a dictionary from drpall matching the plateifu."""
from marvin import config
if drpall:
drpall_table = table.Table.read(drpall)
else:
config_drpver, __ = config.lookUpVersions()
drpver = drpver if drpver else config_drpver
if drpver not in drpTable:
drpall = drpall if drpall else config._getDrpAllPath(drpver=drpver)
drpTable[drpver] = table.Table.read(drpall)
drpall_table = drpTable[drpver]
row = drpall_table[drpall_table['plateifu'] == plateifu]
if len(row) != 1:
raise ValueError('{0} results found for {1} in drpall table'.format(
len(row), plateifu))
return row[0]
def setUpClass(cls):
config.use_sentry = False
config.add_github_message = False
# set initial config variables
cls.init_mode = config.mode
cls.init_sasurl = config.sasurl
cls.init_urlmap = config.urlmap
cls.init_xyorig = config.xyorig
cls.init_traceback = config._traceback
cls.init_keys = ['mode', 'sasurl', 'urlmap', 'xyorig', 'traceback']
# set db stuff
cls._marvindb = marvindb
cls.session = marvindb.session
# set paths
cls.sasbasedir = os.getenv("$SAS_BASE_DIR")
cls.mangaredux = os.getenv("MANGA_SPECTRO_REDUX")
cls.mangaanalysis = os.getenv("MANGA_SPECTRO_ANALYSIS")
# testing data for 8485-1901
cls.set_plateifu(plateifu='8485-1901')
cls.mangaid = '1-209232'
cls.cubepk = 10179
cls.ra = 232.544703894
cls.dec = 48.6902009334
cls.redshift = 0.0407447
cls.dir3d = 'stack'
cls.release = 'MPL-5'
cls.drpver, cls.dapver = config.lookUpVersions(cls.release)
cls.bintemp = _get_bintemps(cls.dapver, default=True)
cls.defaultbin, cls.defaulttemp = cls.bintemp.split('-', 1)
cls.cubename = 'manga-{0}-LOGCUBE.fits.gz'.format(cls.plateifu)
cls.rssname = 'manga-{0}-LOGRSS.fits.gz'.format(cls.plateifu)
cls.imgname = '{0}.png'.format(cls.ifu)
cls.mapsname = 'manga-{0}-MAPS-{1}.fits.gz'.format(cls.plateifu, cls.bintemp)
cls.modelname = 'manga-{0}-LOGCUBE-{1}.fits.gz'.format(cls.plateifu, cls.bintemp)
def load_mangadrp(plateifu, loglin='LOG', rsscube='CUBE', mpl_v=None):
"""retrieve local MaNGA logcube, downloading if necessary
Parameters
----------
plateifu : str
plateifu designation of galaxy
loglin : str, optional
wavelength scale (LIN/LOG)
rsscube : str, optional
RSS or CUBE
mpl_v : None or str, optional
MPL version
Returns
-------
fits.HDUList
HDUList with logcube data
"""
# resolve plateifu into filename
if mpl_v is None:
mpl_v = mconf.release
drpver, dapver = mconf.lookUpVersions(mpl_v)
sas_basedir = os.environ['SAS_BASE_DIR']
plate, ifu = plateifu.split('-')
fname = f'manga-{plateifu}-{loglin.upper()}{rsscube.upper()}.fits.gz'
fpath_full = os.path.join(
sas_basedir, 'mangawork/manga/spectro/redux',
drpver, plate, 'stack', fname)
if not os.path.isfile(fpath_full):
marvin_download([plateifu], dltype=rsscube.lower(), release=mpl_v)
return fits.open(fpath_full)
def bits(request, set_release):
params = datamodel[config.lookUpVersions()[1]].get_plot_params(
request.param)
return params['bitmasks']
def _get_model_cube(name, use_file=False, release=None, **kwargs):
"""Retrieves a Marvin ModelCube object."""
model_cube = None
results = {}
drpver, dapver = config.lookUpVersions(release)
# parse name into either mangaid or plateifu
try:
idtype = parseIdentifier(name)
except Exception as err:
results['error'] = 'Failed to parse input name {0}: {1}'.format(
name, str(err))
return model_cube, results
filename = None
plateifu = None
mangaid = None
bintype = kwargs.pop('bintype')
template = kwargs.pop('template')
try:
if use_file:
if idtype == 'mangaid':
plate, ifu = mangaid2plateifu(name, drpver=drpver)
elif idtype == 'plateifu':
plate, ifu = name.split('-')
if Path is not None:
daptype = '{0}-{1}'.format(bintype, template)
filename = Path().full('mangadap5',
ifu=ifu,
drpver=drpver,
dapver=dapver,
plate=plate,
mode='LOGCUBE',
daptype=daptype)
assert os.path.exists(filename), 'file not found.'
else:
raise MarvinError('cannot create path for MaNGA cube.')
else:
if idtype == 'plateifu':
plateifu = name
elif idtype == 'mangaid':
mangaid = name
else:
raise MarvinError(
'invalid plateifu or mangaid: {0}'.format(idtype))
model_cube = ModelCube(filename=filename,
mangaid=mangaid,
plateifu=plateifu,
release=release,
template=template,
bintype=bintype,
**kwargs)
results['status'] = 1
except Exception as err:
results['error'] = 'Failed to retrieve ModelCube {0}: {1}'.format(
name, str(err))
return model_cube, results
def set_session_versions(version):
''' Set the versions in the session '''
current_session['release'] = version
drpver, dapver = config.lookUpVersions(release=version)
current_session['drpver'] = drpver
current_session['dapver'] = dapver
def versions(release):
"""Yield the DRP and DAP versions for a release."""
drpver, dapver = config.lookUpVersions(release)
return drpver, dapver
