def setGlobalSession(): ''' Sets the global session for Flask ''' mpls = list(config._mpldict.keys()) versions = [{ 'name': mpl, 'subtext': str(config.lookUpVersions(release=mpl)) } for mpl in mpls] current_session['versions'] = versions if 'release' not in current_session: current_session['release'] = config.release drpver, dapver = config.lookUpVersions(release=config.release) current_session['drpver'] = drpver current_session['dapver'] = dapver
def _query_versions(self, mode='local', release=None): release = config.release drpver, dapver = config.lookUpVersions(release=release) p = 'haflux > 25' if '4' in release: name = 'CleanSpaxelProp' else: name = 'CleanSpaxelProp{0}'.format(release.split('-')[1]) if release: q = Query(searchfilter=p, mode=mode, release=release) else: q = Query(searchfilter=p, mode=mode) self.assertEqual(q._drpver, drpver) self.assertEqual(q._dapver, dapver) self.assertEqual(q._release, release) self.assertEqual(q.marvinform._release, release) self.assertEqual(q.marvinform._param_form_lookup._release, release) self.assertEqual( q.marvinform._param_form_lookup['spaxelprop.file'].Meta.model. __name__, name)
def setUp(self): self._reset_the_config() self.set_sasurl('local') self.mode = self.init_mode config.setMPL('MPL-4') config.forceDbOn() self.drpver, __ = config.lookUpVersions(release=config.release)
def update_allowed(): ''' Update the allowed versions shown in the header dropdown ''' logged_in = current_session.get('loginready', None) mpls = list(config._allowed_releases.keys()) # this is to remove MPLs from the list of web versions when not logged in to the collaboration site if not logged_in: # select only DR allowed releases mpls = [mpl for mpl in mpls if 'DR' in mpl] # select the release to switch to; prioritize the session release if 'release' not in current_session: release = config.release if 'DR' in config.release else max(mpls) else: release = current_session['release'] if 'DR' in current_session[ 'release'] else max(mpls) # update the session with the new release set_session_versions(release) versions = [{ 'name': mpl, 'subtext': str(config.lookUpVersions(release=mpl)) } for mpl in mpls] return versions
def test_summary_files(self): config.setDefaultDrpAll() # need to reset the config drp, dap = config.lookUpVersions() assert config.drpall is not None assert config.dapall is not None assert 'drpall-{0}'.format(drp) in config.drpall assert 'dapall-{0}-{1}'.format(drp, dap) in config.dapall
def set_params(self, bintype=None, template=None, release=None): """Set bintype, template, etc.""" self.release = release self.drpver, self.dapver = config.lookUpVersions(self.release) self.drpall = 'drpall-{0}.fits'.format(self.drpver) self.bintype = datamodel[self.dapver].get_bintype(bintype) self.template = datamodel[self.dapver].get_template(template) self.bintemp = '{0}-{1}'.format(self.bintype.name, self.template.name) if release == 'MPL-4': self.niter = int('{0}{1}'.format(self.template.n, self.bintype.n)) else: self.niter = '*' self.access_kwargs = { 'plate': self.plate, 'ifu': self.ifu, 'drpver': self.drpver, 'dapver': self.dapver, 'dir3d': self.dir3d, 'mpl': self.release, 'bintype': self.bintype.name, 'n': self.niter, 'mode': '*', 'daptype': self.bintemp }
def get_drpver(release=None): ''' Get the drpver from the input release from the request ''' if not release: release = config.release drpver, dapver = config.lookUpVersions(release) return drpver
def update_allowed(): ''' Update the allowed versions ''' mpls = list(config._allowed_releases.keys()) versions = [{ 'name': mpl, 'subtext': str(config.lookUpVersions(release=mpl)) } for mpl in mpls] return versions
def __init__(self, input=None, filename=None, mangaid=None, plateifu=None, mode=None, data=None, release=None, drpall=None, download=None, ignore_db=False): self.data = data self.data_origin = None self._ignore_db = ignore_db self.filename = filename self.mangaid = mangaid self.plateifu = plateifu self.mode = mode if mode is not None else config.mode self._release = release if release is not None else config.release self._drpver, self._dapver = config.lookUpVersions( release=self._release) self._drpall = config._getDrpAllPath( self._drpver) if drpall is None else drpall self._forcedownload = download if download is not None else config.download self._determine_inputs(input) assert self.mode in ['auto', 'local', 'remote'] assert self.filename is not None or self.plateifu is not None, 'no inputs set.' self.datamodel = None self._set_datamodel() if self.mode == 'local': self._doLocal() elif self.mode == 'remote': self._doRemote() elif self.mode == 'auto': try: self._doLocal() except Exception as ee: if self.filename: # If the input contains a filename we don't want to go into remote mode. raise (ee) else: warnings.warn('local mode failed. Trying remote now.', MarvinUserWarning) self._doRemote() # Sanity check to make sure data_origin has been properly set. assert self.data_origin in ['file', 'db', 'api'], 'data_origin is not properly set.'
def test_masked(self, maps_release_only): __, dapver = config.lookUpVersions(maps_release_only.release) params = get_default_plot_params(dapver) ha = maps_release_only['emline_gflux_ha_6564'] expected = ha.pixmask.get_mask(params['default']['bitmasks'], dtype=bool) assert ha.masked.data == pytest.approx(ha.value) assert (ha.masked.mask == expected).all()
def set_summary_file(self, release): ''' Sets the path to the Firefly summary file ''' # define the variables to build a unique path to your VAC file # look up the MaNGA drpver from the release drpver, dapver = config.lookUpVersions(release) self.path_params = {'ver': self.version[release], 'drpver': drpver} # get_path returns False if the files do not exist locally self.summary_file = self.get_path('mangaffly', path_params=self.path_params)
def test_cube_remote_drpver_differ_from_global(self, galaxy, monkeyconfig): if galaxy.release == 'MPL-5': pytest.skip('Skipping release for forced global MPL-5') drpver, dapver = config.lookUpVersions(config.release) assert config.release == 'MPL-5' cube = Cube(plateifu=galaxy.plateifu, mode='remote', release=galaxy.release) assert cube.release != config.release assert cube._drpver != drpver
def selectmpl(self): ''' Global selection of the MPL/DR versions ''' args = av.manual_parse(self, request, use_params='index') version = args.get('release', None) current_session['release'] = version drpver, dapver = config.lookUpVersions(release=version) current_session['drpver'] = drpver current_session['dapver'] = dapver out = {'status': 1, 'msg': 'Success', 'current_release': version, 'current_drpver': drpver, 'current_dapver': dapver} return jsonify(result=out)
def _getCube(name, use_file=False, release=None, **kwargs): ''' Retrieve a cube using marvin tools ''' drpver, __ = config.lookUpVersions(release) cube = None results = {} # parse name into either mangaid or plateifu try: idtype = parseIdentifier(name) except Exception as ee: results['error'] = 'Failed to parse input name {0}: {1}'.format(name, str(ee)) return cube, results filename = None plateifu = None mangaid = None try: if use_file: if idtype == 'mangaid': plate, ifu = mangaid2plateifu(name, drpver=drpver) elif idtype == 'plateifu': plate, ifu = name.split('-') if Path is not None: filename = Path().full('mangacube', ifu=ifu, plate=plate, drpver=drpver) assert os.path.exists(filename), 'file not found.' else: raise MarvinError('cannot create path for MaNGA cube.') else: if idtype == 'plateifu': plateifu = name elif idtype == 'mangaid': mangaid = name else: raise MarvinError('invalid plateifu or mangaid: {0}'.format(idtype)) cube = Cube(filename=filename, mangaid=mangaid, plateifu=plateifu, mode='local', release=release) results['status'] = 1 except Exception as ee: results['error'] = 'Failed to retrieve cube {0}: {1}'.format(name, str(ee)) return cube, results
def test_format_use_masks(self, use_masks, mask, expected, set_release): if config.release == 'MPL-4': pytest.skip('Skip MPL-4.') for prop in [ 'stellar_vel', 'stellar_sigma', 'emline_gflux', 'specindex' ]: params = datamodel[config.lookUpVersions()[1]].get_plot_params( prop) actual = mapplot._format_use_masks( use_masks, mask, dapmap=None, default_masks=params['bitmasks']) assert actual == expected
def updateGlobalSession(): ''' updates the Marvin config with the global Flask session ''' # check if mpl versions in session if 'versions' not in current_session: setGlobalSession() elif 'drpver' not in current_session or \ 'dapver' not in current_session: drpver, dapver = config.lookUpVersions(release=config.release) current_session['drpver'] = drpver current_session['dapver'] = dapver elif 'release' not in current_session: current_session['release'] = config.release
def test_format_use_masks_mpl4(self, use_masks, mask, expected, set_release): if config.release != 'MPL-4': pytest.skip('Only include MPL-4.') for prop in [ 'stellar_vel', 'stellar_sigma', 'emline_gflux', 'specindex' ]: params = get_plot_params(dapver=config.lookUpVersions()[1], prop=prop) actual = mapplot._format_use_masks( use_masks, mask, dapmap=None, default_masks=params['bitmasks']) assert actual == expected
def update_allowed(): ''' Update the allowed versions ''' logged_in = current_session.get('loginready', None) mpls = list(config._allowed_releases.keys()) # this is to remove MPLs from the list of web versions when not logged in to the collaboration site if not logged_in: mpls = [mpl for mpl in mpls if 'DR' in mpl] set_session_versions(max(mpls)) versions = [{ 'name': mpl, 'subtext': str(config.lookUpVersions(release=mpl)) } for mpl in mpls] return versions
def create_args(self): ''' Build the final argument list for webargs validation ''' # get the dapver drpver, self.dapver = config.lookUpVersions(self.release) # reset the final args self._reset_final_args() # add view args to the list self._add_view_args() # add form param args to the list if self.use_params: self._add_param_args()
def add_data(rsync, release=None, plate=None, ifu=None, exclude=[]): drpver, dapver = config.lookUpVersions(release) if 'drpall' not in exclude: rsync.add('drpall', drpver=drpver) if 'mangacube' not in exclude: rsync.add('mangacube', plate=plate, ifu=ifu, drpver=drpver) if 'mangarss' not in exclude: rsync.add('mangarss', plate=plate, ifu=ifu, drpver=drpver) if 'mangaimage' not in exclude: rsync.add('mangaimage', plate=plate, drpver=drpver, dir3d='stack', ifu='*') if release in ['MPL-5', 'MPL-6']: if 'mangadap5' not in exclude: rsync.add('mangadap5', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu, daptype='*', mode='*') elif release == 'MPL-4': if 'mangamap' not in exclude: rsync.add('mangamap', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu, bintype='*', mode='*', n='**') if 'mangadefault' not in exclude: rsync.add('mangadefault', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu) return rsync
def test_pickling_data(self, temp_scratch, galaxy): drpver, __ = config.lookUpVersions() maps = Maps(filename=galaxy.mapspath) modelcube = ModelCube(filename=galaxy.modelpath) spaxel = maps.getSpaxel(25, 15, xyorig='lower', drp=False, models=modelcube) file = temp_scratch.join('test_spaxel.mpf') path_saved = spaxel.save(str(file), overwrite=True) assert file.check() is True assert os.path.exists(path_saved) del spaxel spaxel_restored = Spaxel.restore(str(file)) assert spaxel_restored.stellar_vel.value is not None assert spaxel_restored.stellar_vel.bin.binid is not None
def _test_getspaxel(self, spaxel, galaxy): spaxel_drpver, spaxel_dapver = config.lookUpVersions(spaxel.release) assert spaxel_drpver == galaxy.drpver assert spaxel_dapver == galaxy.dapver assert spaxel.plateifu == galaxy.plateifu assert spaxel.mangaid == galaxy.mangaid assert spaxel.getModelCube() is not None assert spaxel.getModelCube().bintype == galaxy.bintype assert spaxel.getModelCube().template == galaxy.template assert spaxel.template == galaxy.template assert spaxel.template == galaxy.template assert spaxel._parent_shape == tuple(galaxy.shape) assert isinstance(spaxel.binned_flux, Spectrum) assert isinstance(spaxel.full_fit, Spectrum) assert isinstance(spaxel.emline_fit, Spectrum)
def test_pickling_all_api(self, temp_scratch, galaxy): drpver, __ = config.lookUpVersions() cube = Cube(plateifu=galaxy.plateifu, mode='remote') maps = Maps(plateifu=galaxy.plateifu, mode='remote') modelcube = ModelCube(plateifu=galaxy.plateifu, mode='remote') spaxel = cube.getSpaxel(1, 3, maps=maps, modelcube=modelcube) assert spaxel._cube.data_origin == 'api' assert spaxel._maps.data_origin == 'api' assert spaxel._modelcube.data_origin == 'api' file = temp_scratch.join('test_spaxel_api.mpf') path_saved = spaxel.save(str(file), overwrite=True) assert file.check() is True assert os.path.exists(path_saved) del spaxel spaxel_restored = Spaxel.restore(str(file)) assert spaxel_restored is not None assert isinstance(spaxel_restored, Spaxel) assert spaxel_restored._cube is not None assert isinstance(spaxel_restored._cube, Cube) assert spaxel_restored._cube.data_origin == 'api' assert spaxel_restored._cube.data is None assert spaxel_restored._cube.header['VERSDRP3'] == drpver assert spaxel_restored._maps is not None assert isinstance(spaxel_restored._maps, Maps) assert spaxel_restored._maps.data_origin == 'api' assert spaxel_restored._maps.data is None assert spaxel_restored._modelcube is not None assert isinstance(spaxel_restored._modelcube, ModelCube) assert spaxel_restored._modelcube.data_origin == 'api' assert spaxel_restored._modelcube.data is None
def get_drpall_row(plateifu, drpver=None, drpall=None): """Returns a dictionary from drpall matching the plateifu.""" from marvin import config if drpall: drpall_table = table.Table.read(drpall) else: config_drpver, __ = config.lookUpVersions() drpver = drpver if drpver else config_drpver if drpver not in drpTable: drpall = drpall if drpall else config._getDrpAllPath(drpver=drpver) drpTable[drpver] = table.Table.read(drpall) drpall_table = drpTable[drpver] row = drpall_table[drpall_table['plateifu'] == plateifu] if len(row) != 1: raise ValueError('{0} results found for {1} in drpall table'.format( len(row), plateifu)) return row[0]
def setUpClass(cls): config.use_sentry = False config.add_github_message = False # set initial config variables cls.init_mode = config.mode cls.init_sasurl = config.sasurl cls.init_urlmap = config.urlmap cls.init_xyorig = config.xyorig cls.init_traceback = config._traceback cls.init_keys = ['mode', 'sasurl', 'urlmap', 'xyorig', 'traceback'] # set db stuff cls._marvindb = marvindb cls.session = marvindb.session # set paths cls.sasbasedir = os.getenv("$SAS_BASE_DIR") cls.mangaredux = os.getenv("MANGA_SPECTRO_REDUX") cls.mangaanalysis = os.getenv("MANGA_SPECTRO_ANALYSIS") # testing data for 8485-1901 cls.set_plateifu(plateifu='8485-1901') cls.mangaid = '1-209232' cls.cubepk = 10179 cls.ra = 232.544703894 cls.dec = 48.6902009334 cls.redshift = 0.0407447 cls.dir3d = 'stack' cls.release = 'MPL-5' cls.drpver, cls.dapver = config.lookUpVersions(cls.release) cls.bintemp = _get_bintemps(cls.dapver, default=True) cls.defaultbin, cls.defaulttemp = cls.bintemp.split('-', 1) cls.cubename = 'manga-{0}-LOGCUBE.fits.gz'.format(cls.plateifu) cls.rssname = 'manga-{0}-LOGRSS.fits.gz'.format(cls.plateifu) cls.imgname = '{0}.png'.format(cls.ifu) cls.mapsname = 'manga-{0}-MAPS-{1}.fits.gz'.format(cls.plateifu, cls.bintemp) cls.modelname = 'manga-{0}-LOGCUBE-{1}.fits.gz'.format(cls.plateifu, cls.bintemp)
def load_mangadrp(plateifu, loglin='LOG', rsscube='CUBE', mpl_v=None): """retrieve local MaNGA logcube, downloading if necessary Parameters ---------- plateifu : str plateifu designation of galaxy loglin : str, optional wavelength scale (LIN/LOG) rsscube : str, optional RSS or CUBE mpl_v : None or str, optional MPL version Returns ------- fits.HDUList HDUList with logcube data """ # resolve plateifu into filename if mpl_v is None: mpl_v = mconf.release drpver, dapver = mconf.lookUpVersions(mpl_v) sas_basedir = os.environ['SAS_BASE_DIR'] plate, ifu = plateifu.split('-') fname = f'manga-{plateifu}-{loglin.upper()}{rsscube.upper()}.fits.gz' fpath_full = os.path.join( sas_basedir, 'mangawork/manga/spectro/redux', drpver, plate, 'stack', fname) if not os.path.isfile(fpath_full): marvin_download([plateifu], dltype=rsscube.lower(), release=mpl_v) return fits.open(fpath_full)
def bits(request, set_release): params = datamodel[config.lookUpVersions()[1]].get_plot_params( request.param) return params['bitmasks']
def _get_model_cube(name, use_file=False, release=None, **kwargs): """Retrieves a Marvin ModelCube object.""" model_cube = None results = {} drpver, dapver = config.lookUpVersions(release) # parse name into either mangaid or plateifu try: idtype = parseIdentifier(name) except Exception as err: results['error'] = 'Failed to parse input name {0}: {1}'.format( name, str(err)) return model_cube, results filename = None plateifu = None mangaid = None bintype = kwargs.pop('bintype') template = kwargs.pop('template') try: if use_file: if idtype == 'mangaid': plate, ifu = mangaid2plateifu(name, drpver=drpver) elif idtype == 'plateifu': plate, ifu = name.split('-') if Path is not None: daptype = '{0}-{1}'.format(bintype, template) filename = Path().full('mangadap5', ifu=ifu, drpver=drpver, dapver=dapver, plate=plate, mode='LOGCUBE', daptype=daptype) assert os.path.exists(filename), 'file not found.' else: raise MarvinError('cannot create path for MaNGA cube.') else: if idtype == 'plateifu': plateifu = name elif idtype == 'mangaid': mangaid = name else: raise MarvinError( 'invalid plateifu or mangaid: {0}'.format(idtype)) model_cube = ModelCube(filename=filename, mangaid=mangaid, plateifu=plateifu, release=release, template=template, bintype=bintype, **kwargs) results['status'] = 1 except Exception as err: results['error'] = 'Failed to retrieve ModelCube {0}: {1}'.format( name, str(err)) return model_cube, results
def set_session_versions(version): ''' Set the versions in the session ''' current_session['release'] = version drpver, dapver = config.lookUpVersions(release=version) current_session['drpver'] = drpver current_session['dapver'] = dapver
def versions(release): """Yield the DRP and DAP versions for a release.""" drpver, dapver = config.lookUpVersions(release) return drpver, dapver