def get_raw_spectra_matrix(raw_data_path): #if os.path.isfile(metadata_path): if False: manga_metadata = pandas.read_csv(metadata_path) x_list = list(manga_metadata['x'].values) y_list = list(manga_metadata['y'].values) ra_list = list(manga_metadata['ra'].values) dec_list = list(manga_metadata['dec'].values) z_list = list(manga_metadata['z'].values) galaxy_name_list = list(manga_metadata['galaxy'].values) file_name_list = list(manga_metadata['cube_file_name'].values) waves = list(numpy.load(wave_path)) spectra = list(numpy.load(specra_path)) all_spxl_count = len(spectra) else: x_list = [] y_list = [] ra_list = [] dec_list = [] z_list = [] galaxy_name_list = [] file_name_list = [] waves = [] spectra = [] all_spxl_count = 0 temp_save_count = 0 for path, subdirs, files in os.walk(raw_data_path): for cube_file_name in files: if not cube_file_name in file_name_list: try: cube_path = os.path.join(path, cube_file_name) cube = Cube(cube_path) galaxy = cube.nsa['iauname'] z = cube.nsa['z'] wave = get_rest_wave(z) maps = cube.getMaps() snr = maps.bin_snr high_snr_pixels = numpy.where(snr.value > 15) nof_high_snr_pixels = high_snr_pixels[0].size spxls = cube[high_snr_pixels] count = 0 for spx in spxls: if spx.quality_flags[1].bits is None: x_list += [spx.x] y_list += [spx.y] z_list += [z] ra_list += [spx.ra] dec_list += [spx.dec] galaxy_name_list += [galaxy] file_name_list += [cube_file_name] spectra += [spx.flux.value] waves += [wave] count = count + 1 all_spxl_count = all_spxl_count + 1 temp_save_count = temp_save_count + 1 if temp_save_count > 50000: manga_metadata = pandas.DataFrame() manga_metadata['x'] = x_list manga_metadata['y'] = y_list manga_metadata['ra'] = ra_list manga_metadata['dec'] = dec_list manga_metadata['galaxy'] = galaxy_name_list manga_metadata['cube_file_name'] = file_name_list manga_metadata.to_csv(metadata_path) numpy.save(wave_path, waves) numpy.save(specra_path, spectra) temp_save_count = 0 print('Got {} pixels from {}. Total so far {}:'.format( count, galaxy, all_spxl_count)) except: print('Error for cube {}'.format(cube_path)) pass manga_metadata = pandas.DataFrame() manga_metadata['x'] = x_list manga_metadata['y'] = y_list manga_metadata['ra'] = ra_list manga_metadata['dec'] = dec_list manga_metadata['galaxy'] = galaxy_name_list manga_metadata['cube_file_name'] = file_name_list return manga_metadata, numpy.array(waves), numpy.array(spectra)
def get(self, galid): ''' Retrieve info for a given cube ''' # determine type of galid args = av.manual_parse(self, request, use_params='galaxy') self.galaxy['id'] = args['galid'] self.galaxy['latest_dr'] = self._release.lower( ) if 'DR' in self._release else marvin.config._get_latest_release( dr_only=True).lower() idtype = parseIdentifier(galid) if idtype in ['plateifu', 'mangaid']: # set plateifu or mangaid self.galaxy['idtype'] = idtype galaxyid = {self.galaxy['idtype']: galid, 'release': self._release} # Get cube try: cube = Cube(**galaxyid) except MarvinError as e: self.galaxy['cube'] = None errmsg = 'MarvinError: {0}'.format(e) # check target status and fine-tune the error message if idtype == 'mangaid': status = target_status(galid, drpver=self._drpver) if status == 'not yet observed': errmsg = '{0} is a valid target but has not yet been observed'.format( galid) elif status == 'not valid target': errmsg = '{0} is not valid MaNGA target. Check your syntax'.format( galid) self.galaxy['error'] = errmsg return render_template("galaxy.html", **self.galaxy) else: dm = datamodel[self._dapver] self.galaxy['cube'] = cube self.galaxy['daplink'] = getDapRedux(release=self._release) # get SAS url links to cube, rss, maps, image if Path: is_public = 'DR' in self._release path_release = self._release.lower() if is_public else None sdss_path = Path(public=is_public, release=path_release) self.galaxy['image'] = cube.getImage().url cubelink = sdss_path.url('mangacube', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu) rsslink = sdss_path.url('mangarss', drpver=cube._drpver, plate=cube.plate, ifu=cube.ifu) daptype = "{0}-{1}".format(dm.default_bintype, dm.default_template) maplink = getDefaultMapPath(release=self._release, plate=cube.plate, ifu=cube.ifu, daptype=daptype, mode='MAPS') mclink = getDefaultMapPath(release=self._release, plate=cube.plate, ifu=cube.ifu, daptype=daptype, mode='LOGCUBE') self.galaxy['links'] = { 'cube': cubelink, 'rss': rsslink, 'map': maplink, 'mc': mclink } else: self.galaxy['image'] = cube.data.image # Get the initial spectrum if cube: dm = datamodel[self._dapver] daplist = [p.full(web=True) for p in dm.properties] dapdefaults = dm.get_default_mapset() self.galaxy['cube'] = cube self.galaxy['toggleon'] = current_session.get( 'toggleon', 'false') self.galaxy['cubehdr'] = cube.header self.galaxy['quality'] = ('DRP3QUAL', cube.quality_flag.mask, cube.quality_flag.labels) self.galaxy['mngtarget'] = { 'bits': [it.mask for it in cube.target_flags if it.mask != 0], 'labels': [ it.labels for it in cube.target_flags if len(it.labels) > 0 ], 'names': [ ''.join(('MNGTARG', it.name[-1])) for it in cube.target_flags if it.mask != 0 ] } # make the nsa dictionary hasnsa = cube.nsa is not None self.galaxy['hasnsa'] = hasnsa if hasnsa: cols = self.galaxy.get('nsaplotcols') nsadict, nsacols = make_nsa_dict(cube.nsa) nsatmp = [ nsacols.pop(nsacols.index(i)) for i in cols if i in nsacols ] nsatmp.extend(nsacols) self.galaxy['nsacols'] = nsatmp self.galaxy['nsadict'] = nsadict self.galaxy['dapmaps'] = daplist self.galaxy['dapmapselect'] = current_session.get( 'selected_dapmaps', dapdefaults) dm = datamodel[self._dapver] self.galaxy['dapbintemps'] = dm.get_bintemps(db_only=True) if 'bintemp' not in current_session or current_session[ 'bintemp'] not in self.galaxy['dapbintemps']: current_session['bintemp'] = '{0}-{1}'.format( dm.get_bintype(), dm.get_template()) # get default map quality maps = cube.getMaps( plateifu=cube.plateifu, mode='local', bintype=current_session['bintemp'].split('-')[0]) mapqual = ('DAPQUAL', maps.quality_flag.mask, maps.quality_flag.labels) self.galaxy['mapquality'] = mapqual # TODO - make this general - see also search.py for querystr self.galaxy['cubestr'] = ( "<html><samp>from marvin.tools.cube import Cube<br>cube = \ Cube(plateifu='{0}')<br># access the header<br>cube.header<br># get NSA data<br>\ cube.nsa<br></samp></html>".format(cube.plateifu)) self.galaxy['spaxelstr'] = ( "<html><samp>from marvin.tools.cube import Cube<br>cube = \ Cube(plateifu='{0}')<br># get a spaxel by slicing cube[i,j]<br>spaxel=cube[16, 16]<br>flux = \ spaxel.flux<br>wave = flux.wavelength<br>ivar = flux.ivar<br>mask = \ flux.mask<br>flux.plot()<br></samp></html>".format( cube.plateifu)) self.galaxy['mapstr'] = ( "<html><samp>from marvin.tools.maps import Maps<br>maps = \ Maps(plateifu='{0}')<br>print(maps)<br># get an emission \ line map<br>haflux = maps.emline_gflux_ha_6564<br>values = \ haflux.value<br>ivar = haflux.ivar<br>mask = haflux.mask<br>haflux.plot()<br>\ </samp></html>".format(cube.plateifu)) else: self.galaxy[ 'error'] = 'Error: Galaxy ID {0} must either be a Plate-IFU, or MaNGA-Id designation.'.format( galid) return render_template("galaxy.html", **self.galaxy) return render_template("galaxy.html", **self.galaxy)