def get_raw_spectra_matrix(raw_data_path):
#if os.path.isfile(metadata_path):
if False:
manga_metadata = pandas.read_csv(metadata_path)
x_list = list(manga_metadata['x'].values)
y_list = list(manga_metadata['y'].values)
ra_list = list(manga_metadata['ra'].values)
dec_list = list(manga_metadata['dec'].values)
z_list = list(manga_metadata['z'].values)
galaxy_name_list = list(manga_metadata['galaxy'].values)
file_name_list = list(manga_metadata['cube_file_name'].values)
waves = list(numpy.load(wave_path))
spectra = list(numpy.load(specra_path))
all_spxl_count = len(spectra)
else:
x_list = []
y_list = []
ra_list = []
dec_list = []
z_list = []
galaxy_name_list = []
file_name_list = []
waves = []
spectra = []
all_spxl_count = 0
temp_save_count = 0
for path, subdirs, files in os.walk(raw_data_path):
for cube_file_name in files:
if not cube_file_name in file_name_list:
try:
cube_path = os.path.join(path, cube_file_name)
cube = Cube(cube_path)
galaxy = cube.nsa['iauname']
z = cube.nsa['z']
wave = get_rest_wave(z)
maps = cube.getMaps()
snr = maps.bin_snr
high_snr_pixels = numpy.where(snr.value > 15)
nof_high_snr_pixels = high_snr_pixels[0].size
spxls = cube[high_snr_pixels]
count = 0
for spx in spxls:
if spx.quality_flags[1].bits is None:
x_list += [spx.x]
y_list += [spx.y]
z_list += [z]
ra_list += [spx.ra]
dec_list += [spx.dec]
galaxy_name_list += [galaxy]
file_name_list += [cube_file_name]
spectra += [spx.flux.value]
waves += [wave]
count = count + 1
all_spxl_count = all_spxl_count + 1
temp_save_count = temp_save_count + 1
if temp_save_count > 50000:
manga_metadata = pandas.DataFrame()
manga_metadata['x'] = x_list
manga_metadata['y'] = y_list
manga_metadata['ra'] = ra_list
manga_metadata['dec'] = dec_list
manga_metadata['galaxy'] = galaxy_name_list
manga_metadata['cube_file_name'] = file_name_list
manga_metadata.to_csv(metadata_path)
numpy.save(wave_path, waves)
numpy.save(specra_path, spectra)
temp_save_count = 0
print('Got {} pixels from {}. Total so far {}:'.format(
count, galaxy, all_spxl_count))
except:
print('Error for cube {}'.format(cube_path))
pass
manga_metadata = pandas.DataFrame()
manga_metadata['x'] = x_list
manga_metadata['y'] = y_list
manga_metadata['ra'] = ra_list
manga_metadata['dec'] = dec_list
manga_metadata['galaxy'] = galaxy_name_list
manga_metadata['cube_file_name'] = file_name_list
return manga_metadata, numpy.array(waves), numpy.array(spectra)
def get(self, galid):
''' Retrieve info for a given cube '''
# determine type of galid
args = av.manual_parse(self, request, use_params='galaxy')
self.galaxy['id'] = args['galid']
self.galaxy['latest_dr'] = self._release.lower(
) if 'DR' in self._release else marvin.config._get_latest_release(
dr_only=True).lower()
idtype = parseIdentifier(galid)
if idtype in ['plateifu', 'mangaid']:
# set plateifu or mangaid
self.galaxy['idtype'] = idtype
galaxyid = {self.galaxy['idtype']: galid, 'release': self._release}
# Get cube
try:
cube = Cube(**galaxyid)
except MarvinError as e:
self.galaxy['cube'] = None
errmsg = 'MarvinError: {0}'.format(e)
# check target status and fine-tune the error message
if idtype == 'mangaid':
status = target_status(galid, drpver=self._drpver)
if status == 'not yet observed':
errmsg = '{0} is a valid target but has not yet been observed'.format(
galid)
elif status == 'not valid target':
errmsg = '{0} is not valid MaNGA target. Check your syntax'.format(
galid)
self.galaxy['error'] = errmsg
return render_template("galaxy.html", **self.galaxy)
else:
dm = datamodel[self._dapver]
self.galaxy['cube'] = cube
self.galaxy['daplink'] = getDapRedux(release=self._release)
# get SAS url links to cube, rss, maps, image
if Path:
is_public = 'DR' in self._release
path_release = self._release.lower() if is_public else None
sdss_path = Path(public=is_public, release=path_release)
self.galaxy['image'] = cube.getImage().url
cubelink = sdss_path.url('mangacube',
drpver=cube._drpver,
plate=cube.plate,
ifu=cube.ifu)
rsslink = sdss_path.url('mangarss',
drpver=cube._drpver,
plate=cube.plate,
ifu=cube.ifu)
daptype = "{0}-{1}".format(dm.default_bintype,
dm.default_template)
maplink = getDefaultMapPath(release=self._release,
plate=cube.plate,
ifu=cube.ifu,
daptype=daptype,
mode='MAPS')
mclink = getDefaultMapPath(release=self._release,
plate=cube.plate,
ifu=cube.ifu,
daptype=daptype,
mode='LOGCUBE')
self.galaxy['links'] = {
'cube': cubelink,
'rss': rsslink,
'map': maplink,
'mc': mclink
}
else:
self.galaxy['image'] = cube.data.image
# Get the initial spectrum
if cube:
dm = datamodel[self._dapver]
daplist = [p.full(web=True) for p in dm.properties]
dapdefaults = dm.get_default_mapset()
self.galaxy['cube'] = cube
self.galaxy['toggleon'] = current_session.get(
'toggleon', 'false')
self.galaxy['cubehdr'] = cube.header
self.galaxy['quality'] = ('DRP3QUAL', cube.quality_flag.mask,
cube.quality_flag.labels)
self.galaxy['mngtarget'] = {
'bits':
[it.mask for it in cube.target_flags if it.mask != 0],
'labels': [
it.labels for it in cube.target_flags
if len(it.labels) > 0
],
'names': [
''.join(('MNGTARG', it.name[-1]))
for it in cube.target_flags if it.mask != 0
]
}
# make the nsa dictionary
hasnsa = cube.nsa is not None
self.galaxy['hasnsa'] = hasnsa
if hasnsa:
cols = self.galaxy.get('nsaplotcols')
nsadict, nsacols = make_nsa_dict(cube.nsa)
nsatmp = [
nsacols.pop(nsacols.index(i)) for i in cols
if i in nsacols
]
nsatmp.extend(nsacols)
self.galaxy['nsacols'] = nsatmp
self.galaxy['nsadict'] = nsadict
self.galaxy['dapmaps'] = daplist
self.galaxy['dapmapselect'] = current_session.get(
'selected_dapmaps', dapdefaults)
dm = datamodel[self._dapver]
self.galaxy['dapbintemps'] = dm.get_bintemps(db_only=True)
if 'bintemp' not in current_session or current_session[
'bintemp'] not in self.galaxy['dapbintemps']:
current_session['bintemp'] = '{0}-{1}'.format(
dm.get_bintype(), dm.get_template())
# get default map quality
maps = cube.getMaps(
plateifu=cube.plateifu,
mode='local',
bintype=current_session['bintemp'].split('-')[0])
mapqual = ('DAPQUAL', maps.quality_flag.mask,
maps.quality_flag.labels)
self.galaxy['mapquality'] = mapqual
# TODO - make this general - see also search.py for querystr
self.galaxy['cubestr'] = (
"<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># access the header<br>cube.header<br># get NSA data<br>\
cube.nsa<br></samp></html>".format(cube.plateifu))
self.galaxy['spaxelstr'] = (
"<html><samp>from marvin.tools.cube import Cube<br>cube = \
Cube(plateifu='{0}')<br># get a spaxel by slicing cube[i,j]<br>spaxel=cube[16, 16]<br>flux = \
spaxel.flux<br>wave = flux.wavelength<br>ivar = flux.ivar<br>mask = \
flux.mask<br>flux.plot()<br></samp></html>".format(
cube.plateifu))
self.galaxy['mapstr'] = (
"<html><samp>from marvin.tools.maps import Maps<br>maps = \
Maps(plateifu='{0}')<br>print(maps)<br># get an emission \
line map<br>haflux = maps.emline_gflux_ha_6564<br>values = \
haflux.value<br>ivar = haflux.ivar<br>mask = haflux.mask<br>haflux.plot()<br>\
</samp></html>".format(cube.plateifu))
else:
self.galaxy[
'error'] = 'Error: Galaxy ID {0} must either be a Plate-IFU, or MaNGA-Id designation.'.format(
galid)
return render_template("galaxy.html", **self.galaxy)
return render_template("galaxy.html", **self.galaxy)
