def test_origins(self, plateifu, origin, dir3d):
im = Image(plateifu)
assert im.plateifu == plateifu
assert im.data_origin == origin
if not check_versions(im._drpver, 'v2_5_3'):
assert dir3d in im._getFullPath()
def getDir3d(inputid, mode=None, release=None):
''' Get the 3d redux Image directory from an input plate or plate-IFU '''
idtype = parseIdentifier(inputid)
if idtype == 'plate':
plateid = inputid
elif idtype == 'plateifu':
plateid, __ = inputid.split('-')
release = marvin.config.release if not release else release
drpver, __ = marvin.config.lookUpVersions(release=release)
if check_versions(drpver, 'v1_5_4'):
from marvin.tools.plate import Plate
try:
plate = Plate(plate=plateid, nocubes=True, mode=mode, release=release)
except Exception as e:
raise MarvinError('Could not retrieve a remote plate. If it is a mastar '
'plate you are after, Marvin currently does not handle those: {0}'.format(e))
else:
dir3d = plate.dir3d
else:
dir3d = 'stack'
return dir3d
def _load_modelcube_from_file(self):
"""Initialises a model cube from a file."""
if self.data is not None:
assert isinstance(self.data,
fits.HDUList), 'data is not an HDUList object'
else:
try:
with gunzip(self.filename) as gg:
self.data = fits.open(gg.name)
except IOError as err:
raise IOError('filename {0} cannot be found: {1}'.format(
self.filename, err))
self.header = self.data[0].header
self._check_file(self.header, self.data, 'ModelCube')
self.wcs = WCS(self.data['FLUX'].header)
self._wavelength = self.data['WAVE'].data
self._redcorr = self.data['REDCORR'].data
self._shape = (self.data['FLUX'].header['NAXIS2'],
self.data['FLUX'].header['NAXIS1'])
self.plateifu = self.header['PLATEIFU']
self.mangaid = self.header['MANGAID']
# Checks and populates release.
file_drpver = self.header['VERSDRP3']
file_drpver = 'v1_5_1' if file_drpver == 'v1_5_0' else file_drpver
file_ver = marvin.config.lookUpRelease(file_drpver)
assert file_ver is not None, 'cannot find file version.'
if file_drpver != self._drpver:
warnings.warn(
'mismatch between file version={0} and object release={1}. '
'Setting object release to {0}'.format(file_ver,
self._release),
marvin.core.exceptions.MarvinUserWarning)
self._release = file_ver
self._drpver, self._dapver = marvin.config.lookUpVersions(
release=self._release)
# Updates datamodel, bintype, and template with the versions from the header.
self.datamodel = datamodel[self._dapver].models
self.bintype = self.datamodel.parent.get_bintype(
self.header['BINKEY'].strip().upper())
if check_versions(self._dapver, datamodel['MPL-8'].release):
tempkey = self.header['DAPTYPE'].split('-', 1)[-1]
else:
tempkey = self.header['SCKEY']
self.template = self.datamodel.parent.get_template(
tempkey.strip().upper())
def download(self):
"""Downloads the image using sdss_access - Rsync,"""
if not self.plateifu:
return None
plate, ifu = self.plateifu.split('-')
dir3d = self._get_image_dir()
# use version to toggle old/new images path
isMPL8 = check_versions(self._drpver, 'v2_5_3')
name = 'mangaimagenew' if isMPL8 else 'mangaimage'
return super(Image, self).download(name,
ifu=ifu,
dir3d=dir3d,
drpver=self._drpver,
plate=plate)
def _getFullPath(self):
"""Returns the full path of the file in the tree."""
if not self.plateifu:
return None
plate, ifu = self.plateifu.split('-')
dir3d = self._get_image_dir()
# use version to toggle old/new images path
isMPL8 = check_versions(self._drpver, 'v2_5_3')
name = 'mangaimagenew' if isMPL8 else 'mangaimage'
return super(Image, self)._getFullPath(name,
ifu=ifu,
dir3d=dir3d,
drpver=self._drpver,
plate=plate)
def set_filepaths(self, pathtype='full'):
"""Set the paths for cube, maps, etc."""
self.path = Path()
if check_versions(self.drpver, 'v2_5_3'):
self.imgpath = self.path.__getattribute__(pathtype)('mangaimagenew', **self.access_kwargs)
else:
self.imgpath = self.path.__getattribute__(pathtype)('mangaimage', **self.access_kwargs)
self.cubepath = self.path.__getattribute__(pathtype)('mangacube', **self.access_kwargs)
self.rsspath = self.path.__getattribute__(pathtype)('mangarss', **self.access_kwargs)
if self.release == 'MPL-4':
self.mapspath = self.path.__getattribute__(pathtype)('mangamap', **self.access_kwargs)
self.modelpath = None
else:
self.access_kwargs.pop('mode')
self.mapspath = self.path.__getattribute__(pathtype)('mangadap5', mode='MAPS',
**self.access_kwargs)
self.modelpath = self.path.__getattribute__(pathtype)('mangadap5', mode='LOGCUBE',
**self.access_kwargs)
def make_paths(request, rsync, mode, asurl, release):
inputs = request.param if hasattr(request, 'param') else None
rmode = 'url' if asurl else 'full'
fullpaths = []
inputs = inputs if inputs else imagelist
for plateifu in inputs:
gal = Galaxy(plateifu)
gal.set_params(release=release)
gal.set_filepaths()
name = 'mangaimagenew' if check_versions(gal.drpver,
'v2_5_3') else 'mangaimage'
if mode == 'local':
path = rsync.__getattribute__(rmode)(name, **gal.access_kwargs)
fullpaths.append(path)
else:
rsync.add(name, **gal.access_kwargs)
rsync.set_stream()
path = rsync.get_urls() if asurl else rsync.get_paths()
fullpaths.extend(path)
return fullpaths
def stellarcont_fit(self):
"""Returns the stellar continuum fit."""
isMPL8 = check_versions(self._dapver, datamodel['MPL-8'].release)
if isMPL8:
array = self._get_extension_data('stellar_fit')
model = self.datamodel['stellar_fit']
mask = self._get_extension_data('stellar_fit', 'mask')
else:
array = (self._get_extension_data('full_fit') -
self._get_extension_data('emline_fit') -
self._get_extension_data('emline_base_fit'))
model = self.datamodel['full_fit']
mask = self._get_extension_data('flux', 'mask')
return DataCube(array,
np.array(self._wavelength),
ivar=None,
mask=mask,
redcorr=self._redcorr,
binid=self.get_binid(model),
unit=model.unit,
pixmask_flag=model.pixmask_flag)
def _load_maps_from_file(self, data=None):
"""Loads a MAPS file."""
if data is not None:
assert isinstance(
data, astropy.io.fits.HDUList), 'data is not a HDUList.'
else:
self.data = astropy.io.fits.open(self.filename)
self.header = self.data[0].header
self.mangaid = self.header['MANGAID'].strip()
self.plateifu = self.header['PLATEIFU'].strip()
self._check_file(self.header, self.data, 'Maps')
# We use EMLINE_GFLUX because is present in MPL-4 and 5 and is not expected to go away.
header = self.data['EMLINE_GFLUX'].header
naxis = header['NAXIS']
wcs_pre = astropy.wcs.WCS(header)
# Takes only the first two axis.
self.wcs = wcs_pre.sub(2) if naxis > 2 else naxis
self._shape = (header['NAXIS2'], header['NAXIS1'])
# Checks and populates release.
file_drpver = self.header['VERSDRP3']
file_drpver = 'v1_5_1' if file_drpver == 'v1_5_0' else file_drpver
file_ver = marvin.config.lookUpRelease(file_drpver)
assert file_ver is not None, 'cannot find file version.'
if file_ver != self._release:
warnings.warn(
'mismatch between file version={0} and object release={1}. '
'Setting object release to {0}'.format(file_ver,
self._release),
marvin.core.exceptions.MarvinUserWarning)
self._release = file_ver
self._drpver, self._dapver = marvin.config.lookUpVersions(
release=self._release)
self.datamodel = datamodel[self._dapver].properties
# Checks the bintype from the header
is_MPL4 = 'MPL-4' in self.datamodel.parent.aliases
if not is_MPL4:
header_bintype = self.data[0].header['BINKEY'].strip().upper()
header_bintype = 'SPX' if header_bintype == 'NONE' else header_bintype
else:
header_bintype = self.data[0].header['BINTYPE'].strip().upper()
# Checks the template from the header
#is_MPL8 = parse_version(self._dapver) >= parse_version(datamodel['MPL-8'].release)
is_MPL8 = check_versions(self._dapver, datamodel['MPL-8'].release)
header_template_key = 'TPLKEY' if is_MPL4 else 'DAPTYPE' if is_MPL8 else 'SCKEY'
if is_MPL8:
template_val = self.data[0].header[header_template_key].split(
'-', 1)[-1]
else:
template_val = self.data[0].header[header_template_key]
header_template = template_val.strip().upper()
if self.bintype.name != header_bintype:
self.bintype = self.datamodel.parent.get_bintype(header_bintype)
if self.template.name != header_template:
self.template = self.datamodel.parent.get_template(header_template)
def showImage(path=None, plateifu=None, release=None, return_image=True, show_image=True, mode=None):
''' Crudely and coarsely show a galaxy image that has been downloaded
.. deprecated:: 2.3.0
Use :class:`marvin.tools.image.Image` or :func:`show_image` instead.
This utility function quickly allows you to display a PNG IFU image that is located in your
local SAS or from the remote Utah SAS. A PIL Image object is also returned which allows you to
manipulate the image after the fact. See :ref:`marvin-image-show` for example usage.
Either the path or plateifu keyword is required.
Parameters:
path (str):
A string filepath to a local IFU image
plateifu (str):
A plateifu designation used to look for the IFU image in your local SAS
return_image (bool):
If ``True``, returns the PIL Image object for image manipulation. Default is ``True``.
show_image (bool):
If ``True``, shows the requested image that is opened internally
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
release (str):
The release version of the images to return
Returns:
image (PIL Image or None):
If return_image is set, returns a PIL Image object to allow for image manipulation, else returns None.
'''
warnings.warn('showImage is deprecated as of Marvin 2.3.0. '
'Please use marvin.tools.image.Image instead.', MarvinDeprecationWarning)
# check inputs
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
args = [path, plateifu]
assert any(args), 'A filepath or plateifu must be specified!'
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# check path
if path:
if type(path) == list and len(path) > 1:
raise MarvinError('showImage currently only works on a single input at a time')
filepath = path[0] if type(path) == list else path
# Deal with the mode
if mode == 'local' and 'https://data.sdss.org' in filepath:
raise MarvinError('Remote url path not allowed in local mode')
elif mode == 'remote' and 'https://data.sdss.org' not in filepath:
raise MarvinError('Local path not allowed in remote mode')
elif mode == 'auto':
if 'https://data.sdss.org' in filepath:
mode = 'remote'
else:
mode = 'local'
def _do_local_plateifu(name):
full = http_access.full(name, plate=plateid, drpver=drpver, ifu=ifu, dir3d='*')
filepath = http_access.expand('', full=full)
if filepath:
filepath = filepath[0]
return filepath
else:
raise MarvinError('Error: No files found locally to match plateifu {0}. '
'Use one of the image utility functions to download them first or '
'switch to remote mode'.format(plateifu))
def _do_remote_plateifu(name):
filepath = http_access.url(name, plate=plateid, drpver=drpver, ifu=ifu, dir3d='stack')
return filepath
# check plateifu
if plateifu:
plateid, ifu = plateifu.split('-')
http_access = HttpAccess(verbose=False)
if mode == 'local':
filepath = _do_local_plateifu(imgname)
elif mode == 'remote':
filepath = _do_remote_plateifu(imgname)
elif mode == 'auto':
try:
filepath = _do_local_plateifu(imgname)
mode = 'local'
except MarvinError:
marvin.log.debug('Local mode failed. Trying remote.')
filepath = _do_remote_plateifu(imgname)
mode = 'remote'
# check if filepath exists either locally or remotely
if mode == 'local':
if not filepath or not os.path.isfile(filepath):
raise MarvinError('Error: local filepath {0} does not exist. '.format(filepath))
else:
fileobj = filepath
elif mode == 'remote':
r = requests.get(filepath)
if not r.ok:
raise MarvinError('Error: remote filepath {0} does not exist'.format(filepath))
else:
fileobj = stringio(r.content)
# Open the image
try:
image = PIL.Image.open(fileobj)
except IOError as e:
print('Error: cannot open image')
image = None
else:
image.filename = filepath
if image and show_image:
# show the image
image.show()
# return the PIL Image object
if return_image:
return image
else:
return None
def getImagesByList(inputlist, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get all images from a list of ids
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_list` instead.
Retrieve a list of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
inputlist (list):
A list of plate-ifus or mangaids for the images you want to retrieve. Required.
download (bool):
Set to download the images from the SAS. Only works in remote mode.
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images you have requested
'''
warnings.warn('getImagesByList is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_list', MarvinDeprecationWarning)
# Check inputs
assert isinstance(inputlist, (list, np.ndarray)), 'Input must be of type list or Numpy array'
idtype = parseIdentifier(inputlist[0])
assert idtype in ['plateifu', 'mangaid'], 'Input must be of type plate-ifu or mangaid'
# mode is checked via decorator
# convert mangaids into plateifus
if idtype == 'mangaid':
newlist = []
for myid in inputlist:
try:
plateifu = mangaid2plateifu(myid)
except MarvinError:
plateifu = None
newlist.append(plateifu)
inputlist = newlist
# setup Rsync Access
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getlist', verbose=verbose, public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
# Get list of images
listofimages = []
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
if as_url:
path = rsync_access.url(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
else:
path = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
listofimages.append(path)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
# Add plateifus to stream
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# get the list
listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getImagesByPlate(plateid, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get all images belonging to a given plate ID
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_plate` instead.
Retrieve all images belonging to a given plate ID from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
plateid (int):
The plate ID to retrieve the images for. Required.
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn('getImagesByPlate is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_plate', MarvinDeprecationWarning)
assert str(plateid).isdigit(), 'Plateid must be a numeric integer value'
# setup marvin inputs
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
#dir3d = getDir3d(plateid, mode=mode, release=release)
# setup Rsync Access
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getplate', verbose=verbose, public=is_public,
release=rsync_release)
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*')
listofimages = rsync_access.expand('', full=full, as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*')
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# get the list
listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getRandomImages(num=10, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get a list of N random images from SAS
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_random_images` instead.
Retrieve a random set of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
num (int):
The number of images to retrieve
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn('getRandomImages is deprecated as of Marvin 2.3.0. '
'Please use get_randome_images', MarvinDeprecationWarning)
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getrandom', verbose=verbose, public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack')
listofimages = rsync_access.random('', full=full, num=num, refine=r'\d{4,5}.png', as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack')
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# refine and randomize
rsync_access.refine_task(r'\d{4,5}.png')
rsync_access.shuffle()
listofimages = rsync_access.get_urls(limit=num) if as_url else rsync_access.get_paths(limit=num)
if download:
rsync_access.commit()
else:
return listofimages
def test_checks(self, v1, exp):
assert check_versions(v1, 'v2_5_3') is exp
