def test_origins(self, plateifu, origin, dir3d): im = Image(plateifu) assert im.plateifu == plateifu assert im.data_origin == origin if not check_versions(im._drpver, 'v2_5_3'): assert dir3d in im._getFullPath()
def getDir3d(inputid, mode=None, release=None): ''' Get the 3d redux Image directory from an input plate or plate-IFU ''' idtype = parseIdentifier(inputid) if idtype == 'plate': plateid = inputid elif idtype == 'plateifu': plateid, __ = inputid.split('-') release = marvin.config.release if not release else release drpver, __ = marvin.config.lookUpVersions(release=release) if check_versions(drpver, 'v1_5_4'): from marvin.tools.plate import Plate try: plate = Plate(plate=plateid, nocubes=True, mode=mode, release=release) except Exception as e: raise MarvinError('Could not retrieve a remote plate. If it is a mastar ' 'plate you are after, Marvin currently does not handle those: {0}'.format(e)) else: dir3d = plate.dir3d else: dir3d = 'stack' return dir3d
def _load_modelcube_from_file(self): """Initialises a model cube from a file.""" if self.data is not None: assert isinstance(self.data, fits.HDUList), 'data is not an HDUList object' else: try: with gunzip(self.filename) as gg: self.data = fits.open(gg.name) except IOError as err: raise IOError('filename {0} cannot be found: {1}'.format( self.filename, err)) self.header = self.data[0].header self._check_file(self.header, self.data, 'ModelCube') self.wcs = WCS(self.data['FLUX'].header) self._wavelength = self.data['WAVE'].data self._redcorr = self.data['REDCORR'].data self._shape = (self.data['FLUX'].header['NAXIS2'], self.data['FLUX'].header['NAXIS1']) self.plateifu = self.header['PLATEIFU'] self.mangaid = self.header['MANGAID'] # Checks and populates release. file_drpver = self.header['VERSDRP3'] file_drpver = 'v1_5_1' if file_drpver == 'v1_5_0' else file_drpver file_ver = marvin.config.lookUpRelease(file_drpver) assert file_ver is not None, 'cannot find file version.' if file_drpver != self._drpver: warnings.warn( 'mismatch between file version={0} and object release={1}. ' 'Setting object release to {0}'.format(file_ver, self._release), marvin.core.exceptions.MarvinUserWarning) self._release = file_ver self._drpver, self._dapver = marvin.config.lookUpVersions( release=self._release) # Updates datamodel, bintype, and template with the versions from the header. self.datamodel = datamodel[self._dapver].models self.bintype = self.datamodel.parent.get_bintype( self.header['BINKEY'].strip().upper()) if check_versions(self._dapver, datamodel['MPL-8'].release): tempkey = self.header['DAPTYPE'].split('-', 1)[-1] else: tempkey = self.header['SCKEY'] self.template = self.datamodel.parent.get_template( tempkey.strip().upper())
def download(self): """Downloads the image using sdss_access - Rsync,""" if not self.plateifu: return None plate, ifu = self.plateifu.split('-') dir3d = self._get_image_dir() # use version to toggle old/new images path isMPL8 = check_versions(self._drpver, 'v2_5_3') name = 'mangaimagenew' if isMPL8 else 'mangaimage' return super(Image, self).download(name, ifu=ifu, dir3d=dir3d, drpver=self._drpver, plate=plate)
def _getFullPath(self): """Returns the full path of the file in the tree.""" if not self.plateifu: return None plate, ifu = self.plateifu.split('-') dir3d = self._get_image_dir() # use version to toggle old/new images path isMPL8 = check_versions(self._drpver, 'v2_5_3') name = 'mangaimagenew' if isMPL8 else 'mangaimage' return super(Image, self)._getFullPath(name, ifu=ifu, dir3d=dir3d, drpver=self._drpver, plate=plate)
def set_filepaths(self, pathtype='full'): """Set the paths for cube, maps, etc.""" self.path = Path() if check_versions(self.drpver, 'v2_5_3'): self.imgpath = self.path.__getattribute__(pathtype)('mangaimagenew', **self.access_kwargs) else: self.imgpath = self.path.__getattribute__(pathtype)('mangaimage', **self.access_kwargs) self.cubepath = self.path.__getattribute__(pathtype)('mangacube', **self.access_kwargs) self.rsspath = self.path.__getattribute__(pathtype)('mangarss', **self.access_kwargs) if self.release == 'MPL-4': self.mapspath = self.path.__getattribute__(pathtype)('mangamap', **self.access_kwargs) self.modelpath = None else: self.access_kwargs.pop('mode') self.mapspath = self.path.__getattribute__(pathtype)('mangadap5', mode='MAPS', **self.access_kwargs) self.modelpath = self.path.__getattribute__(pathtype)('mangadap5', mode='LOGCUBE', **self.access_kwargs)
def make_paths(request, rsync, mode, asurl, release): inputs = request.param if hasattr(request, 'param') else None rmode = 'url' if asurl else 'full' fullpaths = [] inputs = inputs if inputs else imagelist for plateifu in inputs: gal = Galaxy(plateifu) gal.set_params(release=release) gal.set_filepaths() name = 'mangaimagenew' if check_versions(gal.drpver, 'v2_5_3') else 'mangaimage' if mode == 'local': path = rsync.__getattribute__(rmode)(name, **gal.access_kwargs) fullpaths.append(path) else: rsync.add(name, **gal.access_kwargs) rsync.set_stream() path = rsync.get_urls() if asurl else rsync.get_paths() fullpaths.extend(path) return fullpaths
def stellarcont_fit(self): """Returns the stellar continuum fit.""" isMPL8 = check_versions(self._dapver, datamodel['MPL-8'].release) if isMPL8: array = self._get_extension_data('stellar_fit') model = self.datamodel['stellar_fit'] mask = self._get_extension_data('stellar_fit', 'mask') else: array = (self._get_extension_data('full_fit') - self._get_extension_data('emline_fit') - self._get_extension_data('emline_base_fit')) model = self.datamodel['full_fit'] mask = self._get_extension_data('flux', 'mask') return DataCube(array, np.array(self._wavelength), ivar=None, mask=mask, redcorr=self._redcorr, binid=self.get_binid(model), unit=model.unit, pixmask_flag=model.pixmask_flag)
def _load_maps_from_file(self, data=None): """Loads a MAPS file.""" if data is not None: assert isinstance( data, astropy.io.fits.HDUList), 'data is not a HDUList.' else: self.data = astropy.io.fits.open(self.filename) self.header = self.data[0].header self.mangaid = self.header['MANGAID'].strip() self.plateifu = self.header['PLATEIFU'].strip() self._check_file(self.header, self.data, 'Maps') # We use EMLINE_GFLUX because is present in MPL-4 and 5 and is not expected to go away. header = self.data['EMLINE_GFLUX'].header naxis = header['NAXIS'] wcs_pre = astropy.wcs.WCS(header) # Takes only the first two axis. self.wcs = wcs_pre.sub(2) if naxis > 2 else naxis self._shape = (header['NAXIS2'], header['NAXIS1']) # Checks and populates release. file_drpver = self.header['VERSDRP3'] file_drpver = 'v1_5_1' if file_drpver == 'v1_5_0' else file_drpver file_ver = marvin.config.lookUpRelease(file_drpver) assert file_ver is not None, 'cannot find file version.' if file_ver != self._release: warnings.warn( 'mismatch between file version={0} and object release={1}. ' 'Setting object release to {0}'.format(file_ver, self._release), marvin.core.exceptions.MarvinUserWarning) self._release = file_ver self._drpver, self._dapver = marvin.config.lookUpVersions( release=self._release) self.datamodel = datamodel[self._dapver].properties # Checks the bintype from the header is_MPL4 = 'MPL-4' in self.datamodel.parent.aliases if not is_MPL4: header_bintype = self.data[0].header['BINKEY'].strip().upper() header_bintype = 'SPX' if header_bintype == 'NONE' else header_bintype else: header_bintype = self.data[0].header['BINTYPE'].strip().upper() # Checks the template from the header #is_MPL8 = parse_version(self._dapver) >= parse_version(datamodel['MPL-8'].release) is_MPL8 = check_versions(self._dapver, datamodel['MPL-8'].release) header_template_key = 'TPLKEY' if is_MPL4 else 'DAPTYPE' if is_MPL8 else 'SCKEY' if is_MPL8: template_val = self.data[0].header[header_template_key].split( '-', 1)[-1] else: template_val = self.data[0].header[header_template_key] header_template = template_val.strip().upper() if self.bintype.name != header_bintype: self.bintype = self.datamodel.parent.get_bintype(header_bintype) if self.template.name != header_template: self.template = self.datamodel.parent.get_template(header_template)
def showImage(path=None, plateifu=None, release=None, return_image=True, show_image=True, mode=None): ''' Crudely and coarsely show a galaxy image that has been downloaded .. deprecated:: 2.3.0 Use :class:`marvin.tools.image.Image` or :func:`show_image` instead. This utility function quickly allows you to display a PNG IFU image that is located in your local SAS or from the remote Utah SAS. A PIL Image object is also returned which allows you to manipulate the image after the fact. See :ref:`marvin-image-show` for example usage. Either the path or plateifu keyword is required. Parameters: path (str): A string filepath to a local IFU image plateifu (str): A plateifu designation used to look for the IFU image in your local SAS return_image (bool): If ``True``, returns the PIL Image object for image manipulation. Default is ``True``. show_image (bool): If ``True``, shows the requested image that is opened internally mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. release (str): The release version of the images to return Returns: image (PIL Image or None): If return_image is set, returns a PIL Image object to allow for image manipulation, else returns None. ''' warnings.warn('showImage is deprecated as of Marvin 2.3.0. ' 'Please use marvin.tools.image.Image instead.', MarvinDeprecationWarning) # check inputs release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) args = [path, plateifu] assert any(args), 'A filepath or plateifu must be specified!' imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # check path if path: if type(path) == list and len(path) > 1: raise MarvinError('showImage currently only works on a single input at a time') filepath = path[0] if type(path) == list else path # Deal with the mode if mode == 'local' and 'https://data.sdss.org' in filepath: raise MarvinError('Remote url path not allowed in local mode') elif mode == 'remote' and 'https://data.sdss.org' not in filepath: raise MarvinError('Local path not allowed in remote mode') elif mode == 'auto': if 'https://data.sdss.org' in filepath: mode = 'remote' else: mode = 'local' def _do_local_plateifu(name): full = http_access.full(name, plate=plateid, drpver=drpver, ifu=ifu, dir3d='*') filepath = http_access.expand('', full=full) if filepath: filepath = filepath[0] return filepath else: raise MarvinError('Error: No files found locally to match plateifu {0}. ' 'Use one of the image utility functions to download them first or ' 'switch to remote mode'.format(plateifu)) def _do_remote_plateifu(name): filepath = http_access.url(name, plate=plateid, drpver=drpver, ifu=ifu, dir3d='stack') return filepath # check plateifu if plateifu: plateid, ifu = plateifu.split('-') http_access = HttpAccess(verbose=False) if mode == 'local': filepath = _do_local_plateifu(imgname) elif mode == 'remote': filepath = _do_remote_plateifu(imgname) elif mode == 'auto': try: filepath = _do_local_plateifu(imgname) mode = 'local' except MarvinError: marvin.log.debug('Local mode failed. Trying remote.') filepath = _do_remote_plateifu(imgname) mode = 'remote' # check if filepath exists either locally or remotely if mode == 'local': if not filepath or not os.path.isfile(filepath): raise MarvinError('Error: local filepath {0} does not exist. '.format(filepath)) else: fileobj = filepath elif mode == 'remote': r = requests.get(filepath) if not r.ok: raise MarvinError('Error: remote filepath {0} does not exist'.format(filepath)) else: fileobj = stringio(r.content) # Open the image try: image = PIL.Image.open(fileobj) except IOError as e: print('Error: cannot open image') image = None else: image.filename = filepath if image and show_image: # show the image image.show() # return the PIL Image object if return_image: return image else: return None
def getImagesByList(inputlist, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get all images from a list of ids .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_images_by_list` instead. Retrieve a list of images from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: inputlist (list): A list of plate-ifus or mangaids for the images you want to retrieve. Required. download (bool): Set to download the images from the SAS. Only works in remote mode. mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images you have requested ''' warnings.warn('getImagesByList is deprecated as of Marvin 2.3.0. ' 'Please use get_images_by_list', MarvinDeprecationWarning) # Check inputs assert isinstance(inputlist, (list, np.ndarray)), 'Input must be of type list or Numpy array' idtype = parseIdentifier(inputlist[0]) assert idtype in ['plateifu', 'mangaid'], 'Input must be of type plate-ifu or mangaid' # mode is checked via decorator # convert mangaids into plateifus if idtype == 'mangaid': newlist = [] for myid in inputlist: try: plateifu = mangaid2plateifu(myid) except MarvinError: plateifu = None newlist.append(plateifu) inputlist = newlist # setup Rsync Access release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = RsyncAccess(label='marvin_getlist', verbose=verbose, public=is_public, release=rsync_release) imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # if mode is auto, set it to remote: if mode == 'auto': warnings.warn('Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' # do a local or remote thing if mode == 'local': # Get list of images listofimages = [] for plateifu in inputlist: dir3d = getDir3d(plateifu, mode=mode, release=release) plateid, ifu = plateifu.split('-') if as_url: path = rsync_access.url(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) else: path = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) listofimages.append(path) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() # Add plateifus to stream for plateifu in inputlist: dir3d = getDir3d(plateifu, mode=mode, release=release) plateid, ifu = plateifu.split('-') rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) # set the stream try: rsync_access.set_stream() except AccessError as e: raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e)) # get the list listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths() if download: rsync_access.commit() else: return listofimages
def getImagesByPlate(plateid, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get all images belonging to a given plate ID .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_images_by_plate` instead. Retrieve all images belonging to a given plate ID from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: plateid (int): The plate ID to retrieve the images for. Required. download (bool): Set to download the images from the SAS mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images ''' warnings.warn('getImagesByPlate is deprecated as of Marvin 2.3.0. ' 'Please use get_images_by_plate', MarvinDeprecationWarning) assert str(plateid).isdigit(), 'Plateid must be a numeric integer value' # setup marvin inputs release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) #dir3d = getDir3d(plateid, mode=mode, release=release) # setup Rsync Access is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = RsyncAccess(label='marvin_getplate', verbose=verbose, public=is_public, release=rsync_release) # if mode is auto, set it to remote: if mode == 'auto': warnings.warn('Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # do a local or remote thing if mode == 'local': full = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*') listofimages = rsync_access.expand('', full=full, as_url=as_url) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*') # set the stream try: rsync_access.set_stream() except AccessError as e: raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e)) # get the list listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths() if download: rsync_access.commit() else: return listofimages
def getRandomImages(num=10, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get a list of N random images from SAS .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_random_images` instead. Retrieve a random set of images from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: num (int): The number of images to retrieve download (bool): Set to download the images from the SAS mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images ''' warnings.warn('getRandomImages is deprecated as of Marvin 2.3.0. ' 'Please use get_randome_images', MarvinDeprecationWarning) release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = RsyncAccess(label='marvin_getrandom', verbose=verbose, public=is_public, release=rsync_release) imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # if mode is auto, set it to remote: if mode == 'auto': warnings.warn('Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' # do a local or remote thing if mode == 'local': full = rsync_access.full(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack') listofimages = rsync_access.random('', full=full, num=num, refine=r'\d{4,5}.png', as_url=as_url) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() rsync_access.add(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack') try: rsync_access.set_stream() except AccessError as e: raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e)) # refine and randomize rsync_access.refine_task(r'\d{4,5}.png') rsync_access.shuffle() listofimages = rsync_access.get_urls(limit=num) if as_url else rsync_access.get_paths(limit=num) if download: rsync_access.commit() else: return listofimages
def test_checks(self, v1, exp): assert check_versions(v1, 'v2_5_3') is exp