def _download_images(images, label='get_images'):
''' Download a set of images '''
rsync = RsyncAccess(label=label)
rsync.remote()
for image in images:
full = image._getFullPath()
rsync.add('', full=full)
rsync.set_stream()
rsync.commit()
def _get_cube(self, release=None):
if release:
self._update_release(release)
filepath = os.path.join(self.mangaredux, self.drpver,
str(self.new_plate), 'stack', self.new_file)
if not os.path.isfile(filepath):
rsync_access = RsyncAccess(label='marvin_getlist', verbose=False)
rsync_access.remote()
rsync_access.add('mangacube',
plate=self.new_plate,
drpver=self.drpver,
ifu=self.new_ifu,
dir3d='stack')
rsync_access.set_stream()
rsync_access.commit()
def rsync(mode):
''' fixture to create generic rsync object '''
rsync = RsyncAccess(label='marvin_getlist', verbose=False)
if mode != 'local':
rsync.remote()
yield rsync
rsync.reset()
rsync = None
def rsync():
''' fixture to create generic rsync object '''
rsync = RsyncAccess(label='test_rsync')
rsync.remote()
yield rsync
# teardown
rsync.reset()
rsync = None
def getImagesByList(inputlist, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get all images from a list of ids
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_list` instead.
Retrieve a list of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
inputlist (list):
A list of plate-ifus or mangaids for the images you want to retrieve. Required.
download (bool):
Set to download the images from the SAS. Only works in remote mode.
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images you have requested
'''
warnings.warn('getImagesByList is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_list', MarvinDeprecationWarning)
# Check inputs
assert isinstance(inputlist, (list, np.ndarray)), 'Input must be of type list or Numpy array'
idtype = parseIdentifier(inputlist[0])
assert idtype in ['plateifu', 'mangaid'], 'Input must be of type plate-ifu or mangaid'
# mode is checked via decorator
# convert mangaids into plateifus
if idtype == 'mangaid':
newlist = []
for myid in inputlist:
try:
plateifu = mangaid2plateifu(myid)
except MarvinError:
plateifu = None
newlist.append(plateifu)
inputlist = newlist
# setup Rsync Access
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getlist', verbose=verbose, public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
# Get list of images
listofimages = []
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
if as_url:
path = rsync_access.url(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
else:
path = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
listofimages.append(path)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
# Add plateifus to stream
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d)
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# get the list
listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getImagesByPlate(plateid, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get all images belonging to a given plate ID
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_plate` instead.
Retrieve all images belonging to a given plate ID from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
plateid (int):
The plate ID to retrieve the images for. Required.
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn('getImagesByPlate is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_plate', MarvinDeprecationWarning)
assert str(plateid).isdigit(), 'Plateid must be a numeric integer value'
# setup marvin inputs
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
#dir3d = getDir3d(plateid, mode=mode, release=release)
# setup Rsync Access
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getplate', verbose=verbose, public=is_public,
release=rsync_release)
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*')
listofimages = rsync_access.expand('', full=full, as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*')
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# get the list
listofimages = rsync_access.get_urls() if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getRandomImages(num=10, download=False, mode=None, as_url=None, verbose=None, release=None):
''' Get a list of N random images from SAS
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_random_images` instead.
Retrieve a random set of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
num (int):
The number of images to retrieve
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn('getRandomImages is deprecated as of Marvin 2.3.0. '
'Please use get_randome_images', MarvinDeprecationWarning)
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = RsyncAccess(label='marvin_getrandom', verbose=verbose, public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn('Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local', MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack')
listofimages = rsync_access.random('', full=full, num=num, refine=r'\d{4,5}.png', as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode', MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack')
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError('Error with sdss_access rsync.set_stream. AccessError: {0}'.format(e))
# refine and randomize
rsync_access.refine_task(r'\d{4,5}.png')
rsync_access.shuffle()
listofimages = rsync_access.get_urls(limit=num) if as_url else rsync_access.get_paths(limit=num)
if download:
rsync_access.commit()
else:
return listofimages
def download(self, pathType=None, **pathParams):
"""Download using sdss_access Rsync"""
# check for public release
is_public = 'DR' in self._release
rsync_release = self._release.lower() if is_public else None
if not RsyncAccess:
raise MarvinError('sdss_access is not installed')
else:
rsync_access = RsyncAccess(public=is_public, release=rsync_release)
rsync_access.remote()
rsync_access.add(pathType, **pathParams)
rsync_access.set_stream()
rsync_access.commit()
paths = rsync_access.get_paths()
# adding a millisecond pause for download to finish and file existence to register
time.sleep(0.001)
self.filename = paths[0] # doing this for single files, may need to change
def downloadList(inputlist, dltype='cube', **kwargs):
''' Download a list of MaNGA objects
Uses sdss_access to download a list of objects
via rsync. Places them in your local sas path mimicing
the Utah SAS.
i.e. $SAS_BASE_DIR/mangawork/manga/spectro/redux
Can download cubes, rss files, maps, mastar cubes, png images, default maps, or
the entire plate directory.
Parameters:
inputlist (list):
Required. A list of objects to download. Must be a list of plate IDs,
plate-IFUs, or manga-ids
dltype ({'cube', 'map', 'image', 'rss', 'mastar', 'default', 'plate'}):
Indicated type of object to download. Can be any of
plate, cube, imagea, mastar, rss, map, or default (default map).
If not specified, the dltype defaults to cube.
release (str):
The MPL/DR version of the data to download.
Defaults to Marvin config.release.
bintype (str):
The bin type of the DAP maps to download. Defaults to *
binmode (str):
The bin mode of the DAP maps to download. Defaults to *
n (int):
The plan id number [1-12] of the DAP maps to download. Defaults to *
daptype (str):
The daptype of the default map to grab. Defaults to *
dir3d (str):
The directory where the images are located. Either 'stack' or 'mastar'. Defaults to *
verbose (bool):
Turns on verbosity during rsync
limit (int):
A limit to the number of items to download
Returns:
NA: Downloads
'''
# Get some possible keywords
# Necessary rsync variables:
# drpver, plate, ifu, dir3d, [mpl, dapver, bintype, n, mode]
verbose = kwargs.get('verbose', None)
as_url = kwargs.get('as_url', None)
release = kwargs.get('release', marvin.config.release)
drpver, dapver = marvin.config.lookUpVersions(release=release)
bintype = kwargs.get('bintype', '*')
binmode = kwargs.get('binmode', '*')
daptype = kwargs.get('daptype', '*')
dir3d = kwargs.get('dir3d', '*')
n = kwargs.get('n', '*')
limit = kwargs.get('limit', None)
# check for sdss_access
if not RsyncAccess:
raise MarvinError('sdss_access not installed.')
# Assert correct dltype
dltype = 'cube' if not dltype else dltype
assert dltype in [
'plate', 'cube', 'mastar', 'rss', 'map', 'image', 'default'
], 'dltype must be one of plate, cube, mastar, image, rss, map, default'
# Assert correct dir3d
if dir3d != '*':
assert dir3d in ['stack',
'mastar'], 'dir3d must be either stack or mastar'
# Parse and retrieve the input type and the download type
idtype = parseIdentifier(inputlist[0])
if not idtype:
raise MarvinError(
'Input list must be a list of plates, plate-ifus, or mangaids')
# Set download type
if dltype == 'cube':
name = 'mangacube'
elif dltype == 'rss':
name = 'mangarss'
elif dltype == 'default':
name = 'mangadefault'
elif dltype == 'plate':
name = 'mangaplate'
elif dltype == 'map':
if '4' in release:
name = 'mangamap'
elif '5' in release:
name = 'mangadap5'
elif dltype == 'mastar':
name = 'mangamastar'
elif dltype == 'image':
name = 'mangaimage'
# create rsync
rsync_access = RsyncAccess(label='marvin_download', verbose=verbose)
rsync_access.remote()
# Add objects
for item in inputlist:
if idtype == 'mangaid':
try:
plateifu = mangaid2plateifu(item)
except MarvinError as e:
plateifu = None
else:
plateid, ifu = plateifu.split('-')
elif idtype == 'plateifu':
plateid, ifu = item.split('-')
elif idtype == 'plate':
plateid = item
ifu = '*'
rsync_access.add(name,
plate=plateid,
drpver=drpver,
ifu=ifu,
dapver=dapver,
dir3d=dir3d,
mpl=release,
bintype=bintype,
n=n,
mode=binmode,
daptype=daptype)
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError(
'Error with sdss_access rsync.set_stream. AccessError: {0}'.format(
e))
# get the list and download
listofitems = rsync_access.get_urls(
) if as_url else rsync_access.get_paths()
rsync_access.commit(limit=limit)
def download(self, pathType=None, **pathParams):
"""Download using sdss_access Rsync"""
if not RsyncAccess:
raise MarvinError('sdss_access is not installed')
else:
rsync_access = RsyncAccess()
rsync_access.remote()
rsync_access.add(pathType, **pathParams)
rsync_access.set_stream()
rsync_access.commit()
paths = rsync_access.get_paths()
time.sleep(
0.001
) # adding a millisecond pause for download to finish and file extistence to register
self.filename = paths[
0] # doing this for single files, may need to change
#!/usr/bin/env python
import marvin
from marvin import config
try:
from sdss_access import RsyncAccess
except ImportError as e:
RsyncAccess = None
rsync_access = RsyncAccess(label='marvin_get_test_data')
rsync_access.remote()
def add_data(rsync, release=None, plate=None, ifu=None, exclude=[]):
drpver, dapver = config.lookUpVersions(release)
if 'drpall' not in exclude:
rsync.add('drpall', drpver=drpver)
if 'mangacube' not in exclude:
rsync.add('mangacube', plate=plate, ifu=ifu, drpver=drpver)
if 'mangarss' not in exclude:
rsync.add('mangarss', plate=plate, ifu=ifu, drpver=drpver)
if 'mangaimage' not in exclude:
rsync.add('mangaimage',
plate=plate,
drpver=drpver,
dir3d='stack',
ifu='*')
if release in ['MPL-5', 'MPL-6']:
if 'mangadap5' not in exclude:
def download(self, pathType=None, **pathParams):
"""Download using sdss_access Rsync"""
if not RsyncAccess:
raise MarvinError('sdss_access is not installed')
else:
rsync_access = RsyncAccess()
rsync_access.remote()
rsync_access.add(pathType, **pathParams)
rsync_access.set_stream()
rsync_access.commit()
paths = rsync_access.get_paths()
self.filename = paths[
0] # doing this for single files, may need to change
