def _download_images(images, label='get_images'):
''' Download a set of images '''
rsync = Access(label=label)
rsync.remote()
for image in images:
full = image._getFullPath()
rsync.add('', full=full)
rsync.set_stream()
rsync.commit()
def rsync(mode):
''' fixture to create generic rsync object '''
rsync = Access(label='marvin_getlist', verbose=False)
if mode != 'local':
rsync.remote()
yield rsync
rsync.reset()
rsync = None
def create_new_access(release: str) -> Type[Access]:
''' create a new sdss_access instance
Parameters
----------
release : str
The sdss data release
'''
# check for public release
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
if not Access:
raise BrainMissingDependency('sdss_access is not installed')
return Access(public=is_public, release=rsync_release)
#!/usr/bin/env python
from marvin import config
try:
from sdss_access import Access
except ImportError as e:
Access = None
rsync_access = Access(label='marvin_get_test_data')
rsync_access.remote()
def add_data(rsync, release=None, plate=None, ifu=None, exclude=[]):
drpver, dapver = config.lookUpVersions(release)
if 'drpall' not in exclude:
rsync.add('drpall', drpver=drpver)
if 'mangacube' not in exclude:
rsync.add('mangacube', plate=plate, ifu=ifu, drpver=drpver)
if 'mangarss' not in exclude:
rsync.add('mangarss', plate=plate, ifu=ifu, drpver=drpver)
if 'mangaimage' not in exclude:
rsync.add('mangaimage', plate=plate, drpver=drpver, dir3d='stack', ifu='*')
if release in ['MPL-5', 'MPL-6', 'MPL-7']:
if 'mangadap5' not in exclude:
rsync.add('mangadap5', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu, daptype='*', mode='*')
elif release == 'MPL-4':
if 'mangamap' not in exclude:
rsync.add('mangamap', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu, bintype='*', mode='*', n='**')
def getImagesByList(inputlist,
download=False,
mode=None,
as_url=None,
verbose=None,
release=None):
''' Get all images from a list of ids
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_list` instead.
Retrieve a list of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
inputlist (list):
A list of plate-ifus or mangaids for the images you want to retrieve. Required.
download (bool):
Set to download the images from the SAS. Only works in remote mode.
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images you have requested
'''
warnings.warn(
'getImagesByList is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_list', MarvinDeprecationWarning)
# Check inputs
assert isinstance(
inputlist,
(list, np.ndarray)), 'Input must be of type list or Numpy array'
idtype = parseIdentifier(inputlist[0])
assert idtype in ['plateifu',
'mangaid'], 'Input must be of type plate-ifu or mangaid'
# mode is checked via decorator
# convert mangaids into plateifus
if idtype == 'mangaid':
newlist = []
for myid in inputlist:
try:
plateifu = mangaid2plateifu(myid)
except MarvinError:
plateifu = None
newlist.append(plateifu)
inputlist = newlist
# setup Rsync Access
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = Access(label='marvin_getlist',
verbose=verbose,
public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver,
'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn(
'Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local',
MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
# Get list of images
listofimages = []
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
if as_url:
path = rsync_access.url(imgname,
plate=plateid,
drpver=drpver,
ifu=ifu,
dir3d=dir3d)
else:
path = rsync_access.full(imgname,
plate=plateid,
drpver=drpver,
ifu=ifu,
dir3d=dir3d)
listofimages.append(path)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode',
MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
# Add plateifus to stream
for plateifu in inputlist:
dir3d = getDir3d(plateifu, mode=mode, release=release)
plateid, ifu = plateifu.split('-')
rsync_access.add(imgname,
plate=plateid,
drpver=drpver,
ifu=ifu,
dir3d=dir3d)
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError(
'Error with sdss_access rsync.set_stream. AccessError: {0}'.
format(e))
# get the list
listofimages = rsync_access.get_urls(
) if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getImagesByPlate(plateid,
download=False,
mode=None,
as_url=None,
verbose=None,
release=None):
''' Get all images belonging to a given plate ID
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_images_by_plate` instead.
Retrieve all images belonging to a given plate ID from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
plateid (int):
The plate ID to retrieve the images for. Required.
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn(
'getImagesByPlate is deprecated as of Marvin 2.3.0. '
'Please use get_images_by_plate', MarvinDeprecationWarning)
assert str(plateid).isdigit(), 'Plateid must be a numeric integer value'
# setup marvin inputs
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
#dir3d = getDir3d(plateid, mode=mode, release=release)
# setup Rsync Access
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = Access(label='marvin_getplate',
verbose=verbose,
public=is_public,
release=rsync_release)
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn(
'Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local',
MarvinUserWarning)
mode = 'remote'
imgname = 'mangaimagenew' if check_versions(drpver,
'v2_5_3') else 'mangaimage'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname,
plate=plateid,
drpver=drpver,
ifu='*',
dir3d='*')
listofimages = rsync_access.expand('', full=full, as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode',
MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname,
plate=plateid,
drpver=drpver,
ifu='*',
dir3d='*')
# set the stream
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError(
'Error with sdss_access rsync.set_stream. AccessError: {0}'.
format(e))
# get the list
listofimages = rsync_access.get_urls(
) if as_url else rsync_access.get_paths()
if download:
rsync_access.commit()
else:
return listofimages
def getRandomImages(num=10,
download=False,
mode=None,
as_url=None,
verbose=None,
release=None):
''' Get a list of N random images from SAS
.. deprecated:: 2.3.0
Use :class:`marvin.utils.general.images.get_random_images` instead.
Retrieve a random set of images from either your local filesystem SAS
or the Utah SAS. Optionally can download the images by rsync using
sdss_access.
When as_url is False, both local and remote modes will allow you to access
the full path to the images in your local SAS. WHen as_url is True,
local mode generates the Utah SAS url links, while remote mode generates the
Utah SAS rsync links.
Auto mode defaults to remote.
Parameters:
num (int):
The number of images to retrieve
download (bool):
Set to download the images from the SAS
mode ({'local', 'remote', 'auto'}):
The load mode to use. See
:doc:`Mode secision tree</mode_decision>`.
the cube exists.
as_url (bool):
Convert the list of images to use the SAS url (mode=local)
or the SAS rsync url (mode=remote)
verbose (bool):
Turns on verbosity during rsync
release (str):
The release version of the images to return
Returns:
listofimages (list):
The list of images
'''
warnings.warn(
'getRandomImages is deprecated as of Marvin 2.3.0. '
'Please use get_randome_images', MarvinDeprecationWarning)
release = release if release else marvin.config.release
drpver, __ = marvin.config.lookUpVersions(release=release)
is_public = 'DR' in release
rsync_release = release.lower() if is_public else None
rsync_access = Access(label='marvin_getrandom',
verbose=verbose,
public=is_public,
release=rsync_release)
imgname = 'mangaimagenew' if check_versions(drpver,
'v2_5_3') else 'mangaimage'
# if mode is auto, set it to remote:
if mode == 'auto':
warnings.warn(
'Mode is auto. Defaulting to remote. If you want to access your '
'local images, set the mode explicitly to local',
MarvinUserWarning)
mode = 'remote'
# do a local or remote thing
if mode == 'local':
full = rsync_access.full(imgname,
plate='*',
drpver=drpver,
ifu='*',
dir3d='stack')
listofimages = rsync_access.random('',
full=full,
num=num,
refine=r'\d{4,5}.png',
as_url=as_url)
# if download, issue warning that cannot do it
if download:
warnings.warn('Download not available when in local mode',
MarvinUserWarning)
return listofimages
elif mode == 'remote':
rsync_access.remote()
rsync_access.add(imgname,
plate='*',
drpver=drpver,
ifu='*',
dir3d='stack')
try:
rsync_access.set_stream()
except AccessError as e:
raise MarvinError(
'Error with sdss_access rsync.set_stream. AccessError: {0}'.
format(e))
# refine and randomize
rsync_access.refine_task(r'\d{4,5}.png')
rsync_access.shuffle()
listofimages = rsync_access.get_urls(
limit=num) if as_url else rsync_access.get_paths(limit=num)
if download:
rsync_access.commit()
else:
return listofimages
def download(self, pathType=None, **pathParams):
"""Download using sdss_access Rsync"""
# # check for public release
# is_public = 'DR' in self._release
# rsync_release = self._release.lower() if is_public else None
if not Access:
raise MarvinError('sdss_access is not installed')
else:
access = Access(release=self._release)
access.remote()
access.add(pathType, **pathParams)
access.set_stream()
access.commit()
paths = access.get_paths()
# adding a millisecond pause for download to finish and file existence to register
time.sleep(0.001)
self.filename = paths[
0] # doing this for single files, may need to change