def _download_images(images, label='get_images'): ''' Download a set of images ''' rsync = Access(label=label) rsync.remote() for image in images: full = image._getFullPath() rsync.add('', full=full) rsync.set_stream() rsync.commit()
def download(self, pathType=None, **pathParams): """Download using sdss_access Rsync""" # check for public release is_public = 'DR' in self._release rsync_release = self._release.lower() if is_public else None if not Access: raise MarvinError('sdss_access is not installed') else: access = Access(public=is_public, release=rsync_release) access.remote() access.add(pathType, **pathParams) access.set_stream() access.commit() paths = access.get_paths() # adding a millisecond pause for download to finish and file existence to register time.sleep(0.001) self.filename = paths[0] # doing this for single files, may need to change
elif release == 'MPL-4': if 'mangamap' not in exclude: rsync.add('mangamap', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu, bintype='*', mode='*', n='**') if 'mangadefault' not in exclude: rsync.add('mangadefault', plate=plate, drpver=drpver, dapver=dapver, ifu=ifu) return rsync # MPL-7 # rsync_access = add_data(rsync_access, release='MPL-7', plate='8485', ifu='1901') # rsync_access = add_data(rsync_access, release='MPL-7', plate='7443', ifu='12701') # MPL-6 rsync_access = add_data(rsync_access, release='MPL-6', plate='8485', ifu='1901') rsync_access = add_data(rsync_access, release='MPL-6', plate='7443', ifu='12701') # MPL-5 #rsync_access = add_data(rsync_access, release='MPL-5', plate='8485', ifu='1901') #rsync_access = add_data(rsync_access, release='MPL-5', plate='7443', ifu='12701') #, exclude=['mangaimage', 'mangadap5']) rsync_access = add_data(rsync_access, release='MPL-5', plate='8485', ifu='1901', exclude=['mangacube', 'mangarss', 'mangaimage', 'mangadap5']) # MPL-4 #rsync_access = add_data(rsync_access, release='MPL-4', plate='8485', ifu='1901') #rsync_access = add_data(rsync_access, release='MPL-4', plate='7443', ifu='12701') #, exclude=['mangaimage', 'mangamap', 'mangadefault']) rsync_access = add_data(rsync_access, release='MPL-4', plate='7443', ifu='12701', exclude=['mangacube', 'mangarss', 'mangaimage', 'mangamap', 'mangadefault']) # Download rsync_access.set_stream() rsync_access.commit()
def getImagesByList(inputlist, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get all images from a list of ids .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_images_by_list` instead. Retrieve a list of images from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: inputlist (list): A list of plate-ifus or mangaids for the images you want to retrieve. Required. download (bool): Set to download the images from the SAS. Only works in remote mode. mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images you have requested ''' warnings.warn( 'getImagesByList is deprecated as of Marvin 2.3.0. ' 'Please use get_images_by_list', MarvinDeprecationWarning) # Check inputs assert isinstance( inputlist, (list, np.ndarray)), 'Input must be of type list or Numpy array' idtype = parseIdentifier(inputlist[0]) assert idtype in ['plateifu', 'mangaid'], 'Input must be of type plate-ifu or mangaid' # mode is checked via decorator # convert mangaids into plateifus if idtype == 'mangaid': newlist = [] for myid in inputlist: try: plateifu = mangaid2plateifu(myid) except MarvinError: plateifu = None newlist.append(plateifu) inputlist = newlist # setup Rsync Access release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = Access(label='marvin_getlist', verbose=verbose, public=is_public, release=rsync_release) imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # if mode is auto, set it to remote: if mode == 'auto': warnings.warn( 'Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' # do a local or remote thing if mode == 'local': # Get list of images listofimages = [] for plateifu in inputlist: dir3d = getDir3d(plateifu, mode=mode, release=release) plateid, ifu = plateifu.split('-') if as_url: path = rsync_access.url(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) else: path = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) listofimages.append(path) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() # Add plateifus to stream for plateifu in inputlist: dir3d = getDir3d(plateifu, mode=mode, release=release) plateid, ifu = plateifu.split('-') rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu=ifu, dir3d=dir3d) # set the stream try: rsync_access.set_stream() except AccessError as e: raise MarvinError( 'Error with sdss_access rsync.set_stream. AccessError: {0}'. format(e)) # get the list listofimages = rsync_access.get_urls( ) if as_url else rsync_access.get_paths() if download: rsync_access.commit() else: return listofimages
def getImagesByPlate(plateid, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get all images belonging to a given plate ID .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_images_by_plate` instead. Retrieve all images belonging to a given plate ID from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: plateid (int): The plate ID to retrieve the images for. Required. download (bool): Set to download the images from the SAS mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images ''' warnings.warn( 'getImagesByPlate is deprecated as of Marvin 2.3.0. ' 'Please use get_images_by_plate', MarvinDeprecationWarning) assert str(plateid).isdigit(), 'Plateid must be a numeric integer value' # setup marvin inputs release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) #dir3d = getDir3d(plateid, mode=mode, release=release) # setup Rsync Access is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = Access(label='marvin_getplate', verbose=verbose, public=is_public, release=rsync_release) # if mode is auto, set it to remote: if mode == 'auto': warnings.warn( 'Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # do a local or remote thing if mode == 'local': full = rsync_access.full(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*') listofimages = rsync_access.expand('', full=full, as_url=as_url) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() rsync_access.add(imgname, plate=plateid, drpver=drpver, ifu='*', dir3d='*') # set the stream try: rsync_access.set_stream() except AccessError as e: raise MarvinError( 'Error with sdss_access rsync.set_stream. AccessError: {0}'. format(e)) # get the list listofimages = rsync_access.get_urls( ) if as_url else rsync_access.get_paths() if download: rsync_access.commit() else: return listofimages
def getRandomImages(num=10, download=False, mode=None, as_url=None, verbose=None, release=None): ''' Get a list of N random images from SAS .. deprecated:: 2.3.0 Use :class:`marvin.utils.general.images.get_random_images` instead. Retrieve a random set of images from either your local filesystem SAS or the Utah SAS. Optionally can download the images by rsync using sdss_access. When as_url is False, both local and remote modes will allow you to access the full path to the images in your local SAS. WHen as_url is True, local mode generates the Utah SAS url links, while remote mode generates the Utah SAS rsync links. Auto mode defaults to remote. Parameters: num (int): The number of images to retrieve download (bool): Set to download the images from the SAS mode ({'local', 'remote', 'auto'}): The load mode to use. See :doc:`Mode secision tree</mode_decision>`. the cube exists. as_url (bool): Convert the list of images to use the SAS url (mode=local) or the SAS rsync url (mode=remote) verbose (bool): Turns on verbosity during rsync release (str): The release version of the images to return Returns: listofimages (list): The list of images ''' warnings.warn( 'getRandomImages is deprecated as of Marvin 2.3.0. ' 'Please use get_randome_images', MarvinDeprecationWarning) release = release if release else marvin.config.release drpver, __ = marvin.config.lookUpVersions(release=release) is_public = 'DR' in release rsync_release = release.lower() if is_public else None rsync_access = Access(label='marvin_getrandom', verbose=verbose, public=is_public, release=rsync_release) imgname = 'mangaimagenew' if check_versions(drpver, 'v2_5_3') else 'mangaimage' # if mode is auto, set it to remote: if mode == 'auto': warnings.warn( 'Mode is auto. Defaulting to remote. If you want to access your ' 'local images, set the mode explicitly to local', MarvinUserWarning) mode = 'remote' # do a local or remote thing if mode == 'local': full = rsync_access.full(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack') listofimages = rsync_access.random('', full=full, num=num, refine=r'\d{4,5}.png', as_url=as_url) # if download, issue warning that cannot do it if download: warnings.warn('Download not available when in local mode', MarvinUserWarning) return listofimages elif mode == 'remote': rsync_access.remote() rsync_access.add(imgname, plate='*', drpver=drpver, ifu='*', dir3d='stack') try: rsync_access.set_stream() except AccessError as e: raise MarvinError( 'Error with sdss_access rsync.set_stream. AccessError: {0}'. format(e)) # refine and randomize rsync_access.refine_task(r'\d{4,5}.png') rsync_access.shuffle() listofimages = rsync_access.get_urls( limit=num) if as_url else rsync_access.get_paths(limit=num) if download: rsync_access.commit() else: return listofimages