def from_url(cls, url): """ Return object read from URL. Parameters ---------- url : str URL to retrieve the data from """ path = download_file(url, get_and_create_download_dir()) return cls.read(path)
def from_url(cls, url): """ Return object read from URL. Parameters ---------- url : str URL to retrieve the data from """ default_dir = sunpy.config.get("downloads", "download_dir") path = download_file(url, default_dir) return cls.read(path)
def download(urls, directory): """ Download files from urls into directory. Parameters ---------- urls : list of str urls of the files to retrieve directory : str directory to save them in """ return [download_file(url, directory) for url in urls]
def from_url(cls, url): """ Return object read from URL. Parameters ---------- url : str URL to retrieve the data from """ default_dir = config.get("downloads", "download_dir") path = download_file(url, default_dir) return cls.read(path)
def _parse_args(self, *args, **kwargs): """ Parses an args list for data-header pairs. args can contain any mixture of the following entries: * tuples of data,header * data, header not in a tuple * data, wcs object in a tuple * data, wcs object not in a tuple * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. Example ------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_pairs = list() already_maps = list() # Account for nested lists of items args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header or data-WCS pair if isinstance(arg, SUPPORTED_ARRAY_TYPES): arg_header = args[i+1] if isinstance(arg_header, WCS): arg_header = args[i+1].to_header() if self._validate_meta(arg_header): pair = (args[i], OrderedDict(arg_header)) data_header_pairs.append(pair) i += 1 # an extra increment to account for the data-header pairing # File name elif (isinstance(arg, six.string_types) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # Directory elif (isinstance(arg, six.string_types) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Glob elif (isinstance(arg, six.string_types) and '*' in arg): files = glob.glob(os.path.expanduser(arg)) for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Already a Map elif isinstance(arg, GenericMap): already_maps.append(arg) # A URL elif (isinstance(arg, six.string_types) and _is_url(arg)): url = arg path = download_file(url, get_and_create_download_dir()) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # A database Entry elif isinstance(arg, DatabaseEntry): data_header_pairs += self._read_file(arg.path, **kwargs) else: raise ValueError("File not found or invalid input") i += 1 # TODO: # In the end, if there are already maps it should be put in the same # order as the input, currently they are not. return data_header_pairs, already_maps
def _parse_args(self, *args, **kwargs): """ Parses an args list for data-header pairs. args can contain any mixture of the following entries: * tuples of data,header * data, header not in a tuple * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. Example ------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_pairs = list() already_maps = list() # Account for nested lists of items args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header pair in a tuple if ((type(arg) in [tuple, list]) and len(arg) == 2 and isinstance(arg[0],np.ndarray) and isinstance(arg[1],dict)): data_header_pairs.append(arg) # Data-header pair not in a tuple elif (isinstance(arg, np.ndarray) and isinstance(args[i+1],dict)): pair = (args[i], args[i+1]) data_header_pairs.append(pair) i += 1 # an extra increment to account for the data-header pairing # File name elif (isinstance(arg,basestring) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # Directory elif (isinstance(arg,basestring) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Glob elif (isinstance(arg,basestring) and '*' in arg): files = glob.glob( os.path.expanduser(arg) ) for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Already a Map elif isinstance(arg, GenericMap): already_maps.append(arg) # A URL elif (isinstance(arg,basestring) and _is_url(arg)): default_dir = sunpy.config.get("downloads", "download_dir") url = arg path = download_file(url, default_dir) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # A database Entry elif isinstance(arg, DatabaseEntry): data_header_pairs += self._read_file(arg.path, **kwargs) else: raise ValueError("File not found or invalid input") i += 1 #TODO: # In the end, if there are aleady maps it should be put in the same # order as the input, currently they are not. return data_header_pairs, already_maps
def _parse_args(self, *args, **kwargs): """ Parses an `args` list for data-header pairs. `args` can contain any mixture of the following entries: * tuples of (data, header, unit) (1) * data, header not in a tuple (1) * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. (1) header/unit are optional and in either order, but data should be the first entry in each group. Examples -------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_unit_tuples = list() data_header_pairs = list() already_timeseries = list() filepaths = list() # Account for nested lists of items. Simply outputs a single list of # items, nested lists are expanded to element level. args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header pair in a tuple if (isinstance(arg, (np.ndarray, Table, pd.DataFrame))): # and self._validate_meta(args[i+1])): # Assume a Pandas Dataframe is given data = arg units = OrderedDict() meta = MetaDict() # Convert the data argument into a Pandas DataFrame if needed. if isinstance(data, Table): # We have an Astropy Table: data, meta, units = self._from_table(data) elif isinstance(data, np.ndarray): # We have a numpy ndarray. We assume the first column is a dt index data = pd.DataFrame(data=data[:, 1:], index=Time(data[:, 0])) # If there are 1 or 2 more arguments: for _ in range(2): if (len(args) > i+1): # If that next argument isn't data but is metaddata or units: if not isinstance(args[i+1], (np.ndarray, Table, pd.DataFrame)): if self._validate_units(args[i+1]): units.update(args[i+1]) i += 1 # an extra increment to account for the units elif self._validate_meta(args[i+1]): # if we have an astropy.io FITS header then convert # to preserve multi-line comments if isinstance(args[i+1], astropy.io.fits.header.Header): args[i+1] = MetaDict(sunpy.io.header.FileHeader(args[i+1])) meta.update(args[i+1]) i += 1 # an extra increment to account for the meta # Add a 3-tuple for this TimeSeries. data_header_unit_tuples.append((data, meta, units)) # Filepath elif (isinstance(arg, str) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) result = self._read_file(path, **kwargs) data_header_pairs, filepaths = _apply_result(data_header_pairs, filepaths, result) # Directory elif (isinstance(arg, str) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: # returns a boolean telling us if it were read and either a # tuple or the original filepath for reading by a source result = self._read_file(afile, **kwargs) data_header_pairs, filepaths = _apply_result(data_header_pairs, filepaths, result) # Glob elif isinstance(arg, str) and '*' in arg: files = glob.glob(os.path.expanduser(arg)) for afile in files: # returns a boolean telling us if it were read and either a # tuple or the original filepath for reading by a source result = self._read_file(afile, **kwargs) data_header_pairs, filepaths = _apply_result(data_header_pairs, filepaths, result) # Already a TimeSeries elif isinstance(arg, GenericTimeSeries): already_timeseries.append(arg) # A URL elif (isinstance(arg, str) and _is_url(arg)): url = arg path = download_file(url, get_and_create_download_dir()) result = self._read_file(path, **kwargs) data_header_pairs, filepaths = _apply_result(data_header_pairs, filepaths, result) else: raise NoMatchError("File not found or invalid input") i += 1 # TODO: # In the end, if there are already TimeSeries it should be put in the # same order as the input, currently they are not. return data_header_unit_tuples, data_header_pairs, already_timeseries, filepaths
def _parse_args(self, *args, **kwargs): """ Parses an args list for data-header pairs. args can contain any mixture of the following entries: * tuples of (data, header, unit) (1) * data, header not in a tuple (1) * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. (1) Note that header/unit are optional and in either order, but data but be the first entry in each group. Example ------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_unit_tuples = list() data_header_pairs = list() already_timeseries = list() filepaths = list() # Take source kwarg if defined source = kwargs.get('source', None) # Account for nested lists of items. Simply outputs a single list of # items, nested lists are expanded to element level. args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header pair in a tuple if (isinstance(arg, (np.ndarray, Table, pd.DataFrame))):# and self._validate_meta(args[i+1])): # Assume a Pandas Dataframe is given data = arg units = OrderedDict() meta = MetaDict() # Convert the data argument into a Pandas DataFrame if needed. if isinstance(data, Table): # We have an AstroPy Table: data, meta, units = self._from_table(data) elif isinstance(data, np.ndarray): # We have a numpy ndarray. We assume the first column is a dt index data = pd.DataFrame(data=data[:,1:], index=Time(data[:,0])) # If there are 1 or 2 more arguments: for _ in range(2): if (len(args) > i+1): # If that next argument isn't data but is metaddata or units: if not isinstance(args[i+1], (np.ndarray, Table, pd.DataFrame)): if self._validate_units(args[i+1]): units.update(args[i+1]) i += 1 # an extra increment to account for the units elif self._validate_meta(args[i+1]): # if we have an astropy.io FITS header then convert # to preserve multi-line comments if isinstance(args[i+1], astropy.io.fits.header.Header): args[i+1] = MetaDict(sunpy.io.header.FileHeader(args[i+1])) meta.update(args[i+1]) i += 1 # an extra increment to account for the meta # Add a 3-tuple for this TimeSeries. data_header_unit_tuples.append((data, meta, units)) # Filepath elif (isinstance(arg, six.string_types) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) read, result = self._read_file(path, **kwargs) if read: data_header_pairs.append(result) else: filepaths.append(result) # Directory elif (isinstance(arg, six.string_types) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: # returns a boolean telling us if it were read and either a # tuple or the original filepath for reading by a source read, result = self._read_file(afile, **kwargs) if read: data_header_pairs.append(result) else: filepaths.append(result) # Glob elif (isinstance(arg, six.string_types) and '*' in arg): files = glob.glob(os.path.expanduser(arg)) for afile in files: # data_header_unit_tuples += self._read_file(afile, **kwargs) # returns a boolean telling us if it were read and either a # tuple or the original filepath for reading by a source read, result = self._read_file(afile, **kwargs) if read: data_header_pairs.append(result) else: filepaths.append(result) # Already a TimeSeries elif isinstance(arg, GenericTimeSeries): already_timeseries.append(arg) # A URL elif (isinstance(arg,six.string_types) and _is_url(arg)): default_dir = sunpy.config.get("downloads", "download_dir") url = arg path = download_file(url, default_dir) pairs = self._read_file(path, **kwargs) #data_header_pairs += pairs filepaths.append(pairs[1]) else: #raise ValueError("File not found or invalid input") raise NoMatchError("File not found or invalid input") i += 1 # TODO: # In the end, if there are already TimeSeries it should be put in the # same order as the input, currently they are not. return data_header_unit_tuples, data_header_pairs, already_timeseries, filepaths
def _parse_args(self, *args, **kwargs): """ Parses an args list for data-header pairs. args can contain any mixture of the following entries: * tuples of data,header * data, header not in a tuple * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. Example ------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_pairs = list() already_maps = list() # Account for nested lists of items args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header pair in a tuple if ((type(arg) in [tuple, list]) and len(arg) == 2 and isinstance(arg[0], np.ndarray) and isinstance(arg[1], dict)): data_header_pairs.append(arg) # Data-header pair not in a tuple elif (isinstance(arg, np.ndarray) and isinstance(args[i + 1], dict)): pair = (args[i], args[i + 1]) data_header_pairs.append(pair) i += 1 # an extra increment to account for the data-header pairing # File name elif (isinstance(arg, basestring) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # Directory elif (isinstance(arg, basestring) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Glob elif (isinstance(arg, basestring) and '*' in arg): files = glob.glob(os.path.expanduser(arg)) for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Already a Map elif isinstance(arg, GenericMap): already_maps.append(arg) # A URL elif (isinstance(arg, basestring) and _is_url(arg)): default_dir = sunpy.config.get("downloads", "download_dir") url = arg path = download_file(url, default_dir) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # A database Entry elif isinstance(arg, DatabaseEntry): data_header_pairs += self._read_file(arg.path, **kwargs) else: raise ValueError("File not found or invalid input") i += 1 #TODO: # In the end, if there are aleady maps it should be put in the same # order as the input, currently they are not. return data_header_pairs, already_maps
def _parse_url(self, request, **kwargs): path = download_file(request.full_url, get_and_create_download_dir()) return self._parse_path(pathlib.Path(path), **kwargs)
def _parse_args(self, *args, **kwargs): """ Parses an args list for data-header pairs. args can contain any mixture of the following entries: * tuples of data,header * data, header not in a tuple * data, wcs object in a tuple * data, wcs object not in a tuple * filename, which will be read * directory, from which all files will be read * glob, from which all files will be read * url, which will be downloaded and read * lists containing any of the above. Example ------- self._parse_args(data, header, (data, header), ['file1', 'file2', 'file3'], 'file4', 'directory1', '*.fits') """ data_header_pairs = list() already_maps = list() # Account for nested lists of items args = expand_list(args) # For each of the arguments, handle each of the cases i = 0 while i < len(args): arg = args[i] # Data-header or data-WCS pair if isinstance(arg, SUPPORTED_ARRAY_TYPES): arg_header = args[i+1] if isinstance(arg_header, WCS): arg_header = args[i+1].to_header() if self._validate_meta(arg_header): pair = (args[i], OrderedDict(arg_header)) data_header_pairs.append(pair) i += 1 # an extra increment to account for the data-header pairing # File name elif (isinstance(arg, str) and os.path.isfile(os.path.expanduser(arg))): path = os.path.expanduser(arg) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # Directory elif (isinstance(arg, str) and os.path.isdir(os.path.expanduser(arg))): path = os.path.expanduser(arg) files = [os.path.join(path, elem) for elem in os.listdir(path)] for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Glob elif (isinstance(arg, str) and '*' in arg): files = glob.glob(os.path.expanduser(arg)) for afile in files: data_header_pairs += self._read_file(afile, **kwargs) # Already a Map elif isinstance(arg, GenericMap): already_maps.append(arg) # A URL elif (isinstance(arg, str) and _is_url(arg)): url = arg path = download_file(url, get_and_create_download_dir()) pairs = self._read_file(path, **kwargs) data_header_pairs += pairs # A database Entry elif isinstance(arg, DatabaseEntry): data_header_pairs += self._read_file(arg.path, **kwargs) else: raise ValueError("File not found or invalid input") i += 1 # TODO: # In the end, if there are already maps it should be put in the same # order as the input, currently they are not. return data_header_pairs, already_maps