def entries_from_dir(fitsdir, recursive=False, pattern="*", default_waveunit=None): """Search the given directory for FITS files and use the corresponding FITS headers to generate instances of :class:`DatabaseEntry`. FITS files are detected by reading the content of each file, the `pattern` argument may be used to avoid reading entire directories if one knows that all FITS files have the same filename extension. Parameters ---------- fitsdir : string The directory where to look for FITS files. recursive : bool, optional If True, the given directory will be searched recursively. Otherwise, only the given directory and no subdirectories are searched. The default is `False`, i.e. the given directory is not searched recursively. pattern : string, optional The pattern can be used to filter the list of filenames before the files are attempted to be read. The default is to collect all files. This value is passed to the function :func:`fnmatch.filter`, see its documentation for more information on the supported syntax. default_waveunit : str, optional See :meth:`sunpy.database.tables.DatabaseEntry.add_fits_header_entries_from_file`. Returns ------- generator of (DatabaseEntry, str) pairs A generator where each item is a tuple consisting of a :class:`DatabaseEntry` instance and the absolute path to the filename which was used to make the database entry. Examples -------- >>> from sunpy.data.test import rootdir as fitsdir >>> from sunpy.database.tables import entries_from_dir >>> entries = list(entries_from_dir(fitsdir, default_waveunit='angstrom')) >>> len(entries) 38 >>> # and now search `fitsdir` recursive >>> entries = list(entries_from_dir(fitsdir, True, default_waveunit='angstrom')) >>> len(entries) 59 """ for dirpath, dirnames, filenames in os.walk(fitsdir): filename_paths = (os.path.join(dirpath, name) for name in filenames) for path in fnmatch.filter(filename_paths, pattern): try: filetype = sunpy_filetools._detect_filetype(path) except (sunpy_filetools.UnrecognizedFileTypeError, sunpy_filetools.InvalidJPEG2000FileExtension): continue if filetype == "fits": for entry in entries_from_file(path, default_waveunit): yield entry, path if not recursive: break
def entries_from_dir(fitsdir, recursive=False, pattern='*', default_waveunit=None, time_string_parse_format=None): """Search the given directory for FITS files and use the corresponding FITS headers to generate instances of :class:`DatabaseEntry`. FITS files are detected by reading the content of each file, the `pattern` argument may be used to avoid reading entire directories if one knows that all FITS files have the same filename extension. Parameters ---------- fitsdir : string The directory where to look for FITS files. recursive : bool, optional If True, the given directory will be searched recursively. Otherwise, only the given directory and no subdirectories are searched. The default is `False`, i.e. the given directory is not searched recursively. pattern : string, optional The pattern can be used to filter the list of filenames before the files are attempted to be read. The default is to collect all files. This value is passed to the function :func:`fnmatch.filter`, see its documentation for more information on the supported syntax. default_waveunit : str, optional See :meth:`sunpy.database.tables.DatabaseEntry.add_fits_header_entries_from_file`. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Returns ------- generator of (DatabaseEntry, str) pairs A generator where each item is a tuple consisting of a :class:`DatabaseEntry` instance and the absolute path to the filename which was used to make the database entry. Examples -------- >>> import os >>> from sunpy.data.test import rootdir as fitsdir >>> from sunpy.database.tables import entries_from_dir >>> eitdir = os.path.join(fitsdir, 'EIT') >>> entries = list(entries_from_dir(eitdir, default_waveunit='angstrom')) >>> len(entries) 13 """ for dirpath, dirnames, filenames in os.walk(fitsdir): filename_paths = (os.path.join(dirpath, name) for name in filenames) for path in fnmatch.filter(filename_paths, pattern): try: filetype = sunpy_filetools._detect_filetype(path) except (sunpy_filetools.UnrecognizedFileTypeError, sunpy_filetools.InvalidJPEG2000FileExtension): continue if filetype == 'fits': for entry in entries_from_file( path, default_waveunit, time_string_parse_format=time_string_parse_format): yield entry, path if not recursive: break
def entries_from_dir(fitsdir, recursive=False, pattern='*', default_waveunit=None): """Search the given directory for FITS files and use the corresponding FITS headers to generate instances of :class:`DatabaseEntry`. FITS files are detected by reading the content of each file, the `pattern` argument may be used to avoid reading entire directories if one knows that all FITS files have the same filename extension. Parameters ---------- fitsdir : string The directory where to look for FITS files. recursive : bool, optional If True, the given directory will be searched recursively. Otherwise, only the given directory and no subdirectories are searched. The default is `False`, i.e. the given directory is not searched recursively. pattern : string, optional The pattern can be used to filter the list of filenames before the files are attempted to be read. The default is to collect all files. This value is passed to the function :func:`fnmatch.filter`, see its documentation for more information on the supported syntax. default_waveunit : str, optional See :meth:`sunpy.database.tables.DatabaseEntry.add_fits_header_entries_from_file`. Returns ------- generator of (DatabaseEntry, str) pairs A generator where each item is a tuple consisting of a :class:`DatabaseEntry` instance and the absolute path to the filename which was used to make the database entry. Examples -------- >>> from pprint import pprint >>> from sunpy.data.test import rootdir as fitsdir >>> entries = list(entries_from_dir(fitsdir)) >>> len(entries) 2 >>> # and now search `fitsdir` recursive >>> entries = list(entries_from_dir(fitsdir, True)) >>> len(entries) 15 >>> # print the first 5 items of the FITS header of the first found file >>> first_entry, filename = entries[0] >>> pprint(first_entry.fits_header_entries[:5]) [<FitsHeaderEntry(id None, key 'SIMPLE', value True)>, <FitsHeaderEntry(id None, key 'BITPIX', value -64)>, <FitsHeaderEntry(id None, key 'NAXIS', value 2)>, <FitsHeaderEntry(id None, key 'NAXIS1', value 128)>, <FitsHeaderEntry(id None, key 'NAXIS2', value 128)>] """ for dirpath, dirnames, filenames in os.walk(fitsdir): filename_paths = (os.path.join(dirpath, name) for name in filenames) for path in fnmatch.filter(filename_paths, pattern): try: filetype = sunpy_filetools._detect_filetype(path) except (sunpy_filetools.UnrecognizedFileTypeError, sunpy_filetools.InvalidJPEG2000FileExtension): continue if filetype == 'fits': for entry in entries_from_file(path, default_waveunit): yield entry, path if not recursive: break