def directory_to_streams(directory, config=None): """Read in a directory of data to a list of streams. Note: If the directory only includes files that are readable by this library then the task is rather simple. However, often times data directories include random subdirectories and/or zip files, which we try to crawl in a sensible fashion. Args: directory (str): Directory of ground motion files (streams). config (dict): Configuration options. Returns: tuple: (List of obspy streams, List of unprocessed files, List of errors associated with trying to read unprocessed files). """ # Use a temp dir so that we don't modify data on disk since that may not be # expected or desired in all cases. We create the temporary directory in # the parent directory, which permits using shutil.copytree to duplicate # the data prior to processing. intermediate_dir = tempfile.mkdtemp(dir=os.path.dirname(directory)) temp_dir = os.path.join(intermediate_dir, "directory_to_streams") try: shutil.copytree(directory, temp_dir) flatten_directory(temp_dir) # --------------------------------------------------------------------- # Read streams # --------------------------------------------------------------------- streams = [] unprocessed_files = [] unprocessed_file_errors = [] for file_path in glob.glob(os.path.join(temp_dir, "*")): file_name = os.path.basename(file_path) file_ext = os.path.splitext(file_name)[1].lower() if file_ext not in EXT_IGNORE: try: logging.debug(f"Attempting to read: {file_path}") streams += read_data(file_path, config=config) except BaseException as ex: logging.info(f"Failed to read file: {file_name}") unprocessed_files += [file_path] unprocessed_file_errors += [ex] except BaseException as e: raise e finally: try: shutil.rmtree(intermediate_dir) except OSError: shutil.rmtree(intermediate_dir) return streams, unprocessed_files, unprocessed_file_errors
def directory_to_streams(directory): """Read in a directory of data to a list of streams. Note: If the directory only includes files that are readable by this library then the task is rather simple. However, often times data directories include random subdirectories and/or zip files, which we try to crawl in a sensible fashion. Args: directory (str): Directory of ground motion files (streams). Returns: tuple: (List of obspy streams, List of unprocessed files, List of errors associated with trying to read unprocessed files). """ # Use a temp dir so that we don't modify data on disk since that may not be # expected or desired in all cases. temp_dir = os.path.join(tempfile.mkdtemp(), 'directory_to_streams') try: shutil.copytree(directory, temp_dir) flatten_directory(temp_dir) # ------------------------------------------------------------------------- # Read streams # ------------------------------------------------------------------------- streams = [] unprocessed_files = [] unprocessed_file_errors = [] for file_path in glob.glob(os.path.join(temp_dir, "*")): file_ext = os.path.splitext(file_path)[1].lower() if file_ext not in EXT_IGNORE: try: logging.debug('Attempting to read: %s' % file_path) streams += read_data(file_path) except Exception as ex: unprocessed_files += [file_path] unprocessed_file_errors += [ex] except Exception as e: raise e finally: shutil.rmtree(temp_dir) return streams, unprocessed_files, unprocessed_file_errors
def directory_to_streams(directory): """Read in a directory of data to a list of streams. Note: If the directory only includes files that are readable by this library then the task is rather simple. However, often times data directories include random subdirectories and/or zip files, which we try to crawl in a sensible fashion. Args: directory (str): Directory of ground motion files (streams). Returns: tuple: (List of obspy streams, List of unprocessed files, List of errors associated with trying to read unprocessed files). """ # Use a temp dir so that we don't modify data on disk since that may not be # expected or desired in all cases. temp_dir = os.path.join(tempfile.mkdtemp(), 'directory_to_streams') try: shutil.copytree(directory, temp_dir) flatten_directory(temp_dir) # ------------------------------------------------------------------------- # Read streams # ------------------------------------------------------------------------- streams = [] unprocessed_files = [] unprocessed_file_errors = [] for file_path in glob.glob(os.path.join(temp_dir, "*")): file_ext = os.path.splitext(file_path)[1].lower() if file_ext not in EXT_IGNORE: try: logging.debug('Attempting to read: %s' % file_path) streams += read_data(file_path) except Exception as ex: unprocessed_files += [file_path] unprocessed_file_errors += [ex] except Exception as e: raise e finally: shutil.rmtree(temp_dir) return streams, unprocessed_files, unprocessed_file_errors