def initialize(): logging.basicConfig(level=logging.DEBUG, format=LOGGER.fmt, datefmt=LOGGER.datefmt) set_excepthook([logger_excepthook]) args = get_args() if args.log_folder is not None: log_dir = os.path.abspath(args.log_folder) os.makedirs(args.log_folder, exist_ok=True) log_file_name = config.log_file_name.format( datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"), hostname=gethostname()) log_file = os.path.join(log_dir, log_file_name) LOGGER.set_log_file(log_file) input_dir = os.path.abspath(args.input_folder) output_dir = os.path.abspath(args.output_folder) os.makedirs(args.output_folder, exist_ok=True) tree_walker = TreeWalker(input_dir, [output_dir], skip_webp=False, precompute_paths=True) return tree_walker
def initialize(): logging.basicConfig(level=logging.DEBUG, format=LOGGER.fmt, datefmt=LOGGER.datefmt) set_excepthook([logger_excepthook]) args = get_args() if args.log_folder is not None: os.makedirs(args.log_folder, exist_ok=True) log_file_name = config.log_file_name.format( datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"), hostname=gethostname()) log_file = os.path.join(args.log_folder, log_file_name) LOGGER.set_log_file(log_file) tree_walker = TreeWalker(args.input_dir, [], skip_webp=False, precompute_paths=True, ext="json") database_client = DatabaseClient( table_name=args.table_name, max_n_accumulated_rows=config.db_max_n_accumulated_rows, max_n_errors=config.db_max_n_errors, max_cache_size=config.db_max_cache_size, enable_cache=False) return tree_walker, database_client
def initialize(): """ Get command line arguments, and initialize the TreeWalker and Masker. :return: Command line arguments, an instance of `TreeWalker` initialized at the specified directories, and an instance of `Masker` ready for masking. :rtype: argparse.Namespace, TreeWalker, Masker """ # Register the logging excepthook except_hooks = [logger_excepthook] if config.uncaught_exception_email: # Register a custom excepthook which sends an email on uncaught exceptions. from src.email_sender import email_excepthook except_hooks.append(email_excepthook) # Set the exception hook(s) set_excepthook(except_hooks) # Get arguments args = get_args() # Check that the config and command line arguments are valid check_config(args) # Configure logger logging.basicConfig(level=getattr(logging, config.log_level), format=LOGGER.fmt, datefmt=LOGGER.datefmt) # Set log file if args.log_folder is not None: os.makedirs(args.log_folder, exist_ok=True) log_file_name = config.log_file_name.format( datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"), hostname=gethostname()) log_file = os.path.join(args.log_folder, log_file_name) LOGGER.set_log_file(log_file) # Log the call LOGGER.info(__name__, f"Call: {' '.join(sys.argv)}") # Log the current config. LOGGER.info(__name__, "\n" + config_string()) if args.clear_cache: # Clear any cached files clear_cache() # Clear the database cache if database writing is enabled if config.write_exif_to_db: from src.db.DatabaseClient import clear_db_cache clear_db_cache() # Get the absolute path of the directories base_input_dir = os.path.abspath(args.input_folder) base_output_dir = os.path.abspath(args.output_folder) mirror_dirs = [base_output_dir] # Make the output directory os.makedirs(base_output_dir, exist_ok=True) if args.archive_folder is not None: base_archive_dir = os.path.abspath(args.archive_folder) mirror_dirs.append(base_archive_dir) # Make the archive directory os.makedirs(base_archive_dir, exist_ok=True) # Make the cache directory os.makedirs(config.CACHE_DIRECTORY, exist_ok=True) # Configure the logger LOGGER.base_input_dir = base_input_dir LOGGER.base_output_dir = base_output_dir # Initialize the walker tree_walker = TreeWalker(base_input_dir, mirror_dirs, skip_webp=(not config.force_remask), precompute_paths=(not config.lazy_paths)) # Initialize the masker masker = Masker(mask_dilation_pixels=config.mask_dilation_pixels, max_num_pixels=config.max_num_pixels) # Create the TensorFlow datatset dataset_iterator = iter(get_tf_dataset(tree_walker)) # Initialize the ImageProcessor image_processor = ImageProcessor( masker=masker, max_num_async_workers=config.max_num_async_workers) return args, tree_walker, image_processor, dataset_iterator