def initialize():
logging.basicConfig(level=logging.DEBUG,
format=LOGGER.fmt,
datefmt=LOGGER.datefmt)
set_excepthook([logger_excepthook])
args = get_args()
if args.log_folder is not None:
log_dir = os.path.abspath(args.log_folder)
os.makedirs(args.log_folder, exist_ok=True)
log_file_name = config.log_file_name.format(
datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"),
hostname=gethostname())
log_file = os.path.join(log_dir, log_file_name)
LOGGER.set_log_file(log_file)
input_dir = os.path.abspath(args.input_folder)
output_dir = os.path.abspath(args.output_folder)
os.makedirs(args.output_folder, exist_ok=True)
tree_walker = TreeWalker(input_dir, [output_dir],
skip_webp=False,
precompute_paths=True)
return tree_walker
def initialize():
logging.basicConfig(level=logging.DEBUG,
format=LOGGER.fmt,
datefmt=LOGGER.datefmt)
set_excepthook([logger_excepthook])
args = get_args()
if args.log_folder is not None:
os.makedirs(args.log_folder, exist_ok=True)
log_file_name = config.log_file_name.format(
datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"),
hostname=gethostname())
log_file = os.path.join(args.log_folder, log_file_name)
LOGGER.set_log_file(log_file)
tree_walker = TreeWalker(args.input_dir, [],
skip_webp=False,
precompute_paths=True,
ext="json")
database_client = DatabaseClient(
table_name=args.table_name,
max_n_accumulated_rows=config.db_max_n_accumulated_rows,
max_n_errors=config.db_max_n_errors,
max_cache_size=config.db_max_cache_size,
enable_cache=False)
return tree_walker, database_client
def initialize():
"""
Get command line arguments, and initialize the TreeWalker and Masker.
:return: Command line arguments, an instance of `TreeWalker` initialized at the specified directories, and an
instance of `Masker` ready for masking.
:rtype: argparse.Namespace, TreeWalker, Masker
"""
# Register the logging excepthook
except_hooks = [logger_excepthook]
if config.uncaught_exception_email:
# Register a custom excepthook which sends an email on uncaught exceptions.
from src.email_sender import email_excepthook
except_hooks.append(email_excepthook)
# Set the exception hook(s)
set_excepthook(except_hooks)
# Get arguments
args = get_args()
# Check that the config and command line arguments are valid
check_config(args)
# Configure logger
logging.basicConfig(level=getattr(logging, config.log_level),
format=LOGGER.fmt,
datefmt=LOGGER.datefmt)
# Set log file
if args.log_folder is not None:
os.makedirs(args.log_folder, exist_ok=True)
log_file_name = config.log_file_name.format(
datetime=datetime.now().strftime("%Y-%m-%d_%H%M%S"),
hostname=gethostname())
log_file = os.path.join(args.log_folder, log_file_name)
LOGGER.set_log_file(log_file)
# Log the call
LOGGER.info(__name__, f"Call: {' '.join(sys.argv)}")
# Log the current config.
LOGGER.info(__name__, "\n" + config_string())
if args.clear_cache:
# Clear any cached files
clear_cache()
# Clear the database cache if database writing is enabled
if config.write_exif_to_db:
from src.db.DatabaseClient import clear_db_cache
clear_db_cache()
# Get the absolute path of the directories
base_input_dir = os.path.abspath(args.input_folder)
base_output_dir = os.path.abspath(args.output_folder)
mirror_dirs = [base_output_dir]
# Make the output directory
os.makedirs(base_output_dir, exist_ok=True)
if args.archive_folder is not None:
base_archive_dir = os.path.abspath(args.archive_folder)
mirror_dirs.append(base_archive_dir)
# Make the archive directory
os.makedirs(base_archive_dir, exist_ok=True)
# Make the cache directory
os.makedirs(config.CACHE_DIRECTORY, exist_ok=True)
# Configure the logger
LOGGER.base_input_dir = base_input_dir
LOGGER.base_output_dir = base_output_dir
# Initialize the walker
tree_walker = TreeWalker(base_input_dir,
mirror_dirs,
skip_webp=(not config.force_remask),
precompute_paths=(not config.lazy_paths))
# Initialize the masker
masker = Masker(mask_dilation_pixels=config.mask_dilation_pixels,
max_num_pixels=config.max_num_pixels)
# Create the TensorFlow datatset
dataset_iterator = iter(get_tf_dataset(tree_walker))
# Initialize the ImageProcessor
image_processor = ImageProcessor(
masker=masker, max_num_async_workers=config.max_num_async_workers)
return args, tree_walker, image_processor, dataset_iterator