def _build_logfile_name(self) -> str:
"""
Builds the logging directory, and create the directory if necessary.
:return: (str) Absolute path to the logging directory
"""
logfile_name = AppLogging.build_logfile_name(cfg_info=self.app_cfg)
log_dir = os.path.abspath(
os.path.sep.join(logfile_name.split(os.path.sep)[0:-1]))
utils.check_if_location_exists(location=log_dir, create_dir=True)
return logfile_name
def build_json_log_location(cfg: configparser.ConfigParser) -> str:
"""
Build the location of the JSON files from the configuration file.
:param cfg: Instantiated ConfigParser (containing specified CFG file)
:return: (str) Location of <data>.json files
"""
# ======================
# JSON LOGFILE LOCATION
# ======================
json_log_location = cfg.get(AppCfgFileSections.LOGGING,
AppCfgFileSectionKeys.JSON_FILE_DIR)
json_drive = cfg.get(AppCfgFileSections.LOGGING,
AppCfgFileSectionKeys.LOG_DRIVE_LETTER)
if json_drive is not None:
json_log_location = f"{json_drive.strip(':')}:{json_log_location}"
# Verify directory exists
LOG.info(f"Checking directory: {json_log_location}")
if not utils.check_if_location_exists(location=json_log_location, create_dir=False):
LOG.error(f"Unable to find source directory: {json_log_location}")
exit()
return json_log_location
def _build_image_download_dir(self) -> str:
"""
Builds the image download directory, and create the directory if necessary.
:return: (str) Absolute path to the DL directory
"""
dl_dir = self.app_cfg.get(AppCfgFileSections.STORAGE,
AppCfgFileSectionKeys.LOCAL_DIR)
dl_drive = self.app_cfg.get(AppCfgFileSections.STORAGE,
AppCfgFileSectionKeys.LOCAL_DRIVE_LETTER)
if dl_drive not in [None, '']:
dl_dir = f"{dl_drive.strip(':')}:{dl_dir}"
dl_dir = os.path.abspath(dl_dir)
utils.check_if_location_exists(location=dl_dir, create_dir=True)
return dl_dir
def _build_pickle_filename(self) -> str:
"""
Builds the general inventory pickled (binary) data file.
:return: (str) Absolute path to the pickled (binary) data file name.
"""
pickle_location = self.json_log_location
pickle_filename = "{0}{1}".format(
self.engine_cfg.get(ProjectCfgFileSections.PYTHON_PROJECT,
ProjectCfgFileSectionKeys.NAME).upper(),
PICKLE_EXT)
pickle_filename = os.path.abspath(
os.path.sep.join([pickle_location, pickle_filename]))
utils.check_if_location_exists(location=pickle_location,
create_dir=True)
return pickle_filename
def _build_json_log_location(self) -> str:
"""
Builds the JSON inventory logging directory, and create the directory if necessary.
:return: (str) Absolute path to the JSON inventory logging directory
"""
json_log_location = self.app_cfg.get(
AppCfgFileSections.LOGGING, AppCfgFileSectionKeys.JSON_FILE_DIR)
json_drive = self.app_cfg.get(AppCfgFileSections.LOGGING,
AppCfgFileSectionKeys.LOG_DRIVE_LETTER)
if json_drive not in [None, '']:
json_log_location = f"{json_drive.strip(':')}:{json_log_location}"
json_log_location = os.path.abspath(json_log_location)
utils.check_if_location_exists(location=json_log_location,
create_dir=True)
return json_log_location
def configure_logging(cls, app_cfg_obj: PdlConfig) -> Logger:
"""
Sets up the root logger (format, location) and updates the existing the handlers
:param app_cfg_obj: (PdlConfig): Contains the required location, etc., for the logger
:return: Instantiated, updated root logger
"""
# Set log level (CLI overrides the config file)
# ---------------------------------------------
# Get the config-file specified value
log_level = app_cfg_obj.app_cfg.get(
AppCfgFileSections.LOGGING,
AppCfgFileSectionKeys.LOG_LEVEL.lower())
if app_cfg_obj.cli_args.debug:
log_level = 'debug'
# Determine and store the log file name, create directory if needed.
log_dir = os.path.abspath(
os.path.sep.join(
app_cfg_obj.logfile_name.split(os.path.sep)[0:-1]))
utils.check_if_location_exists(location=log_dir, create_dir=True)
# Setup the root logger for the app
logger = Logger(filename=app_cfg_obj.logfile_name,
default_level=Logger.STR_TO_VAL[log_level],
project=app_cfg_obj.app_cfg.get(
AppCfgFileSections.PROJECT,
AppCfgFileSectionKeys.NAME),
set_root=True)
# Show defined loggers and log levels
logger.debug(f"Log File: {app_cfg_obj.logfile_name}")
for line in logger.list_loggers().split('\n'):
logger.debug(line)
return logger
def _build_temp_storage(self) -> str:
"""
Builds the temp (local) file storage directory.
:return: (str) Absolute path to the temp (local) file storage directory.
"""
storage_location = self.app_cfg.get(
AppCfgFileSections.STORAGE,
AppCfgFileSectionKeys.TEMP_STORAGE_PATH)
storage_drive = self.app_cfg.get(
AppCfgFileSections.STORAGE,
AppCfgFileSectionKeys.TEMP_STORAGE_DRIVE)
if storage_drive not in [None, '']:
storage_location = f"{storage_drive.strip(':')}:{storage_location}"
storage_location = os.path.abspath(storage_location)
utils.check_if_location_exists(location=storage_location,
create_dir=True)
return storage_location
def test_check_if_location_exists_and_does_exist(
self):
path_dirs = ['tmp', 'does', 'not', 'exist']
path = os.path.sep.join(path_dirs)
create_dir = True
os.makedirs(path)
result = check_if_location_exists(
location=path, create_dir=create_dir)
try:
os.removedirs(path)
except OSError:
pass # Found a directory that was not empty
assert result is True
assert os.path.exists(path) is False
def test_check_if_location_exists_but_does_not_exist_cannot_create(self):
with patch('PDL.logger.utils.os.makedirs', side_effect=Exception()) as make_dirs_mock:
path_dirs = ['tmp', 'does', 'not', 'exist']
path = os.path.sep.join(path_dirs)
create_dir = True
if os.path.exists(path):
os.removedirs(path)
result = check_if_location_exists(location=path, create_dir=create_dir)
try:
os.removedirs(path)
except OSError:
pass # Found a directory that was not empty
assert result is False
assert make_dirs_mock.call_count == 1
assert os.path.exists(path) is False
def write_file(self,
urls: List[str],
location: str,
filename: Optional[str] = None,
create_dir: bool = False) -> str:
"""
Write the URL list to file. Human readable, but machine parse-able.
:param urls: (list) List of URLs to record
:param location: (str) Location to store url file
:param filename: (str) Name of file (use timestamp if none specified)
:param create_dir: (bool) Create the location if it does not exist
(default = False)
:return: (str) Name and path to file
"""
# Check if location exists, create if requested
if not utils.check_if_location_exists(location=location,
create_dir=create_dir):
return ''
# Create file name
if filename is None:
timestamp = datetime.datetime.now().strftime(self.TIMESTAMP)
filename = f'{timestamp}.{self.EXTENSION}'
filespec = os.path.abspath(os.path.join(location, filename))
# Write URLs to file
LOG.debug(f"Writing url input to file: {filespec}")
with open(filespec, 'w') as url_file:
url_file.write("URL LIST:\n")
for index, url in enumerate(urls):
url_file.write(f"{index + 1:>3}) {url}\n")
url_file.write(f"\n{self.URL_LIST_DELIM}:\n")
url_file.write(f"{self.URL_DELIM.join(urls)}\n")
LOG.info(
f"Wrote urls to the input file: {filespec} --> ({len(urls)} urls)")
return filespec