def test_load_tar_file(tmp_path):
"""Test that load_profile does not delete or modify the tar file."""
tar_path = Path("profile.tar.gz")
profile_path = tmp_path / "browser_profile"
browser_params = BrowserParamsInternal(browser_id=1)
modified_time_before_load = tar_path.stat().st_mtime
load_profile(profile_path, browser_params, tar_path)
assert modified_time_before_load == tar_path.stat().st_mtime
def __init__(
self,
manager_params_temp: ManagerParams,
browser_params_temp: List[BrowserParams],
structured_storage_provider: StructuredStorageProvider,
unstructured_storage_provider: Optional[UnstructuredStorageProvider],
logger_kwargs: Dict[Any, Any] = {},
) -> None:
"""Initialize the TaskManager with browser and manager config params
Parameters
----------
manager_params_temp : ManagerParams
TaskManager configuration parameters
browser_params_temp : list of BrowserParams
Browser configuration parameters. It is a list which
includes individual configurations for each browser.
logger_kwargs : dict, optional
Keyword arguments to pass to MPLogger on initialization.
"""
validate_crawl_configs(manager_params_temp, browser_params_temp)
manager_params = ManagerParamsInternal.from_dict(manager_params_temp.to_dict())
browser_params = [
BrowserParamsInternal.from_dict(bp.to_dict()) for bp in browser_params_temp
]
manager_params.screenshot_path = manager_params.data_directory / "screenshots"
manager_params.source_dump_path = manager_params.data_directory / "sources"
self.manager_params = manager_params
self.browser_params = browser_params
self._logger_kwargs = logger_kwargs
# Create data directories if they do not exist
if not os.path.exists(manager_params.screenshot_path):
os.makedirs(manager_params.screenshot_path)
if not os.path.exists(manager_params.source_dump_path):
os.makedirs(manager_params.source_dump_path)
# Check size of parameter dictionary
self.num_browsers = manager_params.num_browsers
# Parse and flesh out js_instrument_settings
for a_browsers_params in self.browser_params:
js_settings = a_browsers_params.js_instrument_settings
cleaned_js_settings = clean_js_instrumentation_settings(js_settings)
a_browsers_params.cleaned_js_instrument_settings = cleaned_js_settings
# Flow control
self.closing = False
self.failure_status: Optional[Dict[str, Any]] = None
self.threadlock = threading.Lock()
self.failure_count = 0
self.failure_limit = manager_params.failure_limit
# Start logging server thread
self.logging_server = MPLogger(
self.manager_params.log_path,
str(structured_storage_provider),
**self._logger_kwargs
)
self.manager_params.logger_address = self.logging_server.logger_address
self.logger = logging.getLogger("openwpm")
# Initialize the storage controller
self._launch_storage_controller(
structured_storage_provider, unstructured_storage_provider
)
# Sets up the BrowserManager(s) + associated queues
self.browsers = self._initialize_browsers(browser_params)
self._launch_browsers()
# Start the manager watchdog
thread = threading.Thread(target=self._manager_watchdog, args=())
thread.daemon = True
thread.name = "OpenWPM-watchdog"
thread.start()
# Save crawl config information to database
openwpm_v, browser_v = get_version()
self.storage_controller_handle.save_configuration(
manager_params, browser_params, openwpm_v, browser_v
)
self.logger.info(
get_configuration_string(
self.manager_params, browser_params, (openwpm_v, browser_v)
)
)
self.unsaved_command_sequences: Dict[int, CommandSequence] = dict()
self.callback_thread = threading.Thread(
target=self._mark_command_sequences_complete, args=()
)
self.callback_thread.name = "OpenWPM-completion_handler"
self.callback_thread.start()