def test_search_config_validate_invalid_city(keywords, province_or_state,
locale, providers, in_city):
cfg = SearchConfig(keywords,
province_or_state,
locale,
providers,
city=in_city)
with pytest.raises(AssertionError, match="City not set"):
cfg.validate()
def test_search_config_validate_remoteness(keywords, province_or_state, locale,
providers, in_city, in_remoteness):
cfg = SearchConfig(keywords,
province_or_state,
locale,
providers,
city=in_city,
remoteness=in_remoteness)
with pytest.raises(AssertionError, match="Remoteness is UNKNOWN!"):
cfg.validate()
def test_search_config_validate_domain(keywords, province_or_state, locale,
providers, in_city):
cfg = SearchConfig(keywords,
province_or_state,
locale,
providers,
city=in_city)
# We have to force an invalid domain because the constructor ensures that it is valid.
cfg.domain = None
with pytest.raises(AssertionError, match="Domain not set"):
cfg.validate()
def test_search_config_validate_invalid_max_posting_days(
keywords, province_or_state, locale, providers, in_city,
in_max_listing_days):
cfg = SearchConfig(keywords,
province_or_state,
locale,
providers,
city=in_city,
max_listing_days=in_max_listing_days)
with pytest.raises(AssertionError,
match="Cannot set max posting days < 1"):
cfg.validate()
def get_config_manager(config: Dict[str, Any]) -> JobFunnelConfigManager:
"""Method to build JobFunnelConfigManager from a config dictionary
"""
# Build JobFunnelConfigManager
search_cfg = SearchConfig(
keywords=config['search']['keywords'],
province_or_state=config['search']['province_or_state'],
city=config['search']['city'],
distance_radius=config['search']['radius'],
return_similar_results=config['search']['similar_results'],
max_listing_days=config['search']['max_listing_days'],
blocked_company_names=config['search']['company_block_list'],
locale=Locale[config['search']['locale']],
providers=[Provider[p] for p in config['search']['providers']],
remoteness=Remoteness[config['search']['remoteness']],
)
delay_cfg = DelayConfig(
max_duration=config['delay']['max_duration'],
min_duration=config['delay']['min_duration'],
algorithm=DelayAlgorithm[config['delay']['algorithm']],
random=config['delay']['random'],
converge=config['delay']['converging'],
)
if config.get('proxy'):
proxy_cfg = ProxyConfig(
protocol=config['proxy']['protocol'],
ip_address=config['proxy']['ip'],
port=config['proxy']['port'],
)
else:
proxy_cfg = None
funnel_cfg_mgr = JobFunnelConfigManager(
master_csv_file=config['master_csv_file'],
user_block_list_file=config['block_list_file'],
duplicates_list_file=config['duplicates_list_file'],
cache_folder=config['cache_folder'],
log_file=config['log_file'],
log_level=config['log_level'],
no_scrape=config['no_scrape'],
search_config=search_cfg,
delay_config=delay_cfg,
proxy_config=proxy_cfg,
)
return funnel_cfg_mgr
def test_search_config_init(mocker, locale, domain, exp_domain):
"""Make sure the init functions as we expect wrt to domain selection
"""
# FUT
if not locale:
# test our error
with pytest.raises(ValueError, match=r"Unknown domain for locale.*"):
cfg = SearchConfig(
keywords=mocker.Mock(),
province_or_state=mocker.Mock(),
locale=-1, # AKA an unknown Enum entry to Locale
providers=mocker.Mock(),
)
else:
cfg = SearchConfig(
keywords=mocker.Mock(),
province_or_state=mocker.Mock(),
locale=locale,
domain=domain,
providers=mocker.Mock(),
)
# Assertions
assert cfg.domain == exp_domain
def test_search_config_query_string(mocker, keywords, exp_query_str):
"""Test that search config can build keyword query string correctly.
"""
cfg = SearchConfig(
keywords=keywords,
province_or_state=mocker.Mock(),
locale=Locale.CANADA_FRENCH,
providers=mocker.Mock(),
)
# FUT
query_str = cfg.query_string
# Assertions
assert query_str == exp_query_str
