async def generate_example_db(
request: pyd_models.GenerateRequest,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
):
"""
Creates and uploads Example-Data to the Database for testing purposes.
Includes Fetcher, FQDNs and URLs.
- **fetcher_amount** (default: 3): Number of Fetcher to generate
- **fqdn_amount** (default: 20): Number of Web Sites to generate
- **min_url_amount** (default: 10): Minimum Pages per Web Site
- **max_url_amount** (default: 100): Maximum Pages per Web Site
- **visited_ratio** (default: 0.0): Pages which have been visited
- **connection_amount** (default: 0): Amount of incoming Connections per Page
- **fixed_crawl_delay** (default: None): Adjust the Crawl Delay for all Web Sites.
Will be a distributed-randomized Value when no Value is chosen.
"""
if request.min_url_amount > request.max_url_amount:
http_es.raise_http_400(request.min_url_amount, request.max_url_amount)
background_tasks.add_task(
sample_generator.create_sample_fetcher,
db,
amount=request.fetcher_amount,
)
background_tasks.add_task(sample_generator.create_sample_frontier, db,
request)
if database.fqdn_hash_activated(db):
background_tasks.add_task(database.refresh_fqdn_hashes, db)
return Response(status_code=status.HTTP_202_ACCEPTED)
def create_fetcher_settings(
request: pyd_models.FetcherSettings,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
):
"""
Returns the latest settings for every fetcher
"""
updated_fetcher_settings = frontier.set_fetcher_settings(request, db)
if database.fqdn_hash_activated(db):
background_tasks.add_task(database.refresh_fqdn_hashes, db)
return updated_fetcher_settings
def delete_fetcher(
fetcher: pyd_models.DeleteFetcher,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
):
"""
Delete a specific Fetcher
- **uuid**: UUID of the fetcher, which has to be deleted
"""
fetchers.delete_fetcher(db, fetcher)
if database.fqdn_hash_activated(db):
background_tasks.add_task(database.refresh_fqdn_hashes, db)
return Response(status_code=status.HTTP_204_NO_CONTENT)
def register_fetcher(
fetcher: pyd_models.CreateFetcher,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
):
"""
Create a Fetcher
- **contact**: The e-mail address of the fetchers owner
- **name**: A unique name for the fetcher per contact
- **location** (optional): The location where the fetcher resides
- **pref_tld** (optional): The Top-Level-Domain, which the fetcher prefers to crawl
"""
new_fetcher = fetchers.create_fetcher(db, fetcher)
if database.fqdn_hash_activated(db):
background_tasks.add_task(database.refresh_fqdn_hashes, db)
return new_fetcher
def test_consistent_hash_deactivated():
rest.reset_long_term_part_strategy()
assert database.fqdn_hash_activated(db) is False
def test_fqdn_hash_activated():
rest.activate_fqdn_hash()
assert database.fqdn_hash_activated(db) is True