async def generate_example_db( request: pyd_models.GenerateRequest, background_tasks: BackgroundTasks, db: Session = Depends(get_db), ): """ Creates and uploads Example-Data to the Database for testing purposes. Includes Fetcher, FQDNs and URLs. - **fetcher_amount** (default: 3): Number of Fetcher to generate - **fqdn_amount** (default: 20): Number of Web Sites to generate - **min_url_amount** (default: 10): Minimum Pages per Web Site - **max_url_amount** (default: 100): Maximum Pages per Web Site - **visited_ratio** (default: 0.0): Pages which have been visited - **connection_amount** (default: 0): Amount of incoming Connections per Page - **fixed_crawl_delay** (default: None): Adjust the Crawl Delay for all Web Sites. Will be a distributed-randomized Value when no Value is chosen. """ if request.min_url_amount > request.max_url_amount: http_es.raise_http_400(request.min_url_amount, request.max_url_amount) background_tasks.add_task( sample_generator.create_sample_fetcher, db, amount=request.fetcher_amount, ) background_tasks.add_task(sample_generator.create_sample_frontier, db, request) if database.fqdn_hash_activated(db): background_tasks.add_task(database.refresh_fqdn_hashes, db) return Response(status_code=status.HTTP_202_ACCEPTED)
def create_fetcher_settings( request: pyd_models.FetcherSettings, background_tasks: BackgroundTasks, db: Session = Depends(get_db), ): """ Returns the latest settings for every fetcher """ updated_fetcher_settings = frontier.set_fetcher_settings(request, db) if database.fqdn_hash_activated(db): background_tasks.add_task(database.refresh_fqdn_hashes, db) return updated_fetcher_settings
def delete_fetcher( fetcher: pyd_models.DeleteFetcher, background_tasks: BackgroundTasks, db: Session = Depends(get_db), ): """ Delete a specific Fetcher - **uuid**: UUID of the fetcher, which has to be deleted """ fetchers.delete_fetcher(db, fetcher) if database.fqdn_hash_activated(db): background_tasks.add_task(database.refresh_fqdn_hashes, db) return Response(status_code=status.HTTP_204_NO_CONTENT)
def register_fetcher( fetcher: pyd_models.CreateFetcher, background_tasks: BackgroundTasks, db: Session = Depends(get_db), ): """ Create a Fetcher - **contact**: The e-mail address of the fetchers owner - **name**: A unique name for the fetcher per contact - **location** (optional): The location where the fetcher resides - **pref_tld** (optional): The Top-Level-Domain, which the fetcher prefers to crawl """ new_fetcher = fetchers.create_fetcher(db, fetcher) if database.fqdn_hash_activated(db): background_tasks.add_task(database.refresh_fqdn_hashes, db) return new_fetcher
def test_consistent_hash_deactivated(): rest.reset_long_term_part_strategy() assert database.fqdn_hash_activated(db) is False
def test_fqdn_hash_activated(): rest.activate_fqdn_hash() assert database.fqdn_hash_activated(db) is True