SPIDER_MODULES = ["converter.spiders"] NEWSPIDER_MODULE = "converter.spiders" LOG_FILE = env.get("LOG_FILE", allow_null=True) LOG_LEVEL = env.get("LOG_LEVEL", default="INFO") LOG_FORMATTER = "converter.custom_log_formatter.CustomLogFormatter" # Default behaviour for regular crawlers of non-license-controlled content # When set True, every item will have GROUP_EVERYONE attached in edu-sharing # When set False, no permissions are set at all, which can be helpful if you want to control them later (e.g. via inherition) DEFAULT_PUBLIC_STATE = False # Splash (Web Thumbnailer) # Will be rolled out via docker-compose by default SPLASH_URL = (None if env.get_bool("DISABLE_SPLASH", default=False) else "http://localhost:8050") SPLASH_WAIT = 1 # seconds to let the page load SPLASH_HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" } # use chrome to not create warnings on pages # edu-sharing config EDU_SHARING_BASE_URL = env.get("EDU_SHARING_BASE_URL") EDU_SHARING_USERNAME = env.get("EDU_SHARING_USERNAME") EDU_SHARING_PASSWORD = env.get("EDU_SHARING_PASSWORD") # Thumbnail config THUMBNAIL_SMALL_SIZE = 250 * 250 THUMBNAIL_SMALL_QUALITY = 40