min_cache_imgs_before_refill = config.getint("Cache", "Images_min_limit") user_agent = config.get("General", "Useragent") logverbosity = config.get("Logging", "Verbosity") logger = logging.getLogger(config.get("Logging", "Log_name")) hdlr = logging.FileHandler(config.get("Logging", "File")) hdlr.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) call_flush_timeout = 10 # value in seconds call_flush_last = time.time() - call_flush_timeout call_reset_timeout = 10 # value in seconds call_reset_last = time.time() - call_reset_timeout Crawler.headers({'User-Agent': user_agent}) Crawler.set_logger(logger) ### config the crawlers from crawler.reddit import Reddit from crawler.soupio import SoupIO from crawler.pr0gramm import Pr0gramm from crawler.ninegag import NineGag from crawler.instagram import Instagram from crawler.fourchan import Fourchan def get_crawlers(configuration, section): """ parse the config section for crawlers * does recognize (by name) known and implemented crawlers only