def __init__(self, context_name, log_level, log_stream, log_folder,
database_module, db_configuration, **kwargs):
self.logger = get_logger(context_name, log_level, log_stream,
log_folder)
print_start(self.logger)
self.db = database_module(self.logger, **db_configuration)
self.logger.info("{} setup complete !!".format(context_name))
if self.db != None:
self.push_todb = self.db.put
else:
raise ValueError("No database parameter given")
configuration = json.load(file)
Database_module, db_configuration = parse_dbconfig(configuration)
## reading logging configuration
logging_configuration = configuration["logging"]
log_folder = logging_configuration["output"]
if not log_folder in os.listdir('.'):
os.mkdir(log_folder)
logger = get_logger(__name__, log_level, log_stream, log_folder)
## logger for main thread
## logger test in main thread
print_start(logger)
logger.info("Application started , Extracting all the plugins")
## handles creating mutiple process
## from single process using MultiProcessing
import_list = configuration["plugins"]
with MultiProcessingContext(log_level, log_stream, log_folder) as execute:
for attr in import_list:
path = attr["filename"]
class_name = attr["class"]
plugin_module = importlib.import_module(path, ".")
scrapper = plugin_module.__getattribute__(class_name)