def initialize(config, prefix, _logger): """Initialize the worker system""" global pool global storages global urls global jinjaEnv global logger # save the logger logger = _logger # create the worker pool threads = config.getMulti(prefix, "daemon.threads", 1) pool = workerpool.WorkerPool(size = threads) logger.info("Worker pool size %d (# threads)", threads) # get the storages storages = ecommerce.storage.getStorages(config, prefix + ".repositories") logger.info("Repositories: %s", ", ".join(storages.keys())) # prepare the base urls urls = config.getMulti(prefix, "urls", { }) # jinja environment folder = config.getMulti(prefix, "templates.folder", "./templates") jinjaEnv = jinja2.Environment(loader = jinja2.FileSystemLoader(folder)) logger.info("Templates folder: %s", folder)
def get_logger(app_prefix, log_prefix, config = None): global _loggers global _logconf # be sure to have a config if config is None: config = ecommerce.config.getConfig() # configure logging if _loggers is None: # set basic parameters logging.basicConfig(level = logging.DEBUG, datefmt = '%Y-%m-%d %H:%M:%S') # get logging config _logconf = config.getMulti(app_prefix, "logging") if _logconf is not None: # config the logging logging.config.dictConfig(_logconf) _loggers = { } # create the logger if not already created logger = _loggers[log_prefix] if log_prefix in _loggers else None if logger is None: # get a logger if _logconf is not None: # get a logger logger = logging.getLogger(log_prefix) else: logger = logging.getLogger('') # save the logger _loggers[log_prefix] = logger return logger
def daemon(config, prefix, log_prefix, queue): """The main daemon function It gets some options from the configuration and starts trying to pull jobs from the queue. """ set_logger(prefix, log_prefix) # get some config options maxrun = config.getMulti(prefix, "daemon.maxrun", 3600) endBy = time.time() + maxrun logger.info("Daemon started - maxrun %d", maxrun) # initialize the work work.initialize(config, prefix, logger) # set dataset application and preprocessing ecommerce.db.dataset.configApplication(application) ecommerce.db.dataset.setPreProcess(preprocess.preProcess) # start looping now = time.time() logger.info("Daemon will end by %s", endBy) while not terminating() and now < endBy: # try to fetch a job from the queue (non blocking) item = queue.next() while item is None and now < endBy: time.sleep(1) item = queue.next() now = time.time() # if we have a job, process it if item is not None: # decode the job job = jobs.decode(item.content) logger.info("Got job %s, job type %s", item.id, job.get("type", "unknown")) # get the job type and dispatch it jobType = job.get("type") if jobType not in jobTypes: handled = False else: tStart = time.time() handled = jobTypes[jobType](item.id, job) tEnd = time.time() # mark as done or error if handled: logger.info("Job %s processed ok", item.id) queue.done(item) else: logger.info("Job %s processed with ERROR", item.id) queue.error(item) logger.info("Took %3f seconds to complete job", tEnd - tStart) else: logger.debug("Job Queue empty") # sleep for 1 second (if queue is empty) if queue.isEmpty(): logger.debug("Waiting 1 second...") time.sleep(1) # get current time now = time.time() logger.info("Daemon terminating - reason: %s", "command" if terminating() else "maxrun reached") # deinitialize the work work.deinitialize() return 0