def endparallel(test=None): """termine un traitement parallele""" mainmapper = getmainmapper() nb_total = 0 nb_fichs = 0 if mainmapper.ended: # print("pyetl double end", os.getpid()) time.sleep(1) return None try: mainmapper.menage_final() succes = True except StopIteration: nb_total, nb_fichs = mainmapper.sorties.final(mainmapper.idpyetl) mainmapper.padd("_st_wr_fichs", nb_fichs) mainmapper.padd("_st_wr_objs", nb_total) succes = False retour = set_parallelretour(mainmapper, succes) # print("-----pyetl batchworker end", os.getpid(), succes, nb_total, nb_fichs) if succes: mainmapper.logger.info("-----pyetl batchworker end " + str(os.getpid()) + " succes " + str(nb_total) + " " + str(nb_fichs)) else: mainmapper.logger.error("-----pyetl batchworker end " + str(os.getpid()) + " echec " + str(nb_total) + " " + str(nb_fichs)) mainmapper.ended = True return (os.getpid(), retour)
def parallelbatch(id, parametres_batch, regle): """execute divers traitements en parallele""" # print ("pyetl startbatch",os.getpid(), parametres_batch[:3]) numero, mapping, entree, sortie, args = parametres_batch mainmapper = getmainmapper() processor = mainmapper.getpyetl(mapping, liste_params=args, entree=entree, rep_sortie=sortie) if processor is None: print("pyetl echec batchworker", os.getpid(), mapping, args) return (numero, {}) processor.process() retour = set_parallelretour(processor, True) processor.logger.info( "retour batchworker (%d:%d) %s %s -> %s", os.getpid(), processor.idpyetl, str(mapping), str(args), str(processor.retour), ) # print( # "pyetl batchworker", # os.getpid(), # processor.idpyetl, # mapping, # args, # "->", # processor.retour, # ) processor.cleanschemas() return (numero, retour)
def setparallelid(parametres): """positionne un numero de worker et initialise les commandes """ pidset, commandes, args = parametres mainmapper = getmainmapper() if mainmapper.getvar("_wid"): time.sleep(1) return None wid = str(pidset[os.getpid()]) mainmapper.setvar("_wid", wid) log, log_level, log_print = ( mainmapper.getvar("logfile"), mainmapper.getvar("log_level"), mainmapper.getvar("log_print"), ) if log: base, ext = os.path.splitext(mainmapper.getvar("logfile")) log = str(base) + "_" + wid + "." + str(ext) loginfo = log, log_level, log_print init = mainmapper.initpyetl(commandes, args, loginfo=loginfo) if paralleldebug: print("setparallelid apres init", mainmapper.getvar("_wid"), commandes, args) return (os.getpid(), mainmapper.getvar("_wid"), init)
def stoplistener(): mapper = getmainmapper() # print("arret listener", mapper.loglistener) if mapper.loglistener: # mapper.logger.info("arret listener") mapper.loglistener.enqueue_sentinel() mapper.loglistener.stop()
def getqueue(): """recupere le gestionnaire de files""" mapper = getmainmapper() if not mapper.msgqueue: manager = getmanager(mapper) if manager: mapper.msgqueue = mapper.parallelmanager.Queue() mapper.logqueue = mapper.parallelmanager.Queue() return mapper.msgqueue, mapper.logqueue
def __init__(self) -> None: self.liste = [] self.descriptif = dict() self.mapper = getmainmapper() self.mapper.url_for = url_for print( "initialisation mainmapper", self.mapper.version, self.mapper.getvar("mode", "interactif"), ) self.scriptdir = os.path.join(self.mapper.getvar("workdir", "."), "scripts") self.scripts = dict() self.is_api = dict()
def parallelprocess(numero, file, regle): """traitement individuel d'un fichier""" mainmapper = getmainmapper() try: # print ('---------------------------------------' + mainmapper.getvar("_wid") + '-worker:lecture', file, regle) nom, parms = file nb_lu = mainmapper.lecture(file, reglenum=regle, parms=parms) except StopIteration as arret: return numero, -1 except Exception as exc: print("====" + mainmapper.getvar("_wid") + "=erreur de traitement parallele non gérée") print("====regle courante:", regle) printexception() raise return numero, nb_lu
def setparallelid(parametres): """positionne un numero de worker et initialise les commandes """ pidset, commandes, args = parametres mainmapper = getmainmapper() if mainmapper.getvar("_wid"): time.sleep(1) return None wid = str(pidset[os.getpid()]) mainmapper.setvar("_wid", wid) init = mainmapper.initpyetl(commandes, args) setqueuhandler(mainmapper.logqueue, wid=wid) mainmapper.logger.info("pyetl initworker " + str(os.getpid())) if paralleldebug: print("setparallelid apres init", mainmapper.getvar("_wid"), commandes, args) return (os.getpid(), mainmapper.getvar("_wid"), init)
def initparallel(parametres): """initialisatin d'un process worker pour un traitement parallele""" # commandes, args, params, macros, env, log = parametres if parametres: params, macros, env, loginfo, schemas, msgq, logqueue = parametres # print("initialisation worker", os.getpid(), loginfo, msgq, logqueue) else: print("initialisation sans parametres") return (os.getpid(), False) mainmapper = getmainmapper() if mainmapper is None: print("erreur initialisation", os.getpid()) return None if mainmapper.loginited: # print("pyetl double init", os.getpid()) time.sleep(1) return None if paralleldebug: print( "pyetl initworker", os.getpid(), schemas.keys(), mainmapper, mainmapper.schemas, ) mainmapper.worker = True mainmapper.context.update(params) mainmapper.initlog() mainmapper.macrostore.macros.update(macros) mainmapper.msgqueue = msgq mainmapper.logqueue = logqueue # print("initparallel: recuperation parametres", loginfo) # print ('initparallel: valeur de import',params.get('import')) integre_schemas(mainmapper.schemas, schemas) mainmapper.parametres_lancement = parametres time.sleep(1) return (os.getpid(), True)
def setqueuhandler(queue, wid=""): """ ajoute une gestion de file de messages pour le traitment en multiprocessing""" import logging import logging.handlers mapper = getmainmapper() if queue is None: loglistener = logging.handlers.QueueListener( mapper.logqueue, *mapper.logger.handlers, respect_handler_level=True) loglistener.start() mapper.logger.info("gestionnaire logs parallele : demarrage listener") # print("-------------------demarrage listener") mapper.loglistener = loglistener else: # on est sur le worker on ajoute un writer de file mapper.logqueue = queue queueformatter = logging.Formatter("(W" + str(wid) + "):%(message)s") queuehandler = logging.handlers.QueueHandler(queue) queuehandler.setFormatter(queueformatter) mapper.logger.addHandler(queuehandler)
def objprocess(numero, objet, regle): mainmapper = getmainmapper()