def endparallel(test=None):
"""termine un traitement parallele"""
mainmapper = getmainmapper()
nb_total = 0
nb_fichs = 0
if mainmapper.ended:
# print("pyetl double end", os.getpid())
time.sleep(1)
return None
try:
mainmapper.menage_final()
succes = True
except StopIteration:
nb_total, nb_fichs = mainmapper.sorties.final(mainmapper.idpyetl)
mainmapper.padd("_st_wr_fichs", nb_fichs)
mainmapper.padd("_st_wr_objs", nb_total)
succes = False
retour = set_parallelretour(mainmapper, succes)
# print("-----pyetl batchworker end", os.getpid(), succes, nb_total, nb_fichs)
if succes:
mainmapper.logger.info("-----pyetl batchworker end " +
str(os.getpid()) + " succes " + str(nb_total) +
" " + str(nb_fichs))
else:
mainmapper.logger.error("-----pyetl batchworker end " +
str(os.getpid()) + " echec " + str(nb_total) +
" " + str(nb_fichs))
mainmapper.ended = True
return (os.getpid(), retour)
def parallelbatch(id, parametres_batch, regle):
"""execute divers traitements en parallele"""
# print ("pyetl startbatch",os.getpid(), parametres_batch[:3])
numero, mapping, entree, sortie, args = parametres_batch
mainmapper = getmainmapper()
processor = mainmapper.getpyetl(mapping,
liste_params=args,
entree=entree,
rep_sortie=sortie)
if processor is None:
print("pyetl echec batchworker", os.getpid(), mapping, args)
return (numero, {})
processor.process()
retour = set_parallelretour(processor, True)
processor.logger.info(
"retour batchworker (%d:%d) %s %s -> %s",
os.getpid(),
processor.idpyetl,
str(mapping),
str(args),
str(processor.retour),
)
# print(
# "pyetl batchworker",
# os.getpid(),
# processor.idpyetl,
# mapping,
# args,
# "->",
# processor.retour,
# )
processor.cleanschemas()
return (numero, retour)
def setparallelid(parametres):
"""positionne un numero de worker et initialise les commandes """
pidset, commandes, args = parametres
mainmapper = getmainmapper()
if mainmapper.getvar("_wid"):
time.sleep(1)
return None
wid = str(pidset[os.getpid()])
mainmapper.setvar("_wid", wid)
log, log_level, log_print = (
mainmapper.getvar("logfile"),
mainmapper.getvar("log_level"),
mainmapper.getvar("log_print"),
)
if log:
base, ext = os.path.splitext(mainmapper.getvar("logfile"))
log = str(base) + "_" + wid + "." + str(ext)
loginfo = log, log_level, log_print
init = mainmapper.initpyetl(commandes, args, loginfo=loginfo)
if paralleldebug:
print("setparallelid apres init", mainmapper.getvar("_wid"), commandes,
args)
return (os.getpid(), mainmapper.getvar("_wid"), init)
def stoplistener():
mapper = getmainmapper()
# print("arret listener", mapper.loglistener)
if mapper.loglistener:
# mapper.logger.info("arret listener")
mapper.loglistener.enqueue_sentinel()
mapper.loglistener.stop()
def getqueue():
"""recupere le gestionnaire de files"""
mapper = getmainmapper()
if not mapper.msgqueue:
manager = getmanager(mapper)
if manager:
mapper.msgqueue = mapper.parallelmanager.Queue()
mapper.logqueue = mapper.parallelmanager.Queue()
return mapper.msgqueue, mapper.logqueue
def __init__(self) -> None:
self.liste = []
self.descriptif = dict()
self.mapper = getmainmapper()
self.mapper.url_for = url_for
print(
"initialisation mainmapper",
self.mapper.version,
self.mapper.getvar("mode", "interactif"),
)
self.scriptdir = os.path.join(self.mapper.getvar("workdir", "."), "scripts")
self.scripts = dict()
self.is_api = dict()
def parallelprocess(numero, file, regle):
"""traitement individuel d'un fichier"""
mainmapper = getmainmapper()
try:
# print ('---------------------------------------' + mainmapper.getvar("_wid") + '-worker:lecture', file, regle)
nom, parms = file
nb_lu = mainmapper.lecture(file, reglenum=regle, parms=parms)
except StopIteration as arret:
return numero, -1
except Exception as exc:
print("====" + mainmapper.getvar("_wid") +
"=erreur de traitement parallele non gérée")
print("====regle courante:", regle)
printexception()
raise
return numero, nb_lu
def setparallelid(parametres):
"""positionne un numero de worker et initialise les commandes """
pidset, commandes, args = parametres
mainmapper = getmainmapper()
if mainmapper.getvar("_wid"):
time.sleep(1)
return None
wid = str(pidset[os.getpid()])
mainmapper.setvar("_wid", wid)
init = mainmapper.initpyetl(commandes, args)
setqueuhandler(mainmapper.logqueue, wid=wid)
mainmapper.logger.info("pyetl initworker " + str(os.getpid()))
if paralleldebug:
print("setparallelid apres init", mainmapper.getvar("_wid"), commandes,
args)
return (os.getpid(), mainmapper.getvar("_wid"), init)
def initparallel(parametres):
"""initialisatin d'un process worker pour un traitement parallele"""
# commandes, args, params, macros, env, log = parametres
if parametres:
params, macros, env, loginfo, schemas, msgq, logqueue = parametres
# print("initialisation worker", os.getpid(), loginfo, msgq, logqueue)
else:
print("initialisation sans parametres")
return (os.getpid(), False)
mainmapper = getmainmapper()
if mainmapper is None:
print("erreur initialisation", os.getpid())
return None
if mainmapper.loginited:
# print("pyetl double init", os.getpid())
time.sleep(1)
return None
if paralleldebug:
print(
"pyetl initworker",
os.getpid(),
schemas.keys(),
mainmapper,
mainmapper.schemas,
)
mainmapper.worker = True
mainmapper.context.update(params)
mainmapper.initlog()
mainmapper.macrostore.macros.update(macros)
mainmapper.msgqueue = msgq
mainmapper.logqueue = logqueue
# print("initparallel: recuperation parametres", loginfo)
# print ('initparallel: valeur de import',params.get('import'))
integre_schemas(mainmapper.schemas, schemas)
mainmapper.parametres_lancement = parametres
time.sleep(1)
return (os.getpid(), True)
def setqueuhandler(queue, wid=""):
""" ajoute une gestion de file de messages pour le traitment en multiprocessing"""
import logging
import logging.handlers
mapper = getmainmapper()
if queue is None:
loglistener = logging.handlers.QueueListener(
mapper.logqueue,
*mapper.logger.handlers,
respect_handler_level=True)
loglistener.start()
mapper.logger.info("gestionnaire logs parallele : demarrage listener")
# print("-------------------demarrage listener")
mapper.loglistener = loglistener
else:
# on est sur le worker on ajoute un writer de file
mapper.logqueue = queue
queueformatter = logging.Formatter("(W" + str(wid) + "):%(message)s")
queuehandler = logging.handlers.QueueHandler(queue)
queuehandler.setFormatter(queueformatter)
mapper.logger.addHandler(queuehandler)
def objprocess(numero, objet, regle):
mainmapper = getmainmapper()
