def demarrer_contexte(self, idmg: str):
configuration = self._millegrilles[idmg]
configuration_contexte = ConfigurationHebergement(
self.contexte.configuration, configuration)
contexte_hebergement = ContexteRessourcesDocumentsMilleGrilles(
configuration=configuration_contexte)
configuration['contexte'] = contexte_hebergement
contexte_hebergement.initialiser()
# Demarrer le gestionnaire de domaines
gestionnaire_domaines = GestionnaireDomainesMilleGrilles()
configuration['gestionnaire'] = gestionnaire_domaines
gestionnaire_domaines.configurer_parser()
gestionnaire_domaines.parse()
# Bypass initialiser - utilise les parametres system
gestionnaire_domaines.initialiser_2(contexte_hebergement)
thread = Thread(target=gestionnaire_domaines.executer, name=idmg)
configuration['thread'] = thread
thread.start()
def demarrer_contexte(self, idmg: str):
configuration = self._millegrilles[idmg]
configuration_contexte = ConfigurationHebergement(
self.contexte.configuration, configuration)
contexte_hebergement = ContexteRessourcesDocumentsMilleGrilles(
configuration=configuration_contexte)
configuration['contexte'] = contexte_hebergement
contexte_hebergement.initialiser()
# Demarrer le gestionnaire de transaction
consignateur_transactions = ConsignateurTransaction()
configuration['consignateur'] = consignateur_transactions
consignateur_transactions.configurer_parser()
consignateur_transactions.parse()
# Bypass initialiser - utilise les parametres system
consignateur_transactions.initialiser_2(contexte_hebergement)
thread = Thread(target=consignateur_transactions.executer, name=idmg)
configuration['thread'] = thread
thread.start()
def main():
contexte = ContexteRessourcesDocumentsMilleGrilles()
contexte.initialiser(init_document=True)
# requete_agg_backup(contexte)
# requete_quotidien(contexte)
requete_transactions_domaines(contexte)
def main():
contexte = ContexteRessourcesDocumentsMilleGrilles()
contexte.initialiser(init_document=True)
# reset_dates(contexte, ['MaitreDesCles', 'GrosFichiers'], datetime.timedelta(hours=-24))
# reset_dates(contexte, ['MaitreDesCles'], datetime.timedelta(hours=-2))
# reset_dates(contexte, ['MaitreDesCles'], datetime.timedelta(days=-7))
reset_dates(contexte, ['MaitreDesCles', 'GrosFichiers'],
datetime.timedelta(days=-732))
def __init__(self):
self._logger = logging.getLogger('%s' % self.__class__.__name__)
self._logger.setLevel(logging.INFO)
self._contexte = ContexteRessourcesDocumentsMilleGrilles()
self.parser = None # Parser de ligne de commande
self.args = None # Arguments de la ligne de commande
self.__fermeture_event = Event()
self.__certificat_event_handler: GestionnaireEvenementsCertificat = None
self.__channel = None
def initialiser(self, init_document=True):
additionnals = self._contexte_additionnals()
self._contexte = ContexteRessourcesDocumentsMilleGrilles(
configuration=self._configuration, additionals=additionnals)
try:
self._contexte.initialiser(
init_document=init_document,
init_message=True,
connecter=True,
)
except ServerSelectionTimeoutError:
self.__logger.warning("Erreur connexion mongo, ServerSelectionTimeoutError")
if self.__logger.isEnabledFor(logging.DEBUG):
self.__logger.exception("Detail error connexion Mongo")
self._certificat_event_handler = GestionnaireEvenementsCertificat(self._contexte)
self.__commandes_handler = TraitementMessagesMiddleware(self._service_monitor.noeud_id, self._service_monitor.gestionnaire_commandes, self._contexte)
self._contexte.message_dao.register_channel_listener(self)
self._contexte.message_dao.register_channel_listener(self.__commandes_handler)
from millegrilles import Constantes
from millegrilles.domaines.MaitreDesCles import ConstantesMaitreDesCles
from millegrilles.domaines.Parametres import ConstantesParametres
from millegrilles.util.X509Certificate import EnveloppeCleCert
from threading import Event, Thread
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat import primitives
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
from base64 import b64encode
from uuid import uuid4
contexte = ContexteRessourcesDocumentsMilleGrilles()
contexte.initialiser()
class MessagesSample(BaseCallback):
def __init__(self):
super().__init__(contexte)
self.contexte.message_dao.register_channel_listener(self)
self.generateur = GenerateurTransaction(self.contexte)
self.channel = None
self.event_recu = Event()
# self.thread_ioloop = Thread(target=self.run_ioloop)
self.certificat_maitredescles = None
self.cert_maitredescles_recu = Event()
class ConnexionMiddleware:
"""
Connexion au middleware de la MilleGrille en service.
"""
def __init__(self, configuration: TransactionConfiguration, client_docker: docker.DockerClient,
service_monitor, certificats: dict, **kwargs):
self._configuration = configuration
self._docker = client_docker
self._service_monitor = service_monitor
self._certificats = certificats
self._path_secrets: str = kwargs.get('secrets') or '/run/secrets'
self._monitor_keycert_file: str
self._connexion_relai: ConnexionPrincipal = cast(ConnexionPrincipal, None)
self._contexte: ContexteRessourcesDocumentsMilleGrilles = cast(ContexteRessourcesDocumentsMilleGrilles, None)
self._thread: Thread = cast(Thread, None)
self._channel = None
self._fermeture_event = Event()
self._certificat_event_handler: GestionnaireEvenementsCertificat
self.__commandes_handler: TraitementMessagesMiddleware = cast(TraitementMessagesMiddleware, None)
self.__transfert_local_handler: TransfertMessages = cast(TransfertMessages, None)
self.__monitor_cert_file: str
self.__logger = logging.getLogger(__name__ + '.' + self.__class__.__name__)
self._comptes_middleware_ok = False
self._comptes_mq_ok = False
self._prochaine_verification_comptes_noeuds = datetime.datetime.utcnow().timestamp()
self._gestionnaire_mdns = kwargs.get('gestionnaire_mdns')
def start(self):
self.__logger.info("Demarrage ConnexionMiddleware")
# Connecter
# Demarrer thread
self._thread = Thread(target=self.run, name="mw", daemon=True)
self._thread.start()
def stop(self):
self._fermeture_event.set()
try:
self._contexte.message_dao.deconnecter()
self._contexte.document_dao.deconnecter()
except Exception:
pass
def initialiser(self, init_document=True):
additionnals = self._contexte_additionnals()
self._contexte = ContexteRessourcesDocumentsMilleGrilles(
configuration=self._configuration, additionals=additionnals)
try:
self._contexte.initialiser(
init_document=init_document,
init_message=True,
connecter=True,
)
except ServerSelectionTimeoutError:
self.__logger.warning("Erreur connexion mongo, ServerSelectionTimeoutError")
if self.__logger.isEnabledFor(logging.DEBUG):
self.__logger.exception("Detail error connexion Mongo")
self._certificat_event_handler = GestionnaireEvenementsCertificat(self._contexte)
self.__commandes_handler = TraitementMessagesMiddleware(self._service_monitor.noeud_id, self._service_monitor.gestionnaire_commandes, self._contexte)
self._contexte.message_dao.register_channel_listener(self)
self._contexte.message_dao.register_channel_listener(self.__commandes_handler)
def _contexte_additionnals(self) -> list:
ca_certs_file = self._certificats['pki.millegrille.cert']
monitor_cert_file = self._certificats[GestionnaireCertificats.MONITOR_CERT_PATH]
monitor_key_file = path.join(self._path_secrets, self._certificats[GestionnaireCertificats.MONITOR_KEY_FILE])
mq_info = self.get_mq_info()
self.__logger.info("Information de connexion MQ : %s" % str(mq_info))
additionnals = [{
'MG_MQ_HOST': mq_info['host'],
'MG_MQ_PORT': mq_info['port'],
'MG_MQ_CA_CERTS': ca_certs_file,
'MG_MQ_CERTFILE': monitor_cert_file,
'MG_MQ_KEYFILE': monitor_key_file,
'MG_MQ_SSL': 'on',
'MG_MQ_AUTH_CERT': 'on',
}]
return additionnals
def get_mq_info(self):
self.__logger.debug("Demande services mdns pour idmg %s" % self._service_monitor.idmg)
try:
services = self._service_monitor.gestionnaire_commandes.requete_mdns_acteur(self._service_monitor.idmg)
except Exception:
self.__logger.warning("Erreur acces MDNS pour host MQ, tentative utilisation host/port env")
host = self.configuration.mq_host
port = self.configuration.mq_port
else:
self.__logger.debug("Services MDNS detectes : %d" % len(services))
for service in services:
self.__logger.debug("Service %s port %d, addresses : %s" % (service['type'], service['port'], str(service['addresses'])))
services_mq = [s for s in services if s.get('type') == '_mgamqps._tcp.local.']
self.__logger.debug("Services MDNS MQ detectes : %d" % len(services))
for service_mq in services_mq:
self.__logger.debug("Service %s port %d, addresses : %s" % (service_mq['type'], service_mq['port'], str(service_mq['addresses'])))
try:
service_retenu = services_mq[0]
host = service_retenu['addresses'][0]
port = service_retenu['port']
except IndexError:
# Utiliser configuration fourni
self.__logger.info("Information MDNS non disponible, fallback sur configuration environnement")
host = self.configuration.mq_host
port = self.configuration.mq_port
info_mq = {'host': host, 'port': port}
self.__logger.info("Service MDNS MQ detecte : %s" % str(info_mq))
return info_mq
def set_relai(self, connexion_relai: ConnexionPrincipal):
if not self.__transfert_local_handler:
self.__transfert_local_handler = TransfertMessages(
self._contexte, connexion_relai.relayer_message, self._service_monitor.nodename)
self._contexte.message_dao.register_channel_listener(self.__transfert_local_handler)
def relayer_message(self, message_dict, routing_key, exchange, reply_to=None, correlation_id=None):
"""
Relai un message recu sur le noeud principal vers le noeud dependant.
:param message_dict:
:param routing_key:
:param exchange:
:param reply_to:
:param correlation_id:
:return:
"""
headers = {'noeud_source': self._service_monitor.nodename}
if reply_to == TransfertMessages.LOCAL_Q_PLACEHOLDER:
# Ajouter la Q de transfert locale pour recevoir la reponse a relayer
reply_to = self.__transfert_local_handler.queue_name
self.generateur_transactions.emettre_message(message_dict, routing_key, [exchange], reply_to, correlation_id, headers)
def on_channel_open(self, channel):
channel.basic_qos(prefetch_count=1)
channel.add_on_close_callback(self.on_channel_close)
self._channel = channel
self._certificat_event_handler.initialiser()
def on_channel_close(self, channel=None, code=None, reason=None):
self._channel = None
self.__logger.warning("MQ Channel ferme")
if not self._fermeture_event.is_set():
try:
self._contexte.message_dao.enter_error_state()
except Exception:
# Erreur d'activation du error state, la connexion ne peut pas etre reactivee
self.__logger.exception("Erreur fermeture channel")
self._fermeture_event.set() # S'assurer que la fermeture est en cours
def __on_return(self, channel, method, properties, body):
pass
def run(self):
self.__logger.info("Thread middleware demarree")
while not self._fermeture_event.is_set():
try:
self.__entretien_comptes()
self._entretien()
except Exception:
self.__logger.exception("Exception generique")
finally:
self._fermeture_event.wait(30)
self.__logger.info("Fin thread middleware")
def __entretien_comptes(self):
if not self._comptes_middleware_ok or not self._comptes_mq_ok:
comptes_mq_ok = True # Va etre mis a false si un compte n'esp pas ajoute correctement
try:
idmg = self._configuration.idmg
igmd_tronque = idmg[0:12]
roles_comptes = [info['role'] for info in MonitorConstantes.DICT_MODULES_PROTEGES.values() if info.get('role')]
roles_comptes = ['%s.pki.%s.cert' % (igmd_tronque, role) for role in roles_comptes]
roles_mongo = [
ConstantesGenerateurCertificat.ROLE_TRANSACTIONS,
ConstantesGenerateurCertificat.ROLE_DOMAINES,
ConstantesGenerateurCertificat.ROLE_MAITREDESCLES,
]
for role in roles_comptes:
filtre = {'name': role}
configs = self._docker.configs.list(filters=filtre)
if len(configs) > 0:
dict_configs = dict()
for config in configs:
dict_configs[config.name] = config
# Choisir plus recent certificat
liste_configs_str = list(dict_configs.keys())
liste_configs_str.sort()
nom_config = liste_configs_str[-1]
config_cert = dict_configs[nom_config]
# Extraire certificat
cert_pem = b64decode(config_cert.attrs['Spec']['Data'])
clecert = EnveloppeCleCert()
clecert.cert_from_pem_bytes(cert_pem)
# Creer compte
roles_cert = clecert.get_roles
if any([role in roles_mongo for role in roles_cert]):
try:
self.__mongo.creer_compte(clecert)
except DuplicateKeyError:
self.__logger.debug("Compte mongo (deja) cree : %s", nom_config)
try:
gestionnaire_mq: GestionnaireComptesMQ = self._service_monitor.gestionnaire_mq
gestionnaire_mq.ajouter_compte(clecert)
except ValueError:
comptes_mq_ok = False
self._comptes_middleware_ok = True
except Exception:
self.__logger.exception("Erreur enregistrement comptes")
self._comptes_mq_ok = comptes_mq_ok
def _entretien(self):
ts_courant = datetime.datetime.utcnow().timestamp()
exchange = self.exchange
# Emettre message de presence du monitor
self.emettre_presence()
# Transmettre requete pour avoir l'etat de l'hebergement
self.generateur_transactions.transmettre_requete(
dict(), Constantes.ConstantesHebergement.REQUETE_MILLEGRILLES_ACTIVES,
reply_to=self.__commandes_handler.queue_name,
correlation_id=ConstantesServiceMonitor.CORRELATION_HEBERGEMENT_LISTE,
securite=exchange
)
def emettre_presence(self):
info_monitor = dict(self._service_monitor.get_info_monitor(inclure_services=True))
info_monitor['noeud_id'] = self._service_monitor.noeud_id
info_monitor['securite'] = self._service_monitor.securite
domaine_action = Constantes.ConstantesTopologie.EVENEMENT_PRESENCE_MONITOR
self.generateur_transactions.emettre_message(info_monitor, domaine_action, ajouter_certificats=True)
def ajouter_commande(self, commande):
gestionnaire_commandes: GestionnaireCommandes = self._service_monitor.gestionnaire_commandes
gestionnaire_commandes.ajouter_commande(commande)
def rediriger_messages_domaine(self, nom_domaine: str, exchanges_routing: dict):
self.__transfert_local_handler.ajouter_domaine(nom_domaine, exchanges_routing)
def enregistrer_listener(self, methode_initialisation):
"""
Initialise un objet/methode avec le contexte et enregistre le listener retourne
:param methode_initialisation: Methode qui va recevoir contexte, doit retourner l'instance du listener
:return:
"""
listener = methode_initialisation(self._contexte)
self._contexte.message_dao.register_channel_listener(listener)
@property
def document_dao(self) -> MongoDAO:
return self._contexte.document_dao
@property
def configuration(self) -> TransactionConfiguration:
return self._configuration
@property
def generateur_transactions(self):
return self._contexte.generateur_transactions
@property
def exchange(self):
return self._service_monitor.securite
@property
def verificateur_transactions(self):
return self._contexte.verificateur_transaction
class ModeleConfiguration:
def __init__(self):
self._logger = logging.getLogger('%s' % self.__class__.__name__)
self._logger.setLevel(logging.INFO)
self._contexte = ContexteRessourcesDocumentsMilleGrilles()
self.parser = None # Parser de ligne de commande
self.args = None # Arguments de la ligne de commande
self.__fermeture_event = Event()
self.__certificat_event_handler: GestionnaireEvenementsCertificat = None
self.__channel = None
def initialiser(self,
init_document=True,
init_message=True,
connecter=True):
# Gerer les signaux OS, permet de deconnecter les ressources au besoin
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self._contexte.initialiser(init_document=init_document,
init_message=init_message,
connecter=connecter)
if init_message:
self._contexte.message_dao.register_channel_listener(self)
self.initialiser_2(contexte=self._contexte)
def initialiser_2(self, contexte):
if contexte is not None:
self._contexte = contexte
self.__certificat_event_handler = GestionnaireEvenementsCertificat(
self._contexte)
def on_channel_open(self, channel):
channel.basic_qos(prefetch_count=1)
channel.add_on_close_callback(self.on_channel_close)
self.__channel = channel
self.__certificat_event_handler.initialiser()
def on_channel_close(self, channel=None, code=None, reason=None):
self.__channel = None
self._logger.warning("MQ Channel ferme")
if not self.__fermeture_event.is_set():
try:
self.contexte.message_dao.enter_error_state()
except Exception:
# Erreur d'activation du error state, la connexion ne peut pas etre reactivee
self._logger.exception("Erreur fermeture channel")
self.__fermeture_event.set(
) # S'assurer que la fermeture est en cours
def __on_return(self, channel, method, properties, body):
pass
def configurer_parser(self):
self.parser = argparse.ArgumentParser(
description="Fonctionnalite MilleGrilles")
self.parser.add_argument('--verbose',
action="store_true",
required=False,
help="Active le debugging verbose")
self.parser.add_argument(
'--debug',
action="store_true",
required=False,
help="Active le debugging (logger, tres verbose)")
self.parser.add_argument(
'--info',
action="store_true",
required=False,
help="Afficher davantage de messages (verbose)")
def print_help(self):
self.parser.print_help()
def exit_gracefully(self, signum=None, frame=None):
self.__fermeture_event.set()
self.deconnecter()
def parse(self):
self.args = self.parser.parse_args()
def executer(self):
raise NotImplemented("Cette methode doit etre redefinie")
def connecter(self):
if self._contexte.message_dao is not None:
self._contexte.message_dao.connecter()
if self._contexte.document_dao is not None:
self._contexte.document_dao.connecter()
def deconnecter(self):
self._contexte.fermer()
# try:
# if self._contexte.message_dao is not None:
# self._contexte.message_dao.deconnecter()
# except Exception:
# self._logger.warning("Erreur fermeture message_dao")
#
# try:
# if self._contexte.document_dao is not None:
# self._contexte.document_dao.deconnecter()
# except Exception:
# self._logger.warning("Erreur fermeture document_dao")
def set_logging_level(self):
""" Utilise args pour ajuster le logging level (debug, info) """
logging.getLogger('events').setLevel(logging.WARNING)
logging.getLogger('pika').setLevel(logging.WARNING)
if self.args.debug:
self._logger.setLevel(logging.DEBUG)
logging.getLogger('millegrilles').setLevel(logging.DEBUG)
elif self.args.info:
self._logger.setLevel(logging.INFO)
logging.getLogger('millegrilles').setLevel(logging.INFO)
if self.args.verbose:
# Logging de messages frequents, tres verbose
logging.getLogger('trace.millegrilles').setLevel(logging.DEBUG)
def main(self):
code_retour = 0
try:
# Preparer logging
logging.basicConfig(format=Constantes.LOGGING_FORMAT,
level=logging.WARNING)
# logging.getLogger('millegrilles.dao.MessageDAO').setLevel(logging.INFO)
self._logger.info("\n-----------\n\n-----------")
self._logger.info("Demarrage de %s en cours\n-----------" %
self.__class__.__name__)
# Faire le parsing des arguments pour verifier s'il en manque
self.configurer_parser()
self.parse()
self.set_logging_level()
self._logger.info("Initialisation")
self.initialiser() # Initialiser les ressources
self._logger.info("Debut execution")
self.executer() # Executer le download et envoyer message
self.__fermeture_event.set()
self._logger.info("Fin execution " + self.__class__.__name__)
except Exception as e:
# print("MAIN: Erreur fatale, voir log. Erreur %s" % str(e))
self._logger.exception("MAIN: Erreur")
code_retour = 1
finally:
self.exit_gracefully()
self._logger.info("Main terminee, finalisation et sortie.")
try:
self.__finalisation()
finally:
sys.exit(code_retour)
def __finalisation(self):
time.sleep(0.2)
if threading.active_count() > 1:
ok_threads = ['MainThread', 'pymongo_kill_cursors_thread']
for thread in threading.enumerate():
if thread.name not in ok_threads:
self._logger.error(
"Thread ouverte apres demande de fermeture: %s" %
thread.name)
time.sleep(5)
for thread in threading.enumerate():
if thread.name not in ok_threads:
if not thread.isDaemon():
self._logger.warning(
"Non-daemon thread encore ouverte apres demande de fermeture: %s"
% thread.name)
@property
def contexte(self) -> ContexteRessourcesDocumentsMilleGrilles:
return self._contexte
@property
def channel(self):
return self.__channel
@staticmethod
def preparer_mongo_keycert():
json_file = os.getenv('MG_CONFIG_JSON')
if json_file:
with open(json_file, 'r') as fichier:
params = json.load(fichier)
else:
params = os.environ
certfile = params['MG_MQ_CERTFILE']
keyfile = params['MG_MQ_KEYFILE']
mongo_keycert_handle, mongo_keycert = tempfile.mkstemp(dir='/tmp',
text=True)
try:
with open(keyfile, 'rb') as fichiers:
os.write(mongo_keycert_handle, fichiers.read())
with open(certfile, 'rb') as fichiers:
os.write(mongo_keycert_handle, fichiers.read())
os.environ["MG_MONGO_SSL_CERTFILE"] = mongo_keycert
finally:
os.close(mongo_keycert_handle)
return mongo_keycert