def demarrer_contexte(self, idmg: str): configuration = self._millegrilles[idmg] configuration_contexte = ConfigurationHebergement( self.contexte.configuration, configuration) contexte_hebergement = ContexteRessourcesDocumentsMilleGrilles( configuration=configuration_contexte) configuration['contexte'] = contexte_hebergement contexte_hebergement.initialiser() # Demarrer le gestionnaire de domaines gestionnaire_domaines = GestionnaireDomainesMilleGrilles() configuration['gestionnaire'] = gestionnaire_domaines gestionnaire_domaines.configurer_parser() gestionnaire_domaines.parse() # Bypass initialiser - utilise les parametres system gestionnaire_domaines.initialiser_2(contexte_hebergement) thread = Thread(target=gestionnaire_domaines.executer, name=idmg) configuration['thread'] = thread thread.start()
def demarrer_contexte(self, idmg: str): configuration = self._millegrilles[idmg] configuration_contexte = ConfigurationHebergement( self.contexte.configuration, configuration) contexte_hebergement = ContexteRessourcesDocumentsMilleGrilles( configuration=configuration_contexte) configuration['contexte'] = contexte_hebergement contexte_hebergement.initialiser() # Demarrer le gestionnaire de transaction consignateur_transactions = ConsignateurTransaction() configuration['consignateur'] = consignateur_transactions consignateur_transactions.configurer_parser() consignateur_transactions.parse() # Bypass initialiser - utilise les parametres system consignateur_transactions.initialiser_2(contexte_hebergement) thread = Thread(target=consignateur_transactions.executer, name=idmg) configuration['thread'] = thread thread.start()
def main(): contexte = ContexteRessourcesDocumentsMilleGrilles() contexte.initialiser(init_document=True) # requete_agg_backup(contexte) # requete_quotidien(contexte) requete_transactions_domaines(contexte)
def main(): contexte = ContexteRessourcesDocumentsMilleGrilles() contexte.initialiser(init_document=True) # reset_dates(contexte, ['MaitreDesCles', 'GrosFichiers'], datetime.timedelta(hours=-24)) # reset_dates(contexte, ['MaitreDesCles'], datetime.timedelta(hours=-2)) # reset_dates(contexte, ['MaitreDesCles'], datetime.timedelta(days=-7)) reset_dates(contexte, ['MaitreDesCles', 'GrosFichiers'], datetime.timedelta(days=-732))
def __init__(self): self._logger = logging.getLogger('%s' % self.__class__.__name__) self._logger.setLevel(logging.INFO) self._contexte = ContexteRessourcesDocumentsMilleGrilles() self.parser = None # Parser de ligne de commande self.args = None # Arguments de la ligne de commande self.__fermeture_event = Event() self.__certificat_event_handler: GestionnaireEvenementsCertificat = None self.__channel = None
def initialiser(self, init_document=True): additionnals = self._contexte_additionnals() self._contexte = ContexteRessourcesDocumentsMilleGrilles( configuration=self._configuration, additionals=additionnals) try: self._contexte.initialiser( init_document=init_document, init_message=True, connecter=True, ) except ServerSelectionTimeoutError: self.__logger.warning("Erreur connexion mongo, ServerSelectionTimeoutError") if self.__logger.isEnabledFor(logging.DEBUG): self.__logger.exception("Detail error connexion Mongo") self._certificat_event_handler = GestionnaireEvenementsCertificat(self._contexte) self.__commandes_handler = TraitementMessagesMiddleware(self._service_monitor.noeud_id, self._service_monitor.gestionnaire_commandes, self._contexte) self._contexte.message_dao.register_channel_listener(self) self._contexte.message_dao.register_channel_listener(self.__commandes_handler)
from millegrilles import Constantes from millegrilles.domaines.MaitreDesCles import ConstantesMaitreDesCles from millegrilles.domaines.Parametres import ConstantesParametres from millegrilles.util.X509Certificate import EnveloppeCleCert from threading import Event, Thread from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat import primitives from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from base64 import b64encode from uuid import uuid4 contexte = ContexteRessourcesDocumentsMilleGrilles() contexte.initialiser() class MessagesSample(BaseCallback): def __init__(self): super().__init__(contexte) self.contexte.message_dao.register_channel_listener(self) self.generateur = GenerateurTransaction(self.contexte) self.channel = None self.event_recu = Event() # self.thread_ioloop = Thread(target=self.run_ioloop) self.certificat_maitredescles = None self.cert_maitredescles_recu = Event()
class ConnexionMiddleware: """ Connexion au middleware de la MilleGrille en service. """ def __init__(self, configuration: TransactionConfiguration, client_docker: docker.DockerClient, service_monitor, certificats: dict, **kwargs): self._configuration = configuration self._docker = client_docker self._service_monitor = service_monitor self._certificats = certificats self._path_secrets: str = kwargs.get('secrets') or '/run/secrets' self._monitor_keycert_file: str self._connexion_relai: ConnexionPrincipal = cast(ConnexionPrincipal, None) self._contexte: ContexteRessourcesDocumentsMilleGrilles = cast(ContexteRessourcesDocumentsMilleGrilles, None) self._thread: Thread = cast(Thread, None) self._channel = None self._fermeture_event = Event() self._certificat_event_handler: GestionnaireEvenementsCertificat self.__commandes_handler: TraitementMessagesMiddleware = cast(TraitementMessagesMiddleware, None) self.__transfert_local_handler: TransfertMessages = cast(TransfertMessages, None) self.__monitor_cert_file: str self.__logger = logging.getLogger(__name__ + '.' + self.__class__.__name__) self._comptes_middleware_ok = False self._comptes_mq_ok = False self._prochaine_verification_comptes_noeuds = datetime.datetime.utcnow().timestamp() self._gestionnaire_mdns = kwargs.get('gestionnaire_mdns') def start(self): self.__logger.info("Demarrage ConnexionMiddleware") # Connecter # Demarrer thread self._thread = Thread(target=self.run, name="mw", daemon=True) self._thread.start() def stop(self): self._fermeture_event.set() try: self._contexte.message_dao.deconnecter() self._contexte.document_dao.deconnecter() except Exception: pass def initialiser(self, init_document=True): additionnals = self._contexte_additionnals() self._contexte = ContexteRessourcesDocumentsMilleGrilles( configuration=self._configuration, additionals=additionnals) try: self._contexte.initialiser( init_document=init_document, init_message=True, connecter=True, ) except ServerSelectionTimeoutError: self.__logger.warning("Erreur connexion mongo, ServerSelectionTimeoutError") if self.__logger.isEnabledFor(logging.DEBUG): self.__logger.exception("Detail error connexion Mongo") self._certificat_event_handler = GestionnaireEvenementsCertificat(self._contexte) self.__commandes_handler = TraitementMessagesMiddleware(self._service_monitor.noeud_id, self._service_monitor.gestionnaire_commandes, self._contexte) self._contexte.message_dao.register_channel_listener(self) self._contexte.message_dao.register_channel_listener(self.__commandes_handler) def _contexte_additionnals(self) -> list: ca_certs_file = self._certificats['pki.millegrille.cert'] monitor_cert_file = self._certificats[GestionnaireCertificats.MONITOR_CERT_PATH] monitor_key_file = path.join(self._path_secrets, self._certificats[GestionnaireCertificats.MONITOR_KEY_FILE]) mq_info = self.get_mq_info() self.__logger.info("Information de connexion MQ : %s" % str(mq_info)) additionnals = [{ 'MG_MQ_HOST': mq_info['host'], 'MG_MQ_PORT': mq_info['port'], 'MG_MQ_CA_CERTS': ca_certs_file, 'MG_MQ_CERTFILE': monitor_cert_file, 'MG_MQ_KEYFILE': monitor_key_file, 'MG_MQ_SSL': 'on', 'MG_MQ_AUTH_CERT': 'on', }] return additionnals def get_mq_info(self): self.__logger.debug("Demande services mdns pour idmg %s" % self._service_monitor.idmg) try: services = self._service_monitor.gestionnaire_commandes.requete_mdns_acteur(self._service_monitor.idmg) except Exception: self.__logger.warning("Erreur acces MDNS pour host MQ, tentative utilisation host/port env") host = self.configuration.mq_host port = self.configuration.mq_port else: self.__logger.debug("Services MDNS detectes : %d" % len(services)) for service in services: self.__logger.debug("Service %s port %d, addresses : %s" % (service['type'], service['port'], str(service['addresses']))) services_mq = [s for s in services if s.get('type') == '_mgamqps._tcp.local.'] self.__logger.debug("Services MDNS MQ detectes : %d" % len(services)) for service_mq in services_mq: self.__logger.debug("Service %s port %d, addresses : %s" % (service_mq['type'], service_mq['port'], str(service_mq['addresses']))) try: service_retenu = services_mq[0] host = service_retenu['addresses'][0] port = service_retenu['port'] except IndexError: # Utiliser configuration fourni self.__logger.info("Information MDNS non disponible, fallback sur configuration environnement") host = self.configuration.mq_host port = self.configuration.mq_port info_mq = {'host': host, 'port': port} self.__logger.info("Service MDNS MQ detecte : %s" % str(info_mq)) return info_mq def set_relai(self, connexion_relai: ConnexionPrincipal): if not self.__transfert_local_handler: self.__transfert_local_handler = TransfertMessages( self._contexte, connexion_relai.relayer_message, self._service_monitor.nodename) self._contexte.message_dao.register_channel_listener(self.__transfert_local_handler) def relayer_message(self, message_dict, routing_key, exchange, reply_to=None, correlation_id=None): """ Relai un message recu sur le noeud principal vers le noeud dependant. :param message_dict: :param routing_key: :param exchange: :param reply_to: :param correlation_id: :return: """ headers = {'noeud_source': self._service_monitor.nodename} if reply_to == TransfertMessages.LOCAL_Q_PLACEHOLDER: # Ajouter la Q de transfert locale pour recevoir la reponse a relayer reply_to = self.__transfert_local_handler.queue_name self.generateur_transactions.emettre_message(message_dict, routing_key, [exchange], reply_to, correlation_id, headers) def on_channel_open(self, channel): channel.basic_qos(prefetch_count=1) channel.add_on_close_callback(self.on_channel_close) self._channel = channel self._certificat_event_handler.initialiser() def on_channel_close(self, channel=None, code=None, reason=None): self._channel = None self.__logger.warning("MQ Channel ferme") if not self._fermeture_event.is_set(): try: self._contexte.message_dao.enter_error_state() except Exception: # Erreur d'activation du error state, la connexion ne peut pas etre reactivee self.__logger.exception("Erreur fermeture channel") self._fermeture_event.set() # S'assurer que la fermeture est en cours def __on_return(self, channel, method, properties, body): pass def run(self): self.__logger.info("Thread middleware demarree") while not self._fermeture_event.is_set(): try: self.__entretien_comptes() self._entretien() except Exception: self.__logger.exception("Exception generique") finally: self._fermeture_event.wait(30) self.__logger.info("Fin thread middleware") def __entretien_comptes(self): if not self._comptes_middleware_ok or not self._comptes_mq_ok: comptes_mq_ok = True # Va etre mis a false si un compte n'esp pas ajoute correctement try: idmg = self._configuration.idmg igmd_tronque = idmg[0:12] roles_comptes = [info['role'] for info in MonitorConstantes.DICT_MODULES_PROTEGES.values() if info.get('role')] roles_comptes = ['%s.pki.%s.cert' % (igmd_tronque, role) for role in roles_comptes] roles_mongo = [ ConstantesGenerateurCertificat.ROLE_TRANSACTIONS, ConstantesGenerateurCertificat.ROLE_DOMAINES, ConstantesGenerateurCertificat.ROLE_MAITREDESCLES, ] for role in roles_comptes: filtre = {'name': role} configs = self._docker.configs.list(filters=filtre) if len(configs) > 0: dict_configs = dict() for config in configs: dict_configs[config.name] = config # Choisir plus recent certificat liste_configs_str = list(dict_configs.keys()) liste_configs_str.sort() nom_config = liste_configs_str[-1] config_cert = dict_configs[nom_config] # Extraire certificat cert_pem = b64decode(config_cert.attrs['Spec']['Data']) clecert = EnveloppeCleCert() clecert.cert_from_pem_bytes(cert_pem) # Creer compte roles_cert = clecert.get_roles if any([role in roles_mongo for role in roles_cert]): try: self.__mongo.creer_compte(clecert) except DuplicateKeyError: self.__logger.debug("Compte mongo (deja) cree : %s", nom_config) try: gestionnaire_mq: GestionnaireComptesMQ = self._service_monitor.gestionnaire_mq gestionnaire_mq.ajouter_compte(clecert) except ValueError: comptes_mq_ok = False self._comptes_middleware_ok = True except Exception: self.__logger.exception("Erreur enregistrement comptes") self._comptes_mq_ok = comptes_mq_ok def _entretien(self): ts_courant = datetime.datetime.utcnow().timestamp() exchange = self.exchange # Emettre message de presence du monitor self.emettre_presence() # Transmettre requete pour avoir l'etat de l'hebergement self.generateur_transactions.transmettre_requete( dict(), Constantes.ConstantesHebergement.REQUETE_MILLEGRILLES_ACTIVES, reply_to=self.__commandes_handler.queue_name, correlation_id=ConstantesServiceMonitor.CORRELATION_HEBERGEMENT_LISTE, securite=exchange ) def emettre_presence(self): info_monitor = dict(self._service_monitor.get_info_monitor(inclure_services=True)) info_monitor['noeud_id'] = self._service_monitor.noeud_id info_monitor['securite'] = self._service_monitor.securite domaine_action = Constantes.ConstantesTopologie.EVENEMENT_PRESENCE_MONITOR self.generateur_transactions.emettre_message(info_monitor, domaine_action, ajouter_certificats=True) def ajouter_commande(self, commande): gestionnaire_commandes: GestionnaireCommandes = self._service_monitor.gestionnaire_commandes gestionnaire_commandes.ajouter_commande(commande) def rediriger_messages_domaine(self, nom_domaine: str, exchanges_routing: dict): self.__transfert_local_handler.ajouter_domaine(nom_domaine, exchanges_routing) def enregistrer_listener(self, methode_initialisation): """ Initialise un objet/methode avec le contexte et enregistre le listener retourne :param methode_initialisation: Methode qui va recevoir contexte, doit retourner l'instance du listener :return: """ listener = methode_initialisation(self._contexte) self._contexte.message_dao.register_channel_listener(listener) @property def document_dao(self) -> MongoDAO: return self._contexte.document_dao @property def configuration(self) -> TransactionConfiguration: return self._configuration @property def generateur_transactions(self): return self._contexte.generateur_transactions @property def exchange(self): return self._service_monitor.securite @property def verificateur_transactions(self): return self._contexte.verificateur_transaction
class ModeleConfiguration: def __init__(self): self._logger = logging.getLogger('%s' % self.__class__.__name__) self._logger.setLevel(logging.INFO) self._contexte = ContexteRessourcesDocumentsMilleGrilles() self.parser = None # Parser de ligne de commande self.args = None # Arguments de la ligne de commande self.__fermeture_event = Event() self.__certificat_event_handler: GestionnaireEvenementsCertificat = None self.__channel = None def initialiser(self, init_document=True, init_message=True, connecter=True): # Gerer les signaux OS, permet de deconnecter les ressources au besoin signal.signal(signal.SIGINT, self.exit_gracefully) signal.signal(signal.SIGTERM, self.exit_gracefully) self._contexte.initialiser(init_document=init_document, init_message=init_message, connecter=connecter) if init_message: self._contexte.message_dao.register_channel_listener(self) self.initialiser_2(contexte=self._contexte) def initialiser_2(self, contexte): if contexte is not None: self._contexte = contexte self.__certificat_event_handler = GestionnaireEvenementsCertificat( self._contexte) def on_channel_open(self, channel): channel.basic_qos(prefetch_count=1) channel.add_on_close_callback(self.on_channel_close) self.__channel = channel self.__certificat_event_handler.initialiser() def on_channel_close(self, channel=None, code=None, reason=None): self.__channel = None self._logger.warning("MQ Channel ferme") if not self.__fermeture_event.is_set(): try: self.contexte.message_dao.enter_error_state() except Exception: # Erreur d'activation du error state, la connexion ne peut pas etre reactivee self._logger.exception("Erreur fermeture channel") self.__fermeture_event.set( ) # S'assurer que la fermeture est en cours def __on_return(self, channel, method, properties, body): pass def configurer_parser(self): self.parser = argparse.ArgumentParser( description="Fonctionnalite MilleGrilles") self.parser.add_argument('--verbose', action="store_true", required=False, help="Active le debugging verbose") self.parser.add_argument( '--debug', action="store_true", required=False, help="Active le debugging (logger, tres verbose)") self.parser.add_argument( '--info', action="store_true", required=False, help="Afficher davantage de messages (verbose)") def print_help(self): self.parser.print_help() def exit_gracefully(self, signum=None, frame=None): self.__fermeture_event.set() self.deconnecter() def parse(self): self.args = self.parser.parse_args() def executer(self): raise NotImplemented("Cette methode doit etre redefinie") def connecter(self): if self._contexte.message_dao is not None: self._contexte.message_dao.connecter() if self._contexte.document_dao is not None: self._contexte.document_dao.connecter() def deconnecter(self): self._contexte.fermer() # try: # if self._contexte.message_dao is not None: # self._contexte.message_dao.deconnecter() # except Exception: # self._logger.warning("Erreur fermeture message_dao") # # try: # if self._contexte.document_dao is not None: # self._contexte.document_dao.deconnecter() # except Exception: # self._logger.warning("Erreur fermeture document_dao") def set_logging_level(self): """ Utilise args pour ajuster le logging level (debug, info) """ logging.getLogger('events').setLevel(logging.WARNING) logging.getLogger('pika').setLevel(logging.WARNING) if self.args.debug: self._logger.setLevel(logging.DEBUG) logging.getLogger('millegrilles').setLevel(logging.DEBUG) elif self.args.info: self._logger.setLevel(logging.INFO) logging.getLogger('millegrilles').setLevel(logging.INFO) if self.args.verbose: # Logging de messages frequents, tres verbose logging.getLogger('trace.millegrilles').setLevel(logging.DEBUG) def main(self): code_retour = 0 try: # Preparer logging logging.basicConfig(format=Constantes.LOGGING_FORMAT, level=logging.WARNING) # logging.getLogger('millegrilles.dao.MessageDAO').setLevel(logging.INFO) self._logger.info("\n-----------\n\n-----------") self._logger.info("Demarrage de %s en cours\n-----------" % self.__class__.__name__) # Faire le parsing des arguments pour verifier s'il en manque self.configurer_parser() self.parse() self.set_logging_level() self._logger.info("Initialisation") self.initialiser() # Initialiser les ressources self._logger.info("Debut execution") self.executer() # Executer le download et envoyer message self.__fermeture_event.set() self._logger.info("Fin execution " + self.__class__.__name__) except Exception as e: # print("MAIN: Erreur fatale, voir log. Erreur %s" % str(e)) self._logger.exception("MAIN: Erreur") code_retour = 1 finally: self.exit_gracefully() self._logger.info("Main terminee, finalisation et sortie.") try: self.__finalisation() finally: sys.exit(code_retour) def __finalisation(self): time.sleep(0.2) if threading.active_count() > 1: ok_threads = ['MainThread', 'pymongo_kill_cursors_thread'] for thread in threading.enumerate(): if thread.name not in ok_threads: self._logger.error( "Thread ouverte apres demande de fermeture: %s" % thread.name) time.sleep(5) for thread in threading.enumerate(): if thread.name not in ok_threads: if not thread.isDaemon(): self._logger.warning( "Non-daemon thread encore ouverte apres demande de fermeture: %s" % thread.name) @property def contexte(self) -> ContexteRessourcesDocumentsMilleGrilles: return self._contexte @property def channel(self): return self.__channel @staticmethod def preparer_mongo_keycert(): json_file = os.getenv('MG_CONFIG_JSON') if json_file: with open(json_file, 'r') as fichier: params = json.load(fichier) else: params = os.environ certfile = params['MG_MQ_CERTFILE'] keyfile = params['MG_MQ_KEYFILE'] mongo_keycert_handle, mongo_keycert = tempfile.mkstemp(dir='/tmp', text=True) try: with open(keyfile, 'rb') as fichiers: os.write(mongo_keycert_handle, fichiers.read()) with open(certfile, 'rb') as fichiers: os.write(mongo_keycert_handle, fichiers.read()) os.environ["MG_MONGO_SSL_CERTFILE"] = mongo_keycert finally: os.close(mongo_keycert_handle) return mongo_keycert