class TransmettreCertificatTransactionTest: def __init__(self): self.contexte = ContexteRessourcesMilleGrilles() self.contexte.initialiser(init_message=True) self._logger = logging.getLogger('test') self.generateur_transaction = GenerateurTransaction( self.contexte.configuration, self.contexte.message_dao) self.enveloppe_certificat = None def charger_certificat(self, fichier): verificateur = VerificateurCertificats(self.contexte) self.enveloppe_certificat = verificateur.charger_certificat(fichier) verificateur.verifier_chaine(self.enveloppe_certificat) def transmettre(self): certificat = self.enveloppe_certificat.certificat self._logger.debug("Certificat a transmettre: %s" % str(certificat)) certificat_pem = str( certificat.public_bytes(serialization.Encoding.PEM), 'utf-8') self._logger.debug("Certificat string: %s" % certificat_pem) message = { 'certificat_pem': certificat_pem, 'fingerprint': self.enveloppe_certificat.fingerprint_ascii } domaine = '%s.%s' % (ConstantesPki.DOMAINE_NOM, ConstantesPki.TRANSACTION_EVENEMENT_CERTIFICAT) self.generateur_transaction.soumettre_transaction(message, domaine)
class CeduleurMessageTest: def __init__(self): self.contexte = ContexteRessourcesMilleGrilles() self.contexte.initialiser(init_message=True) def deconnecter(self): self.contexte.message_dao.deconnecter() def transmettre_evenement_ceduleur(self): timestamp_utc = datetime.datetime.now(tz=pytz.UTC) ts_dict = { 'UTC': timestamp_utc.timetuple(), 'joursemaine': timestamp_utc.weekday() } # Faire la liste des timezones a inclure. La routing key va utiliser la version courte de la timezone. timezones = [ pytz.UTC, ] indicateurs = ['heure'] nom_timezones = [] for tz in timezones: local_tz_name = str(tz) nom_timezones.append(local_tz_name) ts_dict['timezones'] = nom_timezones self.contexte.message_dao.transmettre_evenement_ceduleur( ts_dict, indicateurs)
def __init__(self, connecter=True): contexte = ContexteRessourcesMilleGrilles() contexte.initialiser(connecter=connecter) super().__init__(contexte) self.contexte.message_dao.register_channel_listener(self) self.generateur = GenerateurTransaction(self.contexte) self.channel = None self.event_recu = Event() self.messages = list()
def __init__(self): contexte = ContexteRessourcesMilleGrilles() print("contexte.initialiser()") contexte.initialiser() print("ioloop MQ") self.thread_ioloop = Thread(name="MQ-ioloop", target=contexte.message_dao.run_ioloop) self.thread_ioloop.start() print("super.init") super().__init__(contexte, intervalle_secs=5)
def __init__(self): contexte = ContexteRessourcesMilleGrilles() print("contexte.initialiser()") contexte.initialiser(init_document=False) self.document_ids = ['514951f2f43211e99259b827eb53ee51'] super().__init__(contexte, senseur_ids=self.document_ids, timezone_horloge='America/Toronto', intervalle_secs=5)
class BaseMongo: def __init__(self): self._contexte = ContexteRessourcesMilleGrilles() self._contexte.initialiser(init_message=True) self.document_dao = self._contexte.document_dao def deconnecter(self): self.document_dao.deconnecter() @property def contexte(self): return self._contexte
def __init__(self): contexte = ContexteRessourcesMilleGrilles() self.__logger = logging.getLogger(__name__ + '.' + self.__class__.__name__) self.__logger.info("contexte.initialiser()") contexte.initialiser() self.__logger.info("ioloop MQ") self.thread_ioloop = Thread(name="MQ-ioloop", target=contexte.message_dao.run_ioloop) self.thread_ioloop.start() self.__logger.info("super.init") super().__init__(contexte, intervalle_secs=5)
class HachageTest: def __init__(self): self.contexte = ContexteRessourcesMilleGrilles() self.contexte.initialiser() self._signateur = SignateurTransaction(self.contexte.configuration) self._logger = logging.getLogger("HachageTest") self._logger.setLevel(logging.DEBUG) def hacher_fichier(self, path): with open(path) as f: dict_message = json.loads(f.read()) hash = self._signateur.hacher_contenu(dict_message) self._logger.info("Hachage: %s" % hash)
class GenererRapportsFenetresSenseurs: def __init__(self): self._logger = logging.getLogger('%s' % self.__class__.__name__) self._logger.setLevel(logging.INFO) self._contexte = ContexteRessourcesMilleGrilles() self._contexte.initialiser(init_document=True, init_message=True) self._producteur_doc_senseurspassifs = ProducteurDocumentSenseurPassif(self._contexte.document_dao) def calculer_fenetre_horaire(self): self._producteur_doc_senseurspassifs.generer_fenetre_horaire() def calculer_fenetre_derniereheure(self): self._producteur_doc_senseurspassifs.ajouter_derniereheure_fenetre_horaire() def calculer_fenetre_quotidienne(self): self._producteur_doc_senseurspassifs.generer_fenetre_quotidienne() def calculer_fenetre_dernierjour(self): self._producteur_doc_senseurspassifs.ajouter_dernierjour_fenetre_quotidienne()
def __init__(self): contexte = ContexteRessourcesMilleGrilles() contexte.initialiser(init_document=False) super().__init__(contexte) self.__thread_ioloop = Thread( name="MQ-ioloop", target=self.contexte.message_dao.run_ioloop) self.__thread_ioloop.start() self.generateur = self.contexte.generateur_transactions self.pret = Event() self.recu = Event() # Enregistrer la reply-to queue print("Attente du channel") self.contexte.message_dao.attendre_channel(5) self.message_dao.inscrire_topic(self.configuration.exchange_noeuds, [], self.set_cb_queue) # self.channel = self.message_dao.channel # self.channel.queue_declare(durable=True, exclusive=True, callback=self.set_cb_queue) self.queue_name = None self.pret.wait(5)
# Script de test pour transmettre message de transaction import datetime import time import json from millegrilles.dao.Configuration import ContexteRessourcesMilleGrilles from millegrilles.dao.MessageDAO import BaseCallback from millegrilles.transaction.GenerateurTransaction import GenerateurTransaction from millegrilles import Constantes from millegrilles.Constantes import ConstantesDomaines, ConstantesBackup from millegrilles.domaines.Principale import ConstantesPrincipale from threading import Thread, Event contexte = ContexteRessourcesMilleGrilles() contexte.initialiser() class MessagesSample(BaseCallback): def __init__(self): super().__init__(contexte) self.contexte.message_dao.register_channel_listener(self) self.generateur = GenerateurTransaction(self.contexte) self.channel = None self.event_recu = Event() def on_channel_open(self, channel): # Enregistrer la reply-to queue self.channel = channel channel.queue_declare(durable=True,
class ConnexionPrincipal: """ Connexion au noeud protege principal """ # def __init__(self, client_docker: docker.DockerClient, service_monitor: ServiceMonitorDependant): def __init__(self, client_docker: docker.DockerClient, service_monitor): self.__docker = client_docker self.__service_monitor = service_monitor self.__contexte: ContexteRessourcesMilleGrilles = cast(ContexteRessourcesMilleGrilles, None) self.__traitement_messages_principal: TraitementMessagesConnexionPrincipale = cast(TraitementMessagesConnexionPrincipale, None) self.__transfert_messages_principal: TransfertMessages = cast(TransfertMessages, None) def connecter(self): gestionnaire_docker = self.__service_monitor.gestionnaire_docker config_connexion_docker = gestionnaire_docker.charger_config_recente('millegrille.connexion')['config'] config_connexion = json.loads(b64decode(config_connexion_docker.attrs['Spec']['Data'])) # clecert_monitor = self.__service_monitor.clc gestionnaire_certificats = self.__service_monitor.gestionnaire_certificats certificats = gestionnaire_certificats.certificats path_secrets = gestionnaire_certificats.secret_path ca_certs_file = certificats['pki.millegrille.cert'] monitor_cert_file = certificats['pki.%s.cert' % ConstantesGenerateurCertificat.ROLE_MONITOR_DEPENDANT] monitor_key_file = path.join(path_secrets, ConstantesServiceMonitor.DOCKER_CONFIG_MONITOR_DEPENDANT_KEY + '.pem') node_name = config_connexion['principal_mq_url'] additionnals = [{ 'MG_MQ_HOST': node_name, 'MG_MQ_PORT': 5673, 'MG_MQ_CA_CERTS': ca_certs_file, 'MG_MQ_CERTFILE': monitor_cert_file, 'MG_MQ_KEYFILE': monitor_key_file, 'MG_MQ_SSL': 'on', 'MG_MQ_AUTH_CERT': 'on', }] configuration = TransactionConfiguration() self.__contexte = ContexteRessourcesMilleGrilles(configuration=configuration, additionals=additionnals) # Connecter a MQ du noeud principal self.__contexte.initialiser(init_message=True, connecter=True) self.__traitement_messages_principal = TraitementMessagesConnexionPrincipale(self.__service_monitor, self.__contexte) self.__contexte.message_dao.register_channel_listener(self.__traitement_messages_principal) def initialiser_relai_messages(self, fonction_relai): self.__transfert_messages_principal = TransfertMessages( self.__contexte, fonction_relai, self.__service_monitor.nodename) self.__contexte.message_dao.register_channel_listener(self.__transfert_messages_principal) def relayer_message(self, message_dict, routing_key, exchange, reply_to=None, correlation_id=None): """ Relai un message recu vers le noeud principal :param message_dict: :param routing_key: :param exchange: :param reply_to: :param correlation_id: :return: """ headers = {'noeud_source': self.__service_monitor.nodename} if reply_to == TransfertMessages.LOCAL_Q_PLACEHOLDER: # Mettre la queue de relai cote principal pour recevoir la reponse reply_to = self.__transfert_messages_principal.queue_name self.generateur_transactions.emettre_message(message_dict, routing_key, [exchange], reply_to, correlation_id, headers) @property def reply_q(self): return self.__traitement_messages_principal.queue_name @property def generateur_transactions(self): return self.__contexte.generateur_transactions def enregistrer_domaine(self, nom_domaine: str, exchanges_routing: dict): self.__transfert_messages_principal.ajouter_domaine(nom_domaine, exchanges_routing)
class TestVerificateurs: def __init__(self): self._contexte = ContexteRessourcesMilleGrilles() self._contexte.initialiser() self.securite = VerificateurCertificats(self._contexte)
import datetime, time from millegrilles.dao.Configuration import ContexteRessourcesMilleGrilles from millegrilles.dao.MessageDAO import BaseCallback from millegrilles.transaction.GenerateurTransaction import GenerateurTransaction from millegrilles import Constantes from millegrilles.domaines.Principale import ConstantesPrincipale from threading import Thread, Event import json import uuid contexte = ContexteRessourcesMilleGrilles() contexte.initialiser(init_document=False) class MessagesSample(BaseCallback): def __init__(self): super().__init__(contexte) self.contexte.message_dao.register_channel_listener(self) self.generateur = GenerateurTransaction(self.contexte) self.queue_name = None self.channel = None self.event_recu = Event() def on_channel_open(self, channel):
class ModeleConfiguration: def __init__(self): self._logger = logging.getLogger('%s' % self.__class__.__name__) self._logger.setLevel(logging.INFO) self._contexte = ContexteRessourcesMilleGrilles() self.parser = None # Parser de ligne de commande self.args = None # Arguments de la ligne de commande self.__fermeture_event = Event() self.__certificat_event_handler = GestionnaireEvenementsCertificat( self._contexte) self.__channel = None def initialiser(self, init_document=False, init_message=True, connecter=True): # Gerer les signaux OS, permet de deconnecter les ressources au besoin signal.signal(signal.SIGINT, self.exit_gracefully) signal.signal(signal.SIGTERM, self.exit_gracefully) self._contexte.initialiser(init_message=init_message, connecter=connecter) if init_message: self._contexte.message_dao.register_channel_listener(self) def on_channel_open(self, channel): channel.basic_qos(prefetch_count=1) channel.add_on_close_callback(self.on_channel_close) self.__channel = channel self.__certificat_event_handler.initialiser() def on_channel_close(self, channel=None, code=None, reason=None): self.__channel = None self._logger.warning("MQ Channel ferme") if not self.__fermeture_event.is_set(): self.contexte.message_dao.enter_error_state() def __on_return(self, channel, method, properties, body): pass def configurer_parser(self): self.parser = argparse.ArgumentParser( description="Fonctionnalite MilleGrilles") self.parser.add_argument( '--debug', action="store_true", required=False, help="Active le debugging (logger, tres verbose)") self.parser.add_argument( '--info', action="store_true", required=False, help="Afficher davantage de messages (verbose)") def print_help(self): self.parser.print_help() def exit_gracefully(self, signum=None, frame=None): self.__fermeture_event.set() self.deconnecter() def parse(self): self.args = self.parser.parse_args() def executer(self): raise NotImplemented("Cette methode doit etre redefinie") def connecter(self): if self._contexte.message_dao is not None: self._contexte.message_dao.connecter() def deconnecter(self): if self._contexte.message_dao is not None: self._contexte.message_dao.deconnecter() def set_logging_level(self): """ Utilise args pour ajuster le logging level (debug, info) """ if self.args.debug: self._logger.setLevel(logging.DEBUG) logging.getLogger('millegrilles').setLevel(logging.DEBUG) logging.getLogger('events').setLevel(logging.WARNING) elif self.args.info: self._logger.setLevel(logging.INFO) logging.getLogger('millegrilles').setLevel(logging.INFO) def main(self): return_code = 0 try: # Preparer logging logging.basicConfig(format=Constantes.LOGGING_FORMAT, level=logging.WARNING) # logging.getLogger('millegrilles.dao.MessageDAO').setLevel(logging.INFO) self._logger.info("\n-----------\n\n-----------") self._logger.info("Demarrage de %s en cours\n-----------" % self.__class__.__name__) # Faire le parsing des arguments pour verifier s'il en manque self.configurer_parser() self.parse() self.set_logging_level() self._logger.info("Initialisation") self.initialiser() # Initialiser les ressources self._logger.info("Debut execution") self.executer() # Executer le download et envoyer message self.__fermeture_event.set() self._logger.info("Fin execution " + self.__class__.__name__) except Exception as e: return_code = 1 print("MAIN: Erreur fatale, voir log. Erreur %s" % str(e)) self._logger.exception("MAIN: Erreur") self.print_help() finally: self.exit_gracefully() self._logger.info("Main terminee, attente cleanup") time.sleep(0.2) self._logger.info("Main terminee, exit.") if threading.active_count() > 1: ok_threads = ['MainThread', 'pymongo_kill_cursors_thread'] for thread in threading.enumerate(): if thread.name not in ok_threads: self._logger.warning( "Thread ouverte apres demande de fermeture: %s" % thread.name) time.sleep(5) thread_encore_ouverte = False for thread in threading.enumerate(): if thread.name not in ok_threads: self._logger.error( "Thread encore ouverte apres demande de fermeture: %s" % thread.name) thread_encore_ouverte = True if thread_encore_ouverte: self._logger.error( "Threads encore ouvertes, on force la sortie") sys.exit(return_code) @property def contexte(self) -> ContexteRessourcesMilleGrilles: return self._contexte @property def channel(self): return self.__channel
class Generateur: def __init__(self, args): self._args = args # Charger signateur de transaction self._contexte = ContexteRessourcesMilleGrilles() self._contexte.initialiser(False, False) self._signateur = SignateurTransaction(self._contexte) self._signateur.initialiser() self._formatteur = FormatteurMessageMilleGrilles( self._contexte.idmg, self._signateur) self.__logger = logging.getLogger(__name__ + '.' + self.__class__.__name__) def generer_catalogue_applications(self): """ Genere les fichiers de configuration d'application et le fichier de catalogue d'applications :return: """ path_catalogues = path.join(self._args.path or '.') path_archives_application = path.join(path_catalogues, 'generes/applications') try: mkdir(path_archives_application) except FileExistsError: pass catalogue_apps = dict() fpconfig, path_config_temp = tempfile.mkstemp() for rep, config in IterateurApplications(path_catalogues): nom_application = config['nom'] self.__logger.debug("Repertoire : %s" % rep) catalogue_apps[nom_application] = {'version': config['version']} # Verifier si on doit creer une archive tar pour cette application # Tous les fichiers sauf docker.json sont inclus et sauvegarde sous une archive tar.xz # dans l'entree de catalogue fichier_app = [f for f in listdir(rep) if f not in ['docker.json']] if len(fichier_app) > 0: with tarfile.open(path_config_temp, 'w:xz') as fichier: # Faire liste de tous les fichiers de configuration de l'application # (exclure docker.json - genere separement) for filename in fichier_app: file_path = path.join(rep, filename) fichier.add(file_path, arcname=filename) # Lire fichier .tar, convertir en base64 with open(path_config_temp, 'rb') as fichier: contenu_tar_b64 = b64encode(fichier.read()) config['scripts'] = contenu_tar_b64.decode('utf-8') # Preparer archive .json.xz avec le fichier de configuration signe et les scripts config = self.signer( config, ConstantesCatalogueApplications. TRANSACTION_CATALOGUE_APPLICATION) path_archive_application = path.join(path_archives_application, nom_application + '.json.xz') with lzma.open(path_archive_application, 'wt') as output: json.dump(config, output) unlink(path_config_temp) # Cleanup fichier temporaire catalogue = {'applications': catalogue_apps} catalogue = self.signer( catalogue, ConstantesCatalogueApplications.TRANSACTION_CATALOGUE_APPLICATIONS) # Exporter fichier de catalogue path_output = path.join(path_catalogues, 'generes', 'catalogue.applications.json.xz') with lzma.open(path_output, 'wt') as output: json.dump(catalogue, output) def generer_catalogue_domaines(self): """ Generer un fichier de catalogue de tous les domaines :return: """ path_catalogues = path.join(self._args.path or '.') path_fichier_domaines = path.join(path_catalogues, 'domaines.json') catalogue_domaines = dict() for nom_domaine, configuration in IterateurDomaines( path_fichier_domaines): self.__logger.debug("Domaine %s, configuration : %s" % (nom_domaine, configuration)) catalogue_domaines[nom_domaine] = configuration # Generer signature du catalogue catalogue = { 'domaines': catalogue_domaines, } catalogue = self.signer( catalogue, ConstantesCatalogueApplications.TRANSACTION_CATALOGUE_DOMAINES) # Exporter fichier de catalogue path_output = path.join(path_catalogues, 'generes', 'catalogue.domaines.json.xz') with lzma.open(path_output, 'wt') as output: json.dump(catalogue, output) def signer(self, contenu: dict, domaine_action: str): message_signe, uuid_enveloppe = self._formatteur.signer_message( contenu, domaine_action, ajouter_chaine_certs=True) return message_signe def generer(self): self.generer_catalogue_applications() self.generer_catalogue_domaines()