Пример #1
0
def purge_queue():
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
        return queue.purge()
    except OSError as error:
        raise JensMessagingError("Failed to purge Queue object (%s)" % error)
Пример #2
0
def purge_queue():
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
        return queue.purge()
    except OSError, error:
        raise JensMessagingError("Failed to purge Queue object (%s)" % error)
Пример #3
0
def count_pending_hints():
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
        return queue.count()
    except OSError as error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)
Пример #4
0
    def __init__(self, qpath, save_msgs, backend, host, port, db, user, pwd,
                 pidfile):
        '''
        Set up the database details.
        '''
        self._db_host = host
        self._db_port = port
        self._db_name = db
        self._db_username = user
        self._db_pwd = pwd
        # Only here (and in the imports) would we change if we needed
        # to change the DB class used.
        self._apeldb = apel.db.ApelDb(backend, self._db_host, self._db_port,
                                      self._db_username, self._db_pwd,
                                      self._db_name)
        # Check the connection while we're setting up.
        self._apeldb.test_connection()

        inqpath = os.path.join(qpath, "incoming")
        rejectqpath = os.path.join(qpath, "reject")
        self._inq = Queue(inqpath, schema=QSCHEMA)
        self._rejectq = Queue(rejectqpath, schema=REJECT_SCHEMA)

        self._save_msgs = save_msgs
        if self._save_msgs:
            acceptqpath = os.path.join(qpath, "accept")
            self._acceptq = Queue(acceptqpath, schema=QSCHEMA)

        # Create a message factory
        self._rf = RecordFactory()
        self._pidfile = pidfile
        log.info("Loader created.")
Пример #5
0
def check_dir(root):
    '''
    Check the directory for incoming, outgoing, reject
    or accept directories.  If they exist, check them for
    messages.
    '''
    print '\nStarting message status script.'
    print 'Root directory: %s\n' % root
    queues = []
    incoming = os.path.join(root, 'incoming')
    if os.path.isdir(incoming):
        queues.append(Queue(incoming, schema=QSCHEMA))
    outgoing = os.path.join(root, 'outgoing')
    # outgoing uses QueueSimple, not Queue
    if os.path.isdir(outgoing):
        queues.append(QueueSimple(outgoing))
    reject = os.path.join(root, 'reject')
    if os.path.isdir(reject):
        queues.append(Queue(reject, schema=QSCHEMA))
    accept = os.path.join(root, 'accept')
    if os.path.isdir(accept):
        queues.append(Queue(accept, schema=QSCHEMA))

    for q in queues:
        msgs, locked = check_queue(q)
        #check_empty_dirs(q)
        print '    Messages: %s' % msgs
        print '    Locked:   %s\n' % locked
        if locked > 0:
            question = 'Unlock %s messages?' % locked
            if ask_user(question):
                clear_locks(q)
Пример #6
0
def count_pending_hints():
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
        return queue.count()
    except OSError, error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)
Пример #7
0
def _fetch_all_messages():
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
    except OSError as error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)
    msgs = []
    for _, name in enumerate(queue):
        try:
            item = queue.dequeue(name)
        except QueueLockError as error:
            logging.warn("Element %s was locked when dequeuing", name)
            continue
        except OSError as error:
            logging.error("I/O error when getting item %s", name)
            continue
        try:
            item['data'] = pickle.loads(item['data'])
        except (pickle.PickleError, EOFError) as error:
            logging.debug("Couldn't unpickle item %s. Will be ignored.", name)
            continue
        logging.debug("Message %s extracted and unpickled", name)
        msgs.append(item)

    return msgs
Пример #8
0
 def test1_init(self):
     """ QueueSet.__init__() """
     self.failUnlessRaises(TypeError, QueueSet, ('a'))
     self.failUnlessRaises(TypeError, QueueSet,
                           (Queue(self.q1, schema={'data': 'string'}), 'a'))
     self.failUnlessRaises(
         TypeError, QueueSet,
         ([Queue(self.q1, schema={'data': 'string'}), 'a']))
     self.failUnlessRaises(TypeError, QueueSet, ([1, 2]))
     self.failUnlessRaises(TypeError, QueueSet, ((1, 2)))
Пример #9
0
def _queue_item(item):
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
    except OSError as error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)

    try:
        queue.add(item)
    except QueueError as error:
        raise JensMessagingError("Failed to element (%s)" % error)
Пример #10
0
 def test2copy(self):
     """ Queue.copy(). """
     q = Queue(self.qdir, schema={'a': 'string'})
     q1 = q.copy()
     q.foo = 1
     try:
         q1.foo
     except AttributeError:
         pass
     else:
         self.fail('Not a copy, but reference.')
Пример #11
0
 def test2copy(self):
     """ Queue.copy(). """
     q = Queue(self.qdir, schema={'a': 'string'})
     q1 = q.copy()
     q.foo = 1
     try:
         q1.foo
     except AttributeError:
         pass
     else:
         self.fail('Not a copy, but reference.')
Пример #12
0
 def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, 
              capath=None, check_crls=False, use_ssl=False, username=None, password=None, 
              enc_cert=None, verify_enc_cert=True, pidfile=None):
     '''
     Creates an SSM2 object.  If a listen value is supplied,
     this SSM2 will be a receiver.
     '''
     self._conn = None
     self._last_msg = None
         
     self._brokers = hosts_and_ports
     self._cert = cert
     self._key = key
     self._enc_cert = enc_cert
     self._capath = capath
     self._check_crls = check_crls
     self._user = username
     self._pwd = password
     self._use_ssl = use_ssl
     # use pwd auth if we're supplied both user and pwd
     self._use_pwd = username is not None and password is not None
     self.connected = False
     
     self._listen = listen
     self._dest = dest
     
     self._valid_dns = []
     self._pidfile = pidfile
     
     # create the filesystem queues for accepted and rejected messages
     if dest is not None and listen is None:
         self._outq = QueueSimple(qpath)
     elif listen is not None:
         inqpath = os.path.join(qpath, 'incoming')
         rejectqpath = os.path.join(qpath, 'reject')
         self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
         self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
     else:
         raise Ssm2Exception('SSM must be either producer or consumer.')
     # check that the cert and key match
     if not crypto.check_cert_key(self._cert, self._key):
         raise Ssm2Exception('Cert and key don\'t match.')
     # check the server certificate provided
     if enc_cert is not None:
         log.info('Messages will be encrypted using %s', enc_cert)
         if not os.path.isfile(self._enc_cert):
             raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
         if verify_enc_cert:
             if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                 raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.' 
                                      % (self._enc_cert, self._capath))
Пример #13
0
 def test1init(self):
     """ Queue.__init__() """
     path = self.tempdir + '/aaa/bbb/ccc'
     umask = None
     maxelts = 10
     good_schemas = [
         {
             'a': 'binary'
         },
         {
             'a': 'binary*'
         },
         {
             'a': 'string'
         },
         {
             'a': 'string*'
         },
         {
             'a': 'table'
         },
         {
             'a': 'binary?',
             'b': 'binary'
         },
         {
             'a': 'binary?',
             'b': 'binary*'
         },
         {
             'a': 'string',
             'b': 'binary?*'
         },
     ]
     for schema in good_schemas:
         try:
             Queue(path, umask=umask, maxelts=maxelts, schema=schema)
         except Exception:
             error = sys.exc_info()[1]
             self.fail("Shouldn't have failed. Exception: %s" % error)
     bad_schemas = [['foo'], {'foo': 1}, {'a': 'binary?'}]
     for schema in bad_schemas:
         self.failUnlessRaises(QueueError,
                               Queue,
                               path,
                               umask=umask,
                               maxelts=maxelts,
                               schema=schema)
     bad_schemas = [{'a': 'strings'}, {'a': 'table??'}]
     for schema in bad_schemas:
         self.failUnlessRaises(QueueError,
                               Queue,
                               path,
                               umask=umask,
                               maxelts=maxelts,
                               schema=schema)
Пример #14
0
 def test3_firstnext(self):
     """ QueueSet.first()/next() """
     q1 = Queue(self.q1, schema={'data': 'string'})
     q2 = Queue(self.q2, schema={'data': 'string'})
     for i in range(10):
         q1.add({'data': '%i A\n' % i})
         q2.add({'data': '%i A\n' % i})
     qs = QueueSet([q1, q2])
     e = qs.first()
     assert isinstance(e, tuple)
     assert isinstance(e[0], Queue)
     assert isinstance(e[1], str)
     while e[0]:
         e = qs.next()
Пример #15
0
 def test3_firstnext(self):
     """ QueueSet.first()/next() """
     q1 = Queue(self.q1, schema={'data': 'string'})
     q2 = Queue(self.q2, schema={'data': 'string'})
     for i in range(10):
         q1.add({'data': '%i A\n' % i})
         q2.add({'data': '%i A\n' % i})
     qs = QueueSet([q1, q2])
     e = qs.first()
     assert isinstance(e, tuple)
     assert isinstance(e[0], Queue)
     assert isinstance(e[1], str)
     while e[0]:
         e = qs.next()
Пример #16
0
Файл: ssm2.py Проект: pkoro/ssm
class Ssm2(object):
    """
    Minimal SSM implementation.
    """

    # Schema for the dirq message queue.
    QSCHEMA = {"body": "string", "signer": "string", "empaid": "string?"}
    REJECT_SCHEMA = {"body": "string", "signer": "string?", "empaid": "string?", "error": "string"}
    CONNECTION_TIMEOUT = 10

    def __init__(
        self,
        hosts_and_ports,
        qpath,
        cert,
        key,
        dest=None,
        listen=None,
        capath=None,
        check_crls=False,
        use_ssl=False,
        username=None,
        password=None,
        enc_cert=None,
        verify_enc_cert=True,
        pidfile=None,
    ):
        """
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        """
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, "incoming")
            rejectqpath = os.path.join(qpath, "reject")
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception("SSM must be either producer or consumer.")
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception("Cert and key don't match.")
        # check the server certificate provided
        if enc_cert is not None:
            log.info("Messages will be encrypted using %s" % enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception("Specified certificate file does not exist: %s." % self._enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                    raise Ssm2Exception(
                        "Failed to verify server certificate %s against CA path %s." % (self._enc_cert, self._capath)
                    )

    def set_dns(self, dn_list):
        """
        Set the list of DNs which are allowed to sign incoming messages.
        """
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, headers, unused_body):
        """
        Called by stomppy when a message is sent.
        """
        log.debug("Sent message: " + headers["empa-id"])

    def on_message(self, headers, body):
        """
        Called by stomppy when a message is received.
        
        Handle the message according to its content and headers.
        """
        try:
            empaid = headers["empa-id"]
            if empaid == "ping":  # ignore ping message
                log.info("Received ping message.")
                return
        except KeyError:
            empaid = "noid"

        log.info("Received message: " + empaid)
        raw_msg, signer = self._handle_msg(body)

        try:
            if raw_msg is None:  # the message has been rejected
                log.warn("Message rejected.")
                if signer is None:  # crypto failed
                    err_msg = "Could not extract message."
                    log.warn(err_msg)
                    signer = "Not available."
                else:  # crypto ok but signer not verified
                    err_msg = "Signer not in valid DNs list."
                    log.warn(err_msg)

                self._rejectq.add({"body": body, "signer": signer, "empaid": empaid, "error": err_msg})
            else:  # message verified ok
                self._inq.add({"body": raw_msg, "signer": signer, "empaid": headers["empa-id"]})
        except OSError, e:
            log.error("Failed to read or write file: %s" % e)
Пример #17
0
 def test2_addremove(self):
     """ QueueSet.add()/remove() """
     q1 = Queue(self.q1, schema={'data': 'string'})
     q2 = Queue(self.q2, schema={'data': 'string'})
     q3 = Queue(self.q3, schema={'data': 'string'})
     q4 = Queue(self.q4, schema={'data': 'string'})
     for i in range(10):
         q1.add({'data': '%i A\n' % i})
         q2.add({'data': '%i A\n' % i})
         q3.add({'data': '%i A\n' % i})
         q4.add({'data': '%i A\n' % i})
     qs = QueueSet([q1, q2])
     qs.add(q3)
     qs.remove(q1)
     qs.add(q1, q4)
     self.failUnlessRaises(QueueError, qs.add, ([q1]))
Пример #18
0
"""Tool to convert unloaded messages to loadable ones."""

from dirq.QueueSimple import QueueSimple
from dirq.queue import Queue

inpath = '/root/iris/messages'
outpath = '/root/iris/messages/incoming'

OUTQSCHEMA = {"body": "string", "signer": "string",
              "empaid": "string?", "error": "string?"}

inq = QueueSimple(inpath)
outq = Queue(outpath, schema=OUTQSCHEMA)

for name in inq:
    if not inq.lock(name):
        continue
    data = inq.get(name)
    outq.add({'body': data, 'signer': 'iris', 'empaid': 'local'})
    inq.remove(name)
Пример #19
0
import tempfile

# total number of elements
COUNT = 9
# queue head directory
path = tempfile.mkdtemp()
# max elements per elements directory
maxelts = 3
# element's schema
schema = {'body': 'string',
          'header': 'table?'}

# ========
# PRODUCER
print("*** PRODUCER")
dirq_p = Queue(path, maxelts=maxelts, schema=schema)

print("adding %d elements to the queue at %s" % (COUNT, path))
done = 1
while done <= COUNT:
    element = {}
    try:
        element['body'] = ('Élément %i \u263A\n' % done).decode("utf-8")
    except AttributeError:
        element['body'] = 'Élément %i \u263A\n' % done
    if done % 2:  # header only for odd sequential elements
        element['header'] = dict(os.environ)
    name = dirq_p.enqueue(element)
    #name = dirq_p.add(element) # same
    print("added %.2i: %s" % (done, name))
    done += 1
Пример #20
0
    def test3_is_locked(self):
        """ Queue._is_locked_*() """
        q = Queue(self.qdir, schema={'a': 'string'})

        assert q._is_locked_nlink('') is False
        assert q._is_locked_nlink('not_there') is False
        os.mkdir(self.qdir + '/a')
        assert q._is_locked_nlink('a') is False
        os.mkdir(self.qdir + '/a/%s' % queue.LOCKED_DIRECTORY)
        assert q._is_locked_nlink('a') is True
        time.sleep(1)
        assert q._is_locked_nlink('a', time.time()) is True

        assert q._is_locked_nonlink('') is False
        assert q._is_locked_nonlink('not_there') is False
        os.mkdir(self.qdir + '/b')
        assert q._is_locked_nonlink('b') is False
        os.mkdir(self.qdir + '/b/%s' % queue.LOCKED_DIRECTORY)
        assert q._is_locked_nonlink('b') is True
        time.sleep(1)
        assert q._is_locked_nonlink('b', time.time()) is True
Пример #21
0
class Ssm2(stomp.ConnectionListener):
    '''
    Minimal SSM implementation.
    '''
    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer':'string', 'empaid':'string?'}
    REJECT_SCHEMA = {'body': 'string', 'signer':'string?', 'empaid':'string?', 'error':'string'}
    CONNECTION_TIMEOUT = 10
    
    def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, 
                 capath=None, check_crls=False, use_ssl=False, username=None, password=None, 
                 enc_cert=None, verify_enc_cert=True, pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None
            
        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False
        
        self._listen = listen
        self._dest = dest
        
        self._valid_dns = []
        self._pidfile = pidfile
        
        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')
        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                    raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.' 
                                         % (self._enc_cert, self._capath))
            
    
    def set_dns(self, dn_list):
        '''
        Set the list of DNs which are allowed to sign incoming messages.
        '''
        self._valid_dns = dn_list
        
    ##########################################################################
    # Methods called by stomppy
    ##########################################################################
            
    def on_send(self, headers, unused_body):
        '''
        Called by stomppy when a message is sent.
        '''
        log.debug('Sent message: %s', headers['empa-id'])

    def on_message(self, headers, body):
        '''
        Called by stomppy when a message is received.
        
        Handle the message according to its content and headers.
        '''
        try:
            empaid = headers['empa-id']
            if empaid == 'ping': # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info("Received message. ID = %s", empaid)
        raw_msg, signer = self._handle_msg(body)

        try:
            if raw_msg is None:  # the message has been rejected
                if signer is None:  # crypto failed
                    err_msg = 'Could not extract message.'
                    signer = 'Not available.'
                else:  # crypto ok but signer not verified
                    err_msg = 'Signer not in valid DNs list.'
                log.warn("Message rejected: %s", err_msg)

                name = self._rejectq.add({'body': body,
                                          'signer': signer,
                                          'empaid': empaid,
                                          'error': err_msg})
                log.info("Message saved to reject queue as %s", name)
            else:  # message verified ok
                name = self._inq.add({'body': raw_msg,
                                      'signer': signer,
                                      'empaid': empaid})
                log.info("Message saved to incoming queue as %s", name)
        except OSError, e:
            log.error('Failed to read or write file: %s', e)
Пример #22
0
    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')
        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s' % enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))
Пример #23
0
    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired. Please obtain the '
                    'new one from the final server receiving your messages.' %
                    enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see log entries from stomp.py at the warning level and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see log entries from stomp.py at the info level and above.
        elif logging.getLogger(
                "ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)
Пример #24
0
    def post(self, request, format=None):
        """
        Submit Cloud Accounting Records.

        .../api/v1/cloud/record

        Will save Cloud Accounting Records for later loading.
        """
        try:
            empaid = request.META['HTTP_EMPA_ID']
        except KeyError:
            empaid = 'noid'

        self.logger.info("Received message. ID = %s", empaid)

        try:
            signer = request.META['SSL_CLIENT_S_DN']
        except KeyError:
            self.logger.error("No DN supplied in header")
            return Response(status=401)

        # authorise DNs here
        if not self._signer_is_valid(signer):
            self.logger.error("%s not a valid provider", signer)
            return Response(status=403)

        if "_content" in request.POST.dict():
            # then POST likely to come via the rest api framework
            # hence use the content of request.POST as message
            body = request.POST.get('_content')

        else:
            # POST likely to comes through a browser client or curl
            # hence use request.body as message
            body = request.body

        self.logger.debug("Message body received: %s", body)

        for header in request.META:
            self.logger.debug("%s: %s", header, request.META[header])

        # taken from ssm2
        QSCHEMA = {'body': 'string',
                   'signer': 'string',
                   'empaid': 'string?'}

        inqpath = os.path.join(settings.QPATH, 'incoming')
        inq = Queue(inqpath, schema=QSCHEMA)

        try:
            name = inq.add({'body': body,
                            'signer': signer,
                            'empaid': empaid})
        except QueueError as err:
            self.logger.error("Could not save %s/%s: %s", inqpath, name, err)

            response = "Data could not be saved to disk, please try again."
            return Response(response, status=500)

        self.logger.info("Message saved to in queue as %s/%s", inqpath, name)

        response = "Data successfully saved for future loading."
        return Response(response, status=202)
Пример #25
0
# number of queues
QUEUES = 4

print("*** Setup & populate queues")
# generate paths
paths = []
for i in range(QUEUES):
    paths.append(working_dir + '/q-%i' % i)
COUNT = 5

print("creating %i initial queues. adding %i elements into each." %
      (QUEUES, COUNT))
queues = []
queue_num = 0
while queue_num < QUEUES:
    queue = Queue(paths[queue_num], maxelts=5, schema={'body': 'string'})
    print("adding %d elements to the queue %s" % (COUNT, paths[queue_num]))
    element = {}
    done = 0
    while not COUNT or done < COUNT:
        done += 1
        element['body'] = 'Queue %i. Element %i' % (queue_num, done)
        queue.add(element)
    queues.append(queue)
    queue_num += 1
print("done.")

print("*** Browse")
i = 2
queue_set = QueueSet(queues[0:i])
queue_set_count = queue_set.count()
Пример #26
0
 def test2_addremove(self):
     """ QueueSet.add()/remove() """
     q1 = Queue(self.q1, schema={'data': 'string'})
     q2 = Queue(self.q2, schema={'data': 'string'})
     q3 = Queue(self.q3, schema={'data': 'string'})
     q4 = Queue(self.q4, schema={'data': 'string'})
     for i in range(10):
         q1.add({'data': '%i A\n' % i})
         q2.add({'data': '%i A\n' % i})
         q3.add({'data': '%i A\n' % i})
         q4.add({'data': '%i A\n' % i})
     qs = QueueSet([q1, q2])
     qs.add(q3)
     qs.remove(q1)
     qs.add(q1, q4)
     self.failUnlessRaises(QueueError, qs.add, ([q1]))
Пример #27
0
class Ssm2(stomp.ConnectionListener):
    '''
    Minimal SSM implementation.
    '''
    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
    REJECT_SCHEMA = {
        'body': 'string',
        'signer': 'string?',
        'empaid': 'string?',
        'error': 'string'
    }
    CONNECTION_TIMEOUT = 10

    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired. Please obtain the '
                    'new one from the final server receiving your messages.' %
                    enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see log entries from stomp.py at the warning level and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see log entries from stomp.py at the info level and above.
        elif logging.getLogger(
                "ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)

    def set_dns(self, dn_list):
        '''
        Set the list of DNs which are allowed to sign incoming messages.
        '''
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, frame, unused_body=None):
        '''
        Called by stomppy when a message is sent.

        unused_body is only present to have a backward compatible
        method signature when using stomp.py v3.1.X
        '''
        try:
            # Try the stomp.py v4 way first
            empaid = frame.headers['empa-id']
        except KeyError:
            # Then you are most likely using stomp.py v4.
            # on_send is now triggered on non message frames
            # (such as 'CONNECT' frames) and as such without an empa-id.
            empaid = 'no empa-id'
        except AttributeError:
            # Then you are likely using stomp.py v3
            empaid = frame['empa-id']

        log.debug('Sent message: %s', empaid)

    def on_message(self, headers, body):
        '''
        Called by stomppy when a message is received.

        Handle the message according to its content and headers.
        '''
        try:
            empaid = headers['empa-id']
            if empaid == 'ping':  # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info("Received message. ID = %s", empaid)
        extracted_msg, signer, err_msg = self._handle_msg(body)

        try:
            # If the message is empty or the error message is not empty
            # then reject the message.
            if extracted_msg is None or err_msg is not None:
                if signer is None:  # crypto failed
                    signer = 'Not available.'
                elif extracted_msg is not None:
                    # If there is a signer then it was rejected for not being
                    # in the DNs list, so we can use the extracted msg, which
                    # allows the msg to be reloaded if needed.
                    body = extracted_msg

                log.warn("Message rejected: %s", err_msg)

                name = self._rejectq.add({
                    'body': body,
                    'signer': signer,
                    'empaid': empaid,
                    'error': err_msg
                })
                log.info("Message saved to reject queue as %s", name)

            else:  # message verified ok
                name = self._inq.add({
                    'body': extracted_msg,
                    'signer': signer,
                    'empaid': empaid
                })
                log.info("Message saved to incoming queue as %s", name)

        except (IOError, OSError) as e:
            log.error('Failed to read or write file: %s', e)

    def on_error(self, unused_headers, body):
        '''
        Called by stomppy when an error frame is received.
        '''
        log.warn('Error message received: %s', body)
        raise Ssm2Exception()

    def on_connected(self, unused_headers, unused_body):
        '''
        Called by stomppy when a connection is established.

        Track the connection.
        '''
        self.connected = True
        log.info('Connected.')

    def on_disconnected(self):
        '''
        Called by stomppy when disconnected from the broker.
        '''
        log.info('Disconnected from broker.')
        self.connected = False

    def on_receipt(self, headers, unused_body):
        '''
        Called by stomppy when the broker acknowledges receipt of a message.
        '''
        log.info('Broker received message: %s', headers['receipt-id'])
        self._last_msg = headers['receipt-id']

    def on_receiver_loop_completed(self, _unused_headers, _unused_body):
        """
        Called by stompy when the receiver loop ends.

        This is usually trigger as part of a disconnect.
        """
        log.debug('on_receiver_loop_completed called.')

    ##########################################################################
    # Message handling methods
    ##########################################################################

    def _handle_msg(self, text):
        '''
        Deal with the raw message contents appropriately:
        - decrypt if necessary
        - verify signature
        Return plain-text message, signer's DN and an error/None.
        '''
        if text is None or text == '':
            warning = 'Empty text passed to _handle_msg.'
            log.warn(warning)
            return None, None, warning
#        if not text.startswith('MIME-Version: 1.0'):
#            raise Ssm2Exception('Not a valid message.')

# encrypted - this could be nicer
        if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
            try:
                text = crypto.decrypt(text, self._cert, self._key)
            except crypto.CryptoException as e:
                error = 'Failed to decrypt message: %s' % e
                log.error(error)
                return None, None, error

        # always signed
        try:
            message, signer = crypto.verify(text, self._capath,
                                            self._check_crls)
        except crypto.CryptoException as e:
            error = 'Failed to verify message: %s' % e
            log.error(error)
            return None, None, error

        if signer not in self._valid_dns:
            warning = 'Signer not in valid DNs list: %s' % signer
            log.warn(warning)
            return None, signer, warning
        else:
            log.info('Valid signer: %s', signer)

        return message, signer, None

    def _send_msg(self, message, msgid):
        '''
        Send one message using stomppy.  The message will be signed using
        the host cert and key.  If an encryption certificate
        has been supplied, the message will also be encrypted.
        '''
        log.info('Sending message: %s', msgid)
        headers = {
            'destination': self._dest,
            'receipt': msgid,
            'empa-id': msgid
        }

        if message is not None:
            to_send = crypto.sign(message, self._cert, self._key)
            if self._enc_cert is not None:
                to_send = crypto.encrypt(to_send, self._enc_cert)
        else:
            to_send = ''

        try:
            # Try using the v4 method signiture
            self._conn.send(self._dest, to_send, headers=headers)
        except TypeError:
            # If it fails, use the v3 metod signiture
            self._conn.send(to_send, headers=headers)

    def send_ping(self):
        '''
        If a STOMP connection is left open with no activity for an hour or
        so, it stops responding. Stomppy 3.1.3 has two ways of handling
        this, but stomppy 3.0.3 (EPEL 5 and 6) has neither.
        To get around this, we begin and then abort a STOMP transaction to
        keep the connection active.
        '''
        # Use time as transaction id to ensure uniqueness within each connection
        transaction_id = str(time.time())

        self._conn.begin({'transaction': transaction_id})
        self._conn.abort({'transaction': transaction_id})

    def has_msgs(self):
        '''
        Return True if there are any messages in the outgoing queue.
        '''
        return self._outq.count() > 0

    def send_all(self):
        '''
        Send all the messages in the outgoing queue.
        '''
        log.info('Found %s messages.', self._outq.count())
        for msgid in self._outq:
            if not self._outq.lock(msgid):
                log.warn('Message was locked. %s will not be sent.', msgid)
                continue

            text = self._outq.get(msgid)
            self._send_msg(text, msgid)

            log.info('Waiting for broker to accept message.')
            while self._last_msg is None:
                if not self.connected:
                    raise Ssm2Exception('Lost connection.')

                time.sleep(0.1)

            self._last_msg = None
            self._outq.remove(msgid)

        log.info('Tidying message directory.')
        try:
            # Remove empty dirs and unlock msgs older than 5 min (default)
            self._outq.purge()
        except OSError as e:
            log.warn('OSError raised while purging message queue: %s', e)

    ############################################################################
    # Connection handling methods
    ############################################################################

    def _initialise_connection(self, host, port):
        '''
        Create the self._connection object with the appropriate properties,
        but don't try to start the connection.
        '''
        log.info("Established connection to %s, port %i", host, port)
        if self._use_ssl:
            if ssl is None:
                raise ImportError(
                    "SSL connection requested but the ssl module "
                    "wasn't found.")
            log.info('Connecting using SSL...')
        else:
            log.warning("SSL connection not requested, your messages may be "
                        "intercepted.")

        # _conn will use the default SSL version specified by stomp.py
        self._conn = stomp.Connection([(host, port)],
                                      use_ssl=self._use_ssl,
                                      ssl_key_file=self._key,
                                      ssl_cert_file=self._cert,
                                      timeout=Ssm2.CONNECTION_TIMEOUT)

        self._conn.set_listener('SSM', self)

    def handle_connect(self):
        '''
        Assuming that the SSM has retrieved the details of the broker or
        brokers it wants to connect to, connect to one.

        If more than one is in the list self._network_brokers, try to
        connect to each in turn until successful.
        '''
        for host, port in self._brokers:
            self._initialise_connection(host, port)
            try:
                self.start_connection()
                break
            except ConnectFailedException as e:
                # ConnectFailedException doesn't provide a message.
                log.warn('Failed to connect to %s:%s.', host, port)
            except Ssm2Exception as e:
                log.warn('Failed to connect to %s:%s: %s', host, port, e)

        if not self.connected:
            raise Ssm2Exception(
                'Attempts to start the SSM failed.  The system will exit.')

    def handle_disconnect(self):
        '''
        When disconnected, attempt to reconnect using the same method as used
        when starting up.
        '''
        self.connected = False
        # Shut down properly
        self.close_connection()

        # Sometimes the SSM will reconnect to the broker before it's properly
        # shut down!  This prevents that.
        time.sleep(2)

        # Try again according to the same logic as the initial startup
        try:
            self.handle_connect()
        except Ssm2Exception:
            self.connected = False

        # If reconnection fails, admit defeat.
        if not self.connected:
            err_msg = 'Reconnection attempts failed and have been abandoned.'
            raise Ssm2Exception(err_msg)

    def start_connection(self):
        '''
        Once self._connection exists, attempt to start it and subscribe
        to the relevant topics.

        If the timeout is reached without receiving confirmation of
        connection, raise an exception.
        '''
        if self._conn is None:
            raise Ssm2Exception('Called start_connection() before a \
                    connection object was initialised.')

        self._conn.start()
        self._conn.connect(wait=True)

        if self._dest is not None:
            log.info('Will send messages to: %s', self._dest)

        if self._listen is not None:
            # Use a static ID for the subscription ID because we only ever have
            # one subscription within a connection and ID is only considered
            # to differentiate subscriptions within a connection.
            self._conn.subscribe(destination=self._listen, id=1, ack='auto')
            log.info('Subscribing to: %s', self._listen)

        i = 0
        while not self.connected:
            time.sleep(0.1)
            if i > Ssm2.CONNECTION_TIMEOUT * 10:
                err = 'Timed out while waiting for connection. '
                err += 'Check the connection details.'
                raise Ssm2Exception(err)
            i += 1

    def close_connection(self):
        '''
        Close the connection.  This is important because it runs
        in a separate thread, so it can outlive the main process
        if it is not ended.
        '''
        try:
            self._conn.disconnect()
        except (stomp.exception.NotConnectedException, socket.error):
            self._conn = None
        except AttributeError:
            # AttributeError if self._connection is None already
            pass

        log.info('SSM connection ended.')

    def startup(self):
        '''
        Create the pidfile then start the connection.
        '''
        if self._pidfile is not None:
            try:
                f = open(self._pidfile, 'w')
                f.write(str(os.getpid()))
                f.write('\n')
                f.close()
            except IOError as e:
                log.warn('Failed to create pidfile %s: %s', self._pidfile, e)

        self.handle_connect()

    def shutdown(self):
        '''
        Close the connection then remove the pidfile.
        '''
        self.close_connection()
        if self._pidfile is not None:
            try:
                if os.path.exists(self._pidfile):
                    os.remove(self._pidfile)
                else:
                    log.warn('pidfile %s not found.', self._pidfile)
            except IOError as e:
                log.warn('Failed to remove pidfile %s: %e', self._pidfile, e)
                log.warn('SSM may not start again until it is removed.')
Пример #28
0
class Ssm2(stomp.ConnectionListener):
    """Minimal SSM implementation."""

    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
    REJECT_SCHEMA = {'body': 'string', 'signer': 'string?',
                     'empaid': 'string?', 'error': 'string'}
    CONNECTION_TIMEOUT = 10

    # Messaging protocols
    STOMP_MESSAGING = 'STOMP'
    AMS_MESSAGING = 'AMS'

    def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None,
                 capath=None, check_crls=False, use_ssl=False, username=None, password=None,
                 enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq',
                 protocol=STOMP_MESSAGING, project=None, token=''):
        """Create an SSM2 object.

        If a listen value is supplied, this SSM2 will be a receiver.
        """
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # Used to differentiate between STOMP and AMS methods
        self._protocol = protocol

        # Used when interacting with an Argo Messaging Service
        self._project = project
        self._token = token

        if self._protocol == Ssm2.AMS_MESSAGING:
            if ArgoMessagingService is None:
                raise ImportError(
                    "The Python package argo_ams_library must be installed to "
                    "use AMS. Please install or use STOMP."
                )
            self._ams = ArgoMessagingService(endpoint=self._brokers[0],
                                             token=self._token,
                                             cert=self._cert,
                                             key=self._key,
                                             project=self._project)

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            # Determine what sort of outgoing structure to make
            if path_type == 'dirq':
                if QueueSimple is None:
                    raise ImportError("dirq path_type requested but the dirq "
                                      "module wasn't found.")

                self._outq = QueueSimple(qpath)

            elif path_type == 'directory':
                self._outq = MessageDirectory(qpath)
            else:
                raise Ssm2Exception('Unsupported path_type variable.')

        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')

            # Receivers must use the dirq module, so make a quick sanity check
            # that dirq is installed.
            if Queue is None:
                raise ImportError("Receiving SSMs must use dirq, but the dirq "
                                  "module wasn't found.")

            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired or will expire '
                                'within a day.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired or will expire '
                    'within a day. Please obtain the new one from the final '
                    'server receiving your messages.' % enc_cert
                )
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                    raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.'
                                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see entries from stomp.py and connectionpool at WARNING and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
            logging.getLogger("requests.packages.urllib3.connectionpool"
                              ).setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see entries from stomp.py and connectionpool at INFO above.
        elif logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)
            logging.getLogger("requests.packages.urllib3.connectionpool"
                              ).setLevel(logging.INFO)

    def set_dns(self, dn_list):
        """Set the list of DNs which are allowed to sign incoming messages."""
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, frame, unused_body=None):
        """Log sending of message with empaid if available.

        Called by stomppy when a message is sent. 'unused_body' is only present
        to have a backward compatible method signature for stomp.py v3.1.X
        """
        try:
            # Try the stomp.py v4 way first
            empaid = frame.headers['empa-id']
        except KeyError:
            # Then you are most likely using stomp.py v4.
            # on_send is now triggered on non message frames
            # (such as 'CONNECT' frames) and as such without an empa-id.
            empaid = 'no empa-id'
        except AttributeError:
            # Then you are likely using stomp.py v3
            empaid = frame['empa-id']

        log.debug('Sent message: %s', empaid)

    def on_message(self, headers, body):
        """Handle the message according to its content and headers.

        Called by stomppy when a message is received.
        """
        try:
            empaid = headers['empa-id']
            if empaid == 'ping':  # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info("Received message. ID = %s", empaid)
        # Save the message to either accept or reject queue.
        self._save_msg_to_queue(body, empaid)

    def on_error(self, headers, body):
        """Log error messages.

        Called by stomppy when an error frame is received.
        """
        if 'No user for client certificate: ' in headers['message']:
            log.error('The following certificate is not authorised: %s',
                      headers['message'].split(':')[1])
        else:
            log.error('Error message received: %s', body)

    def on_connected(self, unused_headers, unused_body):
        """Track the connection.

        Called by stomppy when a connection is established.
        """
        self.connected = True
        log.info('Connected.')

    def on_disconnected(self):
        """Log disconnection and set 'connected' to 'False'.

        Called by stomppy when disconnected from the broker.
        """
        log.info('Disconnected from broker.')
        self.connected = False

    def on_receipt(self, headers, unused_body):
        """Log receipt of message by broker and set '_last_msg'.

        Called by stomppy when the broker acknowledges receipt of a message.
        """
        log.info('Broker received message: %s', headers['receipt-id'])
        self._last_msg = headers['receipt-id']

    def on_receiver_loop_completed(self, _unused_headers, _unused_body):
        """Log receiver loop complete for debug only.

        Called by stompy when the receiver loop ends. This is usually triggered
        as part of a disconnect.
        """
        log.debug('on_receiver_loop_completed called.')

    ##########################################################################
    # Message handling methods
    ##########################################################################

    def _handle_msg(self, text):
        """Deal with the raw message contents appropriately.

        Namely:
        - decrypt if necessary
        - verify signature
        - Return plain-text message, signer's DN and an error/None.
        """
        if text is None or text == '':
            warning = 'Empty text passed to _handle_msg.'
            log.warn(warning)
            return None, None, warning
#        if not text.startswith('MIME-Version: 1.0'):
#            raise Ssm2Exception('Not a valid message.')

        # encrypted - this could be nicer
        if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
            try:
                text = crypto.decrypt(text, self._cert, self._key)
            except crypto.CryptoException as e:
                error = 'Failed to decrypt message: %s' % e
                log.error(error)
                return None, None, error

        # always signed
        try:
            message, signer = crypto.verify(text, self._capath, self._check_crls)
        except crypto.CryptoException as e:
            error = 'Failed to verify message: %s' % e
            log.error(error)
            return None, None, error

        if signer not in self._valid_dns:
            warning = 'Signer not in valid DNs list: %s' % signer
            log.warn(warning)
            return None, signer, warning
        else:
            log.info('Valid signer: %s', signer)

        return message, signer, None

    def _save_msg_to_queue(self, body, empaid):
        """Extract message contents and add to the accept or reject queue."""
        extracted_msg, signer, err_msg = self._handle_msg(body)
        try:
            # If the message is empty or the error message is not empty
            # then reject the message.
            if extracted_msg is None or err_msg is not None:
                if signer is None:  # crypto failed
                    signer = 'Not available.'
                elif extracted_msg is not None:
                    # If there is a signer then it was rejected for not being
                    # in the DNs list, so we can use the extracted msg, which
                    # allows the msg to be reloaded if needed.
                    body = extracted_msg

                log.warn("Message rejected: %s", err_msg)

                name = self._rejectq.add({'body': body,
                                          'signer': signer,
                                          'empaid': empaid,
                                          'error': err_msg})
                log.info("Message saved to reject queue as %s", name)

            else:  # message verified ok
                name = self._inq.add({'body': extracted_msg,
                                      'signer': signer,
                                      'empaid': empaid})
                log.info("Message saved to incoming queue as %s", name)

        except (IOError, OSError) as error:
            log.error('Failed to read or write file: %s', error)

    def _send_msg(self, message, msgid):
        """Send one message using stomppy.

        The message will be signed using the host cert and key. If an
        encryption certificate has been supplied, it will also be encrypted.
        """
        log.info('Sending message: %s', msgid)
        headers = {'destination': self._dest, 'receipt': msgid,
                   'empa-id': msgid}

        if message is not None:
            to_send = crypto.sign(message, self._cert, self._key)
            if self._enc_cert is not None:
                to_send = crypto.encrypt(to_send, self._enc_cert)
        else:
            to_send = ''

        try:
            # Try using the v4 method signiture
            self._conn.send(self._dest, to_send, headers=headers)
        except TypeError:
            # If it fails, use the v3 metod signiture
            self._conn.send(to_send, headers=headers)

    def _send_msg_ams(self, text, msgid):
        """Send one message using AMS, returning the AMS ID of the mesage.

        The message will be signed using the host cert and key. If an
        encryption certificate has been supplied, the message will also be
        encrypted.
        """
        log.info('Sending message: %s', msgid)
        if text is not None:
            # First we sign the message
            to_send = crypto.sign(text, self._cert, self._key)
            # Possibly encrypt the message.
            if self._enc_cert is not None:
                to_send = crypto.encrypt(to_send, self._enc_cert)
            # Then we need to wrap text up as an AMS Message.
            message = AmsMessage(data=to_send,
                                 attributes={'empaid': msgid}).dict()

            argo_response = self._ams.publish(self._dest, message, retry=3)
            return argo_response['messageIds'][0]

    def pull_msg_ams(self):
        """Pull 1 message from the AMS and acknowledge it."""
        if self._protocol != Ssm2.AMS_MESSAGING:
            # Then this method should not be called,
            # raise an exception if it is.
            raise Ssm2Exception('pull_msg_ams called, '
                                'but protocol not set to AMS. '
                                'Protocol: %s' % self._protocol)

        # This method is setup so that you could pull down and
        # acknowledge more than one message at a time, but
        # currently there is no use case for it.
        messages_to_pull = 1
        # ack id's will be stored in this list and then acknowledged
        ackids = []

        for msg_ack_id, msg in self._ams.pull_sub(self._listen,
                                                  messages_to_pull,
                                                  retry=3):
            # Get the AMS message id
            msgid = msg.get_msgid()
            # Get the SSM dirq id
            try:
                empaid = msg.get_attr().get('empaid')
            except AttributeError:
                # A message without an empaid could be received if it wasn't
                # sent via the SSM, we need to pull down that message
                # to prevent it blocking the message queue.
                log.debug("Message %s has no empaid.", msgid)
                empaid = "N/A"
            # get the message body
            body = msg.get_data()

            log.info('Received message. ID = %s, Argo ID = %s', empaid, msgid)
            # Save the message to either accept or reject queue.
            self._save_msg_to_queue(body, empaid)

            # The message has either been saved or there's been a problem with
            # writing it out, but either way we add the ack ID to the list
            # of those to be acknowledged so that we don't get stuck reading
            # the same message.
            ackids.append(msg_ack_id)

        # pass list of extracted ackIds to AMS Service so that
        # it can move the offset for the next subscription pull
        # (basically acknowledging pulled messages)
        if ackids:
            self._ams.ack_sub(self._listen, ackids, retry=3)

    def send_ping(self):
        """Perform connection stay-alive steps.

        If a STOMP connection is left open with no activity for an hour or
        so, it stops responding. Stomppy 3.1.3 has two ways of handling
        this, but stomppy 3.0.3 (EPEL 5 and 6) has neither.
        To get around this, we begin and then abort a STOMP transaction to
        keep the connection active.
        """
        # Use time as transaction id to ensure uniqueness within each connection
        transaction_id = str(time.time())

        self._conn.begin({'transaction': transaction_id})
        self._conn.abort({'transaction': transaction_id})

    def has_msgs(self):
        """Return True if there are any messages in the outgoing queue."""
        return self._outq.count() > 0

    def send_all(self):
        """
        Send all the messages in the outgoing queue.

        Either via STOMP or HTTPS (to an Argo Message Broker).
        """
        log.info('Found %s messages.', self._outq.count())
        for msgid in self._outq:
            if not self._outq.lock(msgid):
                log.warn('Message was locked. %s will not be sent.', msgid)
                continue

            text = self._outq.get(msgid)

            if self._protocol == Ssm2.STOMP_MESSAGING:
                # Then we are sending to a STOMP message broker.
                self._send_msg(text, msgid)

                log.info('Waiting for broker to accept message.')
                while self._last_msg is None:
                    if not self.connected:
                        raise Ssm2Exception('Lost connection.')
                    # Small sleep to avoid hammering the CPU
                    time.sleep(0.01)

                log_string = "Sent %s" % msgid

            elif self._protocol == Ssm2.AMS_MESSAGING:
                # Then we are sending to an Argo Messaging Service instance.
                argo_id = self._send_msg_ams(text, msgid)

                log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id)

            else:
                # The SSM has been improperly configured
                raise Ssm2Exception('Unknown messaging protocol: %s' %
                                    self._protocol)

            # log that the message was sent
            log.info(log_string)

            self._last_msg = None
            self._outq.remove(msgid)

        log.info('Tidying message directory.')
        try:
            # Remove empty dirs and unlock msgs older than 5 min (default)
            self._outq.purge()
        except OSError as e:
            log.warn('OSError raised while purging message queue: %s', e)

    ###########################################################################
    # Connection handling methods
    ###########################################################################

    def _initialise_connection(self, host, port):
        """Create the self._connection object with the appropriate properties.

        This doesn't start the connection.
        """
        log.info("Established connection to %s, port %i", host, port)
        if self._use_ssl:
            log.info('Connecting using SSL...')
        else:
            log.warning("SSL connection not requested, your messages may be "
                        "intercepted.")

        # _conn will use the default SSL version specified by stomp.py
        self._conn = stomp.Connection([(host, port)],
                                      use_ssl=self._use_ssl,
                                      ssl_key_file=self._key,
                                      ssl_cert_file=self._cert,
                                      timeout=Ssm2.CONNECTION_TIMEOUT)

        self._conn.set_listener('SSM', self)

    def handle_connect(self):
        """Connect to broker.

        Assuming that the SSM has retrieved the details of the broker or
        brokers it wants to connect to, connect to one.

        If more than one is in the list self._network_brokers, try to
        connect to each in turn until successful.
        """
        if self._protocol == Ssm2.AMS_MESSAGING:
            log.debug('handle_connect called for AMS, doing nothing.')
            return

        log.info("Using stomp.py version %s.%s.%s.", *stomp.__version__)

        for host, port in self._brokers:
            self._initialise_connection(host, port)
            try:
                self.start_connection()
                break
            except ConnectFailedException as e:
                # ConnectFailedException doesn't provide a message.
                log.warn('Failed to connect to %s:%s.', host, port)
            except Ssm2Exception as e:
                log.warn('Failed to connect to %s:%s: %s', host, port, e)

        if not self.connected:
            raise Ssm2Exception('Attempts to start the SSM failed. The system will exit.')

    def handle_disconnect(self):
        """Attempt to reconnect using the same method as when starting up."""
        if self._protocol == Ssm2.AMS_MESSAGING:
            log.debug('handle_disconnect called for AMS, doing nothing.')
            return

        self.connected = False
        # Shut down properly
        self.close_connection()

        # Sometimes the SSM will reconnect to the broker before it's properly
        # shut down!  This prevents that.
        time.sleep(2)

        # Try again according to the same logic as the initial startup
        try:
            self.handle_connect()
        except Ssm2Exception:
            self.connected = False

        # If reconnection fails, admit defeat.
        if not self.connected:
            err_msg = 'Reconnection attempts failed and have been abandoned.'
            raise Ssm2Exception(err_msg)

    def start_connection(self):
        """Start existing connection and subscribe to the relevant topics.

        If the timeout is reached without receiving confirmation of
        connection, raise an exception.
        """
        if self._protocol == Ssm2.AMS_MESSAGING:
            log.debug('start_connection called for AMS, doing nothing.')
            return

        if self._conn is None:
            raise Ssm2Exception('Called start_connection() before a \
                    connection object was initialised.')

        self._conn.start()
        self._conn.connect(wait=False)

        i = 0
        while not self.connected:
            time.sleep(0.1)
            if i > Ssm2.CONNECTION_TIMEOUT * 10:
                err = 'Timed out while waiting for connection. '
                err += 'Check the connection details.'
                raise Ssm2Exception(err)
            i += 1

        if self._dest is not None:
            log.info('Will send messages to: %s', self._dest)

        if self._listen is not None:
            # Use a static ID for the subscription ID because we only ever have
            # one subscription within a connection and ID is only considered
            # to differentiate subscriptions within a connection.
            self._conn.subscribe(destination=self._listen, id=1, ack='auto')
            log.info('Subscribing to: %s', self._listen)

    def close_connection(self):
        """Close the connection.

        This is important because it runs in a separate thread, so it can
        outlive the main process if it is not ended.
        """
        if self._protocol == Ssm2.AMS_MESSAGING:
            log.debug('close_connection called for AMS, doing nothing.')
            return

        try:
            self._conn.disconnect()
        except (stomp.exception.NotConnectedException, socket.error):
            self._conn = None
        except AttributeError:
            # AttributeError if self._connection is None already
            pass

        log.info('SSM connection ended.')

    def startup(self):
        """Create the pidfile then start the connection."""
        if self._pidfile is not None:
            try:
                f = open(self._pidfile, 'w')
                f.write(str(os.getpid()))
                f.write('\n')
                f.close()
            except IOError as e:
                log.warn('Failed to create pidfile %s: %s', self._pidfile, e)

        self.handle_connect()

    def shutdown(self):
        """Close the connection then remove the pidfile."""
        self.close_connection()
        if self._pidfile is not None:
            try:
                if os.path.exists(self._pidfile):
                    os.remove(self._pidfile)
                else:
                    log.warn('pidfile %s not found.', self._pidfile)
            except IOError as e:
                log.warn('Failed to remove pidfile %s: %e', self._pidfile, e)
                log.warn('SSM may not start again until it is removed.')
Пример #29
0
# Then : cleanup previous runs already emptied by the apeldbloader
for root, dirs, files in os.walk(arcdir,topdown=False):
     for name in dirs:
         fname = os.path.join(root,name)
         if re.compile('[0-9]{8,8}').match(name) and not os.listdir(fname): #to check wither the dir is empty and of the form 01234567 (the dirq tmp dirs)
             logging.info("removing emtpty dir " + fname)
             os.removedirs(fname)

#then, make sure we don't remove CAR records, just archive them
if not os.path.isdir(archives):
    os.mkdir(archives)
if not os.path.isdir(failures):
    os.mkdir(failures)

#init structures and archive file, chdir
dirq = Queue(arcdir, schema=QSCHEMA)
tarball = tarfile.open(tarfileN,mode='a:')
tarmembers = tarball.getnames()
processed=0

try:
  os.chdir(arcdir)
  #create the incoming dir pointing lo cwd, so that apeldbloader is happy
  if not os.path.isdir('incoming'):
    os.symlink('.','incoming')
  #process CAR xml records
  for filename in os.listdir('.'):
      if filename.startswith("usagerecordCAR"):
          doc=xml.dom.minidom.parse(filename)
          #fixNS(doc)
          for node in doc.childNodes:
Пример #30
0
def _fetch_all_messages(settings):
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
    except OSError, error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)
Пример #31
0
    def test3_is_locked(self):
        """ Queue._is_locked_*() """
        q = Queue(self.qdir, schema={'a': 'string'})

        assert q._is_locked_nlink('') is False
        assert q._is_locked_nlink('not_there') is False
        os.mkdir(self.qdir + '/a')
        assert q._is_locked_nlink('a') is False
        os.mkdir(self.qdir + '/a/%s' % queue.LOCKED_DIRECTORY)
        assert q._is_locked_nlink('a') is True
        time.sleep(1)
        assert q._is_locked_nlink('a', time.time()) is True

        assert q._is_locked_nonlink('') is False
        assert q._is_locked_nonlink('not_there') is False
        os.mkdir(self.qdir + '/b')
        assert q._is_locked_nonlink('b') is False
        os.mkdir(self.qdir + '/b/%s' % queue.LOCKED_DIRECTORY)
        assert q._is_locked_nonlink('b') is True
        time.sleep(1)
        assert q._is_locked_nonlink('b', time.time()) is True
Пример #32
0
    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None,
                 path_type='dirq',
                 protocol=STOMP_MESSAGING,
                 project=None,
                 token=''):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # Used to differentiate between STOMP and AMS methods
        self._protocol = protocol

        # Used when interacting with an Argo Messaging Service
        self._project = project
        self._token = token

        if self._protocol == Ssm2.AMS_MESSAGING:
            self._ams = ArgoMessagingService(endpoint=self._brokers[0],
                                             token=self._token,
                                             cert=self._cert,
                                             key=self._key,
                                             project=self._project)

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            # Determine what sort of outgoing structure to make
            if path_type == 'dirq':
                if QueueSimple is None:
                    raise ImportError("dirq path_type requested but the dirq "
                                      "module wasn't found.")

                self._outq = QueueSimple(qpath)

            elif path_type == 'directory':
                self._outq = MessageDirectory(qpath)
            else:
                raise Ssm2Exception('Unsupported path_type variable.')

        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')

            # Receivers must use the dirq module, so make a quick sanity check
            # that dirq is installed.
            if Queue is None:
                raise ImportError("Receiving SSMs must use dirq, but the dirq "
                                  "module wasn't found.")

            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired or will expire '
                                'within a day.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired or will expire '
                    'within a day. Please obtain the new one from the final '
                    'server receiving your messages.' % enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see entries from stomp.py and connectionpool at WARNING and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
            logging.getLogger(
                "requests.packages.urllib3.connectionpool").setLevel(
                    logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see entries from stomp.py and connectionpool at INFO above.
        elif logging.getLogger(
                "ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)
            logging.getLogger(
                "requests.packages.urllib3.connectionpool").setLevel(
                    logging.INFO)
Пример #33
0
class Ssm2(stomp.ConnectionListener):
    '''
    Minimal SSM implementation.
    '''
    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
    REJECT_SCHEMA = {
        'body': 'string',
        'signer': 'string?',
        'empaid': 'string?',
        'error': 'string'
    }
    CONNECTION_TIMEOUT = 10

    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired. Please obtain the '
                    'new one from the final server receiving your messages.' %
                    enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see log entries from stomp.py at the warning level and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see log entries from stomp.py at the info level and above.
        elif logging.getLogger(
                "ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)

    def set_dns(self, dn_list):
        '''
        Set the list of DNs which are allowed to sign incoming messages.
        '''
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, frame, unused_body=None):
        '''
        Called by stomppy when a message is sent.

        unused_body is only present to have a backward compatible
        method signature when using stomp.py v3.1.X
        '''
        try:
            # Try the stomp.py v4 way first
            empaid = frame.headers['empa-id']
        except KeyError:
            # Then you are most likely using stomp.py v4.
            # on_send is now triggered on non message frames
            # (such as 'CONNECT' frames) and as such without an empa-id.
            empaid = 'no empa-id'
        except AttributeError:
            # Then you are likely using stomp.py v3
            empaid = frame['empa-id']

        log.debug('Sent message: %s', empaid)

    def on_message(self, headers, body):
        '''
        Called by stomppy when a message is received.

        Handle the message according to its content and headers.
        '''
        try:
            empaid = headers['empa-id']
            if empaid == 'ping':  # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info("Received message. ID = %s", empaid)
        extracted_msg, signer, err_msg = self._handle_msg(body)

        try:
            # If the message is empty or the error message is not empty
            # then reject the message.
            if extracted_msg is None or err_msg is not None:
                if signer is None:  # crypto failed
                    signer = 'Not available.'
                elif extracted_msg is not None:
                    # If there is a signer then it was rejected for not being
                    # in the DNs list, so we can use the extracted msg, which
                    # allows the msg to be reloaded if needed.
                    body = extracted_msg

                log.warn("Message rejected: %s", err_msg)

                name = self._rejectq.add({
                    'body': body,
                    'signer': signer,
                    'empaid': empaid,
                    'error': err_msg
                })
                log.info("Message saved to reject queue as %s", name)

            else:  # message verified ok
                name = self._inq.add({
                    'body': extracted_msg,
                    'signer': signer,
                    'empaid': empaid
                })
                log.info("Message saved to incoming queue as %s", name)

        except (IOError, OSError) as e:
            log.error('Failed to read or write file: %s', e)

    def on_error(self, unused_headers, body):
        '''
        Called by stomppy when an error frame is received.
        '''
        log.warn('Error message received: %s', body)
        raise Ssm2Exception()

    def on_connected(self, unused_headers, unused_body):
        '''
        Called by stomppy when a connection is established.

        Track the connection.
        '''
        self.connected = True
        log.info('Connected.')

    def on_disconnected(self):
        '''
        Called by stomppy when disconnected from the broker.
        '''
        log.info('Disconnected from broker.')
        self.connected = False

    def on_receipt(self, headers, unused_body):
        '''
        Called by stomppy when the broker acknowledges receipt of a message.
        '''
        log.info('Broker received message: %s', headers['receipt-id'])
        self._last_msg = headers['receipt-id']

    def on_receiver_loop_completed(self, _unused_headers, _unused_body):
        """
        Called by stompy when the receiver loop ends.

        This is usually trigger as part of a disconnect.
        """
        log.debug('on_receiver_loop_completed called.')

    ##########################################################################
    # Message handling methods
    ##########################################################################

    def _handle_msg(self, text):
        '''
        Deal with the raw message contents appropriately:
        - decrypt if necessary
        - verify signature
        Return plain-text message, signer's DN and an error/None.
        '''
        if text is None or text == '':
            warning = 'Empty text passed to _handle_msg.'
            log.warn(warning)
            return None, None, warning
#        if not text.startswith('MIME-Version: 1.0'):
#            raise Ssm2Exception('Not a valid message.')

# encrypted - this could be nicer
        if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
            try:
                text = crypto.decrypt(text, self._cert, self._key)
            except crypto.CryptoException, e:
                error = 'Failed to decrypt message: %s' % e
                log.error(error)
                return None, None, error

        # always signed
        try:
            message, signer = crypto.verify(text, self._capath,
                                            self._check_crls)
        except crypto.CryptoException, e:
            error = 'Failed to verify message: %s' % e
            log.error(error)
            return None, None, error
Пример #34
0
Файл: ssm2.py Проект: apel/ssm
class Ssm2(stomp.ConnectionListener):
    '''
    Minimal SSM implementation.
    '''
    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer':'string', 'empaid':'string?'}
    REJECT_SCHEMA = {'body': 'string', 'signer':'string?', 'empaid':'string?', 'error':'string'}
    CONNECTION_TIMEOUT = 10

    def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None,
                 capath=None, check_crls=False, use_ssl=False, username=None, password=None,
                 enc_cert=None, verify_enc_cert=True, pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired. Please obtain the '
                    'new one from the final server receiving your messages.' %
                    enc_cert
                )
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                    raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.'
                                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see log entries from stomp.py at the warning level and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see log entries from stomp.py at the info level and above.
        elif logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)

    def set_dns(self, dn_list):
        '''
        Set the list of DNs which are allowed to sign incoming messages.
        '''
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, frame, unused_body=None):
        '''
        Called by stomppy when a message is sent.

        unused_body is only present to have a backward compatible
        method signature when using stomp.py v3.1.X
        '''
        try:
            # Try the stomp.py v4 way first
            empaid = frame.headers['empa-id']
        except KeyError:
            # Then you are most likely using stomp.py v4.
            # on_send is now triggered on non message frames
            # (such as 'CONNECT' frames) and as such without an empa-id.
            empaid = 'no empa-id'
        except AttributeError:
            # Then you are likely using stomp.py v3
            empaid = frame['empa-id']

        log.debug('Sent message: %s', empaid)

    def on_message(self, headers, body):
        '''
        Called by stomppy when a message is received.

        Handle the message according to its content and headers.
        '''
        try:
            empaid = headers['empa-id']
            if empaid == 'ping': # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info("Received message. ID = %s", empaid)
        extracted_msg, signer, err_msg = self._handle_msg(body)

        try:
            # If the message is empty or the error message is not empty
            # then reject the message.
            if extracted_msg is None or err_msg is not None:
                if signer is None:  # crypto failed
                    signer = 'Not available.'
                elif extracted_msg is not None:
                    # If there is a signer then it was rejected for not being
                    # in the DNs list, so we can use the extracted msg, which
                    # allows the msg to be reloaded if needed.
                    body = extracted_msg

                log.warn("Message rejected: %s", err_msg)

                name = self._rejectq.add({'body': body,
                                          'signer': signer,
                                          'empaid': empaid,
                                          'error': err_msg})
                log.info("Message saved to reject queue as %s", name)

            else:  # message verified ok
                name = self._inq.add({'body': extracted_msg,
                                      'signer': signer,
                                      'empaid': empaid})
                log.info("Message saved to incoming queue as %s", name)

        except (IOError, OSError) as e:
            log.error('Failed to read or write file: %s', e)

    def on_error(self, unused_headers, body):
        '''
        Called by stomppy when an error frame is received.
        '''
        log.warn('Error message received: %s', body)
        raise Ssm2Exception()

    def on_connected(self, unused_headers, unused_body):
        '''
        Called by stomppy when a connection is established.

        Track the connection.
        '''
        self.connected = True
        log.info('Connected.')

    def on_disconnected(self):
        '''
        Called by stomppy when disconnected from the broker.
        '''
        log.info('Disconnected from broker.')
        self.connected = False

    def on_receipt(self, headers, unused_body):
        '''
        Called by stomppy when the broker acknowledges receipt of a message.
        '''
        log.info('Broker received message: %s', headers['receipt-id'])
        self._last_msg = headers['receipt-id']

    def on_receiver_loop_completed(self, _unused_headers, _unused_body):
        """
        Called by stompy when the receiver loop ends.

        This is usually trigger as part of a disconnect.
        """
        log.debug('on_receiver_loop_completed called.')

    ##########################################################################
    # Message handling methods
    ##########################################################################

    def _handle_msg(self, text):
        '''
        Deal with the raw message contents appropriately:
        - decrypt if necessary
        - verify signature
        Return plain-text message, signer's DN and an error/None.
        '''
        if text is None or text == '':
            warning = 'Empty text passed to _handle_msg.'
            log.warn(warning)
            return None, None, warning
#        if not text.startswith('MIME-Version: 1.0'):
#            raise Ssm2Exception('Not a valid message.')

        # encrypted - this could be nicer
        if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
            try:
                text = crypto.decrypt(text, self._cert, self._key)
            except crypto.CryptoException, e:
                error = 'Failed to decrypt message: %s' % e
                log.error(error)
                return None, None, error

        # always signed
        try:
            message, signer = crypto.verify(text, self._capath, self._check_crls)
        except crypto.CryptoException, e:
            error = 'Failed to verify message: %s' % e
            log.error(error)
            return None, None, error
Пример #35
0
class Ssm2(object):
    '''
    Minimal SSM implementation.
    '''
    # Schema for the dirq message queue.
    QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
    REJECT_SCHEMA = {
        'body': 'string',
        'signer': 'string?',
        'empaid': 'string?',
        'error': 'string'
    }
    CONNECTION_TIMEOUT = 10

    def __init__(self,
                 hosts_and_ports,
                 qpath,
                 cert,
                 key,
                 dest=None,
                 listen=None,
                 capath=None,
                 check_crls=False,
                 use_ssl=False,
                 username=None,
                 password=None,
                 enc_cert=None,
                 verify_enc_cert=True,
                 pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')
        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s' % enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception(
                    'Specified certificate file does not exist: %s.' %
                    self._enc_cert)
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath,
                                               self._check_crls):
                    raise Ssm2Exception(
                        'Failed to verify server certificate %s against CA path %s.'
                        % (self._enc_cert, self._capath))

    def set_dns(self, dn_list):
        '''
        Set the list of DNs which are allowed to sign incoming messages.
        '''
        self._valid_dns = dn_list

    ##########################################################################
    # Methods called by stomppy
    ##########################################################################

    def on_send(self, headers, unused_body):
        '''
        Called by stomppy when a message is sent.
        '''
        log.debug('Sent message: ' + headers['empa-id'])

    def on_message(self, headers, body):
        '''
        Called by stomppy when a message is received.
        
        Handle the message according to its content and headers.
        '''
        try:
            empaid = headers['empa-id']
            if empaid == 'ping':  # ignore ping message
                log.info('Received ping message.')
                return
        except KeyError:
            empaid = 'noid'

        log.info('Received message: ' + empaid)
        raw_msg, signer = self._handle_msg(body)

        try:
            if raw_msg is None:  # the message has been rejected
                log.warn('Message rejected.')
                if signer is None:  # crypto failed
                    err_msg = 'Could not extract message.'
                    log.warn(err_msg)
                    signer = 'Not available.'
                else:  # crypto ok but signer not verified
                    err_msg = 'Signer not in valid DNs list.'
                    log.warn(err_msg)

                self._rejectq.add({
                    'body': body,
                    'signer': signer,
                    'empaid': empaid,
                    'error': err_msg
                })
            else:  # message verified ok
                self._inq.add({
                    'body': raw_msg,
                    'signer': signer,
                    'empaid': headers['empa-id']
                })
        except OSError, e:
            log.error('Failed to read or write file: %s' % e)
Пример #36
0
Файл: ssm2.py Проект: apel/ssm
    def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None,
                 capath=None, check_crls=False, use_ssl=False, username=None, password=None,
                 enc_cert=None, verify_enc_cert=True, pidfile=None):
        '''
        Creates an SSM2 object.  If a listen value is supplied,
        this SSM2 will be a receiver.
        '''
        self._conn = None
        self._last_msg = None

        self._brokers = hosts_and_ports
        self._cert = cert
        self._key = key
        self._enc_cert = enc_cert
        self._capath = capath
        self._check_crls = check_crls
        self._user = username
        self._pwd = password
        self._use_ssl = use_ssl
        # use pwd auth if we're supplied both user and pwd
        self._use_pwd = username is not None and password is not None
        self.connected = False

        self._listen = listen
        self._dest = dest

        self._valid_dns = []
        self._pidfile = pidfile

        # create the filesystem queues for accepted and rejected messages
        if dest is not None and listen is None:
            self._outq = QueueSimple(qpath)
        elif listen is not None:
            inqpath = os.path.join(qpath, 'incoming')
            rejectqpath = os.path.join(qpath, 'reject')
            self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
            self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
        else:
            raise Ssm2Exception('SSM must be either producer or consumer.')
        # check that the cert and key match
        if not crypto.check_cert_key(self._cert, self._key):
            raise Ssm2Exception('Cert and key don\'t match.')

        # Check that the certificate has not expired.
        if not crypto.verify_cert_date(self._cert):
            raise Ssm2Exception('Certificate %s has expired.' % self._cert)

        # check the server certificate provided
        if enc_cert is not None:
            log.info('Messages will be encrypted using %s', enc_cert)
            if not os.path.isfile(self._enc_cert):
                raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
            # Check that the encyption certificate has not expired.
            if not crypto.verify_cert_date(enc_cert):
                raise Ssm2Exception(
                    'Encryption certificate %s has expired. Please obtain the '
                    'new one from the final server receiving your messages.' %
                    enc_cert
                )
            if verify_enc_cert:
                if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
                    raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.'
                                        % (self._enc_cert, self._capath))

        # If the overall SSM log level is info, we want to only
        # see log entries from stomp.py at the warning level and above.
        if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
            logging.getLogger("stomp.py").setLevel(logging.WARNING)
        # If the overall SSM log level is debug, we want to only
        # see log entries from stomp.py at the info level and above.
        elif logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
            logging.getLogger("stomp.py").setLevel(logging.INFO)
Пример #37
0
import sys
import tempfile

# total number of elements
COUNT = 9
# queue head directory
path = tempfile.mkdtemp()
# max elements per elements directory
maxelts = 3
# element's schema
schema = {'body': 'string', 'header': 'table?'}

# ========
# PRODUCER
print("*** PRODUCER")
dirq_p = Queue(path, maxelts=maxelts, schema=schema)

print("adding %d elements to the queue at %s" % (COUNT, path))
done = 1
while done <= COUNT:
    element = {}
    try:
        element['body'] = ('Élément %i \u263A\n' % done).decode("utf-8")
    except AttributeError:
        element['body'] = 'Élément %i \u263A\n' % done
    if done % 2:  # header only for odd sequential elements
        element['header'] = dict(os.environ)
    name = dirq_p.enqueue(element)
    # name = dirq_p.add(element) # same
    print("added %.2i: %s" % (done, name))
    done += 1
Пример #38
0
def add_msg_to_queue(settings, msg):
    queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
    queue.enqueue(msg)
Пример #39
0
# number of queues
QUEUES = 4

print("*** Setup & populate queues")
# generate paths
paths = []
for i in range(QUEUES):
    paths.append(working_dir + '/q-%i' % i)
COUNT = 5

print("creating %i initial queues. adding %i elements into each." % (QUEUES,
                                                                     COUNT))
queues = []
queue_num = 0
while queue_num < QUEUES:
    queue = Queue(paths[queue_num], maxelts=5, schema={'body': 'string'})
    print("adding %d elements to the queue %s" % (COUNT, paths[queue_num]))
    element = {}
    done = 0
    while not COUNT or done < COUNT:
        done += 1
        element['body'] = 'Queue %i. Element %i' % (queue_num, done)
        queue.add(element)
    queues.append(queue)
    queue_num += 1
print("done.")


print("*** Browse")
i = 2
queue_set = QueueSet(queues[0:i])
Пример #40
0
def _queue_item(item):
    settings = Settings()
    try:
        queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
    except OSError, error:
        raise JensMessagingError("Failed to create Queue object (%s)" % error)