예제 #1
0
class WhenConnecting(DingusTestCase(Stomp)):

    def setup(self):
        super(WhenConnecting, self).setup()
        self.host = 'localhost'
        self.port = 61613
        self.stomp = Stomp(self.host, self.port)
        self.sock = self.stomp.sock
        self.frame = self.stomp.frame
        self.stomp.connected = True

    def should_set_socket_opts(self):
        assert stompy.stomp.socket.calls('socket', DontCare, DontCare)

    def should_connect(self):
        self.stomp.connect()
        assert self.frame.calls('connect', self.sock)
        assert self.sock.calls('connect', (self.host, self.port))

    def should_disconnect(self):
        self.stomp.disconnect()
        built_frame = self.frame.calls('build_frame', DontCare).one()
        built_frame_args = built_frame.args[0]
        send_args = built_frame[3]

        assert built_frame_args['command'] == 'DISCONNECT'
        assert self.frame.calls('send_frame', send_args.as_string())
        assert self.stomp.sock.calls('shutdown', 0)

    def should_raise_not_connected_error(self):
        self.stomp.disconnect()
예제 #2
0
class WhenConsumingMessages(DingusTestCase(Stomp)):

    def setup(self):
        super(WhenConsumingMessages, self).setup()
        self.host = 'localhost'
        self.port = 61613
        self.stomp = Stomp(self.host, self.port)
        self.frame = self.stomp.frame
        self.sock = self.stomp.sock
        self.stomp.connected = True
        self.headers = {'destination': '/queue/nose_test',
                        'ack': 'client'}

    def should_subscribe(self):
        self.stomp.subscribe(self.headers)
        built_frame = self.frame.calls('build_frame', DontCare).one()
        built_frame_args = built_frame.args[0]
        send_args = built_frame[3]

        assert self.stomp.subscribed
        assert built_frame_args['command'] == 'SUBSCRIBE'
        assert self.frame.calls('send_frame', send_args.as_string())

    def should_poll(self):
        this_frame = self.stomp.poll()
        assert self.stomp.frame.calls('get_message', nb=True)

    def should_receive_and_ack(self):
        this_frame = self.stomp.receive_frame()
        assert self.stomp.frame.calls('get_message')

        self.stomp.ack(this_frame)
        built_frame = self.frame.calls('build_frame', DontCare).one()
        built_frame_args = built_frame.args[0]
        send_args = built_frame[3]

        assert built_frame_args['command'] == 'ACK'
        assert self.frame.calls('send_frame', send_args.as_string())

    def should_register_callback(self):
        callback_func = lambda message: 'pass'
        this_frame = self.stomp.receive_frame(callback=callback_func)
        assert self.stomp.frame.calls('get_message')
        assert isinstance(self.stomp._callback, type(callback_func))

    def should_unsubscribe(self):
        self.stomp.unsubscribe({'destination': '/queue/nose-test'})
        built_frame = self.frame.calls('build_frame', DontCare).one()
        built_frame_args = built_frame.args[0]
        send_args = built_frame[3]

        assert built_frame_args['command'] == 'UNSUBSCRIBE'
        assert self.frame.calls('send_frame', send_args.as_string())
        assert not self.stomp.subscribed

    def should_unsub_via_disco(self):
        self.stomp._subscribed_to["/queue/nose-test"] = True
        self.stomp.disconnect()
        assert not self.stomp.subscribed
class Stomp(object):
    conn = None

    def __init__(self):
        if not config.no_effect:
            self.conn = DistStomp(config.stomp.server, config.stomp.port)
            self.conn.connect()

    def __del__(self):
        if self.conn:
            self.conn.disconnect()

            # Let the STOMP library catch up
            time.sleep(1)

    def send(self, queue_key, body):
        if config.no_effect:
            log.info("not queueing message. " + json.dumps(body))
            return

        self.conn.send(self.create_message(queue_key, body))

    def create_message(self, queue_key, body):
        msg = {
            'destination': config.stomp.queues[queue_key],
            'persistent': 'true',
        }
        msg.update(Stomp.source_meta())

        if 'gateway' in body and 'gateway_txn_id' in body:
            msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'], id=body['gateway_txn_id'])

        msg.update({'body': json.dumps(body)})

        return msg

    @staticmethod
    def source_meta():
        return {
            'source_name': os.path.basename(sys.argv[0]),
            # FIXME: the controlling script should pass its own source_type
            'source_type': 'audit',
            'source_run_id': os.getpid(),
            'source_version': process.version_stamp.source_revision(),
            'source_enqueued_time': time.time(),
            'source_host': socket.gethostname(),
        }
예제 #4
0
 def should_set_disconnected_even_when_nc(self):
     mystomp = Stomp('localhost', 99999)
     mystomp.disconnect()
     assert not mystomp.connected
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile, auditPayments, auditRefunds):
    global _config
    global _civiDB
    global _awsLink
    global _stompLink
    global _logFile

    startTime = datetime.fromtimestamp(int(time.time()) - int(secondsToAudit), pytz.utc)
    endTime = datetime.fromtimestamp(int(time.time()) - int(gracePeriod), pytz.utc)
    print("AWS audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))

    # === Initialize the configuration file ===
    localdir = os.path.dirname(os.path.abspath(__file__))
    _config = SafeConfigParser()
    fileList = ["%s/amazon-audit.cfg" % localdir]
    if configFile is not None:
        fileList.append(configFile)
    _config.read(fileList)

    # === Open up ze STOMP ===
    _stompLink = DistStomp(_config.get('Stomp', 'server'), _config.getint('Stomp', 'port'))
    _stompLink.connect()
    
    # === Connection to Amazon ===
    _awsLink = Amazon(
        awsEndpoint=_config.get('AwsConfig', 'endpoint'),
        awsAccessKey=_config.get('AwsConfig', 'accessKey'),
        awsSecret=_config.get('AwsConfig', 'secretKey')
    )

    # === Connection to MySQL ===
    _civiDB = MySQL.connect(
        _config.get('MySQL', 'host'),
        _config.get('MySQL', 'user'),
        _config.get('MySQL', 'password'),
        _config.get('MySQL', 'schema')
    )

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    if historyFile and os.path.exists(historyFile):
        hfile = open(historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    if logFile:
        _logFile = open(logFile, 'a')
        _logFile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
         startTime = endTime
    
    # === Main Application ===
    awsTransactions = []

    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {
        'Success': 0,
        'Pending': 0,
        'Failed': 0,
        'Ignored': 0
    }
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            awsTransactions.append(json.loads(txn))
        hfile.close()

    # --- Obtain AWS history ---
    if auditPayments:
        print("Obtaining AWS payment transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
        print("Obtained %d transactions" % len(awsTransactions))

    if auditRefunds:
        print("Obtaining AWS refund transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Refund')
        print("Obtained %d transactions" % len(awsTransactions))
    
    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = dispatchTransaction(txn, auditPayments, auditRefunds)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Write the history file ---
    if historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics
    print("Processed %d AWS messages" % txncount)
    print(" ... of which %d messages were from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    _civiDB.close()
    _stompLink.disconnect()

    if hfile:
        hfile.close()
    if _logFile:
        _logFile.close()

    time.sleep(1)   # Let the STOMP library catch up
예제 #6
0
def handle_message(msg):
    body = msg.get_body()
    print("got this message: %s\n" % body)
    sys.stdout.flush()
    # spawn another script to handle notifications of new packages:
    script_path = os.path.dirname(os.path.realpath(__file__))
    arg = base64.b64encode(body)
    cmd = "ruby %s/new_package_notifier.rb %s" % (script_path, arg)
    result_code = subprocess.call(cmd, shell=True)
    this_frame = stomp.send({'destination': "/topic/buildjobs",
      'body': body,
      'persistent': 'true'})
    print("Receipt: %s" % this_frame.headers.get('receipt-id'))
    sys.stdout.flush()
    return True

while (True):
    try:
        m = q.read()
        if (m != None):
            result = handle_message(m)
            if (result):
                q.delete_message(m)
        time.sleep(5)
    except KeyboardInterrupt:
        stomp.disconnect()
        break


def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile,
         auditPayments, auditRefunds):
    global _config
    global _civiDB
    global _awsLink
    global _stompLink
    global _logFile

    startTime = datetime.fromtimestamp(
        int(time.time()) - int(secondsToAudit), pytz.utc)
    endTime = datetime.fromtimestamp(
        int(time.time()) - int(gracePeriod), pytz.utc)
    print("AWS audit requested from %s to %s" %
          (startTime.isoformat(), endTime.isoformat()))

    # === Initialize the configuration file ===
    localdir = os.path.dirname(os.path.abspath(__file__))
    _config = SafeConfigParser()
    fileList = ["%s/amazon-audit.cfg" % localdir]
    if configFile is not None:
        fileList.append(configFile)
    _config.read(fileList)

    # === Open up ze STOMP ===
    _stompLink = DistStomp(_config.get('Stomp', 'server'),
                           _config.getint('Stomp', 'port'))
    _stompLink.connect()

    # === Connection to Amazon ===
    _awsLink = Amazon(awsEndpoint=_config.get('AwsConfig', 'endpoint'),
                      awsAccessKey=_config.get('AwsConfig', 'accessKey'),
                      awsSecret=_config.get('AwsConfig', 'secretKey'))

    # === Connection to MySQL ===
    _civiDB = MySQL.connect(_config.get('MySQL', 'host'),
                            _config.get('MySQL', 'user'),
                            _config.get('MySQL', 'password'),
                            _config.get('MySQL', 'schema'))

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    if historyFile and os.path.exists(historyFile):
        hfile = open(historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" %
                  (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    if logFile:
        _logFile = open(logFile, 'a')
        _logFile.write("!!! Starting run for dates %s -> %s\n" %
                       (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
        startTime = endTime

    # === Main Application ===
    awsTransactions = []

    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {'Success': 0, 'Pending': 0, 'Failed': 0, 'Ignored': 0}
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            awsTransactions.append(json.loads(txn))
        hfile.close()

    # --- Obtain AWS history ---
    if auditPayments:
        print("Obtaining AWS payment transactions for the period %s -> %s" %
              (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime,
                                                       endDate=endTime,
                                                       fpsOperation='Pay')
        print("Obtained %d transactions" % len(awsTransactions))

    if auditRefunds:
        print("Obtaining AWS refund transactions for the period %s -> %s" %
              (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime,
                                                       endDate=endTime,
                                                       fpsOperation='Refund')
        print("Obtained %d transactions" % len(awsTransactions))

    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = dispatchTransaction(txn, auditPayments, auditRefunds)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Write the history file ---
    if historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" %
                    (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(),
                     endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics
    print("Processed %d AWS messages" % txncount)
    print(" ... of which %d messages were from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    _civiDB.close()
    _stompLink.disconnect()

    if hfile:
        hfile.close()
    if _logFile:
        _logFile.close()

    time.sleep(1)  # Let the STOMP library catch up
def main():
    # === Extract options ===
    parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
    parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
    parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
    parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
    parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
    (options, args) = parser.parse_args()

    if len(args) != 1:
        parser.print_usage()
        exit()

    startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
    endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
    print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))

    # === Get the configuration options ===
    config = SafeConfigParser()
    fileList = ['./amazon-config.cfg']
    if options.configFile is not None:
        fileList.append(options.configFile)
    config.read(fileList)

    # === Open up ze STOMP ===
    sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
    sc.connect()

    # === Connection to Amazon ===
    aws = Amazon(
        awsEndpoint = config.get('AwsConfig', 'endpoint'),
        awsAccessKey = config.get('AwsConfig', 'accessKey'),
        awsSecret = config.get('AwsConfig', 'secretKey')
    )

    # === Connection to MySQL ===
    dbcon = MySQL.connect(
        config.get('MySQL', 'host'),
        config.get('MySQL', 'user'),
        config.get('MySQL', 'password'),
        config.get('MySQL', 'schema')
    )

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    historyEnd = endTime
    if options.historyFile and os.path.exists(options.historyFile):
        hfile = open(options.historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    sfile = None
    if options.logFile:
        sfile = open(options.logFile, 'a')
        sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
         startTime = endTime

    # === Main Application ===
    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {
        'Success': 0,
        'Pending': 0,
        'Failed': 0,
        'Ignored': 0
    }
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            txn = json.loads(txn)
            result = processTransaction(txn, dbcon, aws, sc, sfile, config)
            historyStats[result] += 1
            if result == 'Pending':
                historyList.append(txn)
        hfile.close()

    # --- Obtain AWS history ---
    print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
    awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
    print("Obtained %d transactions" % len(awsTransactions))

    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = processTransaction(txn, dbcon, aws, sc, sfile, config)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Prepare the history file for write ---
    if options.historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(options.historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics ---
    print("%d new AWS messages" % txncount)
    print(" Additionally %d messages were processed from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    dbcon.close()
    sc.disconnect()

    if hfile:
        hfile.close()
    if sfile:
        sfile.close()

    time.sleep(1)   # Let the STOMP library catch up