Exemple #1
0
class WhenConnecting(DingusTestCase(Stomp)):

    def setup(self):
        super(WhenConnecting, self).setup()
        self.host = 'localhost'
        self.port = 61613
        self.stomp = Stomp(self.host, self.port)
        self.sock = self.stomp.sock
        self.frame = self.stomp.frame
        self.stomp.connected = True

    def should_set_socket_opts(self):
        assert stompy.stomp.socket.calls('socket', DontCare, DontCare)

    def should_connect(self):
        self.stomp.connect()
        assert self.frame.calls('connect', self.sock)
        assert self.sock.calls('connect', (self.host, self.port))

    def should_disconnect(self):
        self.stomp.disconnect()
        built_frame = self.frame.calls('build_frame', DontCare).one()
        built_frame_args = built_frame.args[0]
        send_args = built_frame[3]

        assert built_frame_args['command'] == 'DISCONNECT'
        assert self.frame.calls('send_frame', send_args.as_string())
        assert self.stomp.sock.calls('shutdown', 0)

    def should_raise_not_connected_error(self):
        self.stomp.disconnect()
Exemple #2
0
def setup_stomp():
    global stomp
    try:
        stomp = Stomp("broker.bioconductor.org", 61613)
        # optional connect keyword args "username" and "password" like so:
        # stomp.connect(username="******", password="******")
        stomp.connect()
    except:
        print("Cannot connect")
        raise
class Stomp(object):
    conn = None

    def __init__(self):
        if not config.no_effect:
            self.conn = DistStomp(config.stomp.server, config.stomp.port)
            self.conn.connect()

    def __del__(self):
        if self.conn:
            self.conn.disconnect()

            # Let the STOMP library catch up
            time.sleep(1)

    def send(self, queue_key, body):
        if config.no_effect:
            log.info("not queueing message. " + json.dumps(body))
            return

        self.conn.send(self.create_message(queue_key, body))

    def create_message(self, queue_key, body):
        msg = {
            'destination': config.stomp.queues[queue_key],
            'persistent': 'true',
        }
        msg.update(Stomp.source_meta())

        if 'gateway' in body and 'gateway_txn_id' in body:
            msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'], id=body['gateway_txn_id'])

        msg.update({'body': json.dumps(body)})

        return msg

    @staticmethod
    def source_meta():
        return {
            'source_name': os.path.basename(sys.argv[0]),
            # FIXME: the controlling script should pass its own source_type
            'source_type': 'audit',
            'source_run_id': os.getpid(),
            'source_version': process.version_stamp.source_revision(),
            'source_enqueued_time': time.time(),
            'source_host': socket.gethostname(),
        }
def getOldStompConnection():
    try:
        log.debug("Attempting to open connection to ActiveMQ at '%s:%s'.",
            stompHost,stompPort)
        # Connect using the old model
        stompClient = Stomp(stompHost, stompPort)
        if (CONFIG_ENVIRONMENT == "production"):
            log.debug("Not attempting authentication")
            stompClient.connect()
        else:
            log.debug("Attempting authentication with user: '******'.", ACTIVEMQ_USER)
            stompClient.connect(username=ACTIVEMQ_USER, password=ACTIVEMQ_PASS)
        log.debug("Stomp connection established.")
    except:
        log.error("Cannot connect to Stomp at '%s:%s'.", stompHost, stompPort)
        raise
    
    return stompClient
import time
import tempfile
import os
import subprocess
import platform
import ConfigParser
import requests
import cookielib
import threading
import urllib
from stompy import Stomp
import mechanize

try:
    stomp = Stomp("broker.bioconductor.org", 61613)
    stomp.connect()
except:
    print ("Cannot connect")
    raise

stomp.subscribe({"destination": "/topic/builderevents", "ack": "client"})

global tracker_base_url
global build_counter
build_counter = {}
# FIXME - get this from canonical source (config.yaml)
hosts = ["zin2", "morelia", "moscato2"]


def handle_builder_event(obj):
    global build_counter
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile, auditPayments, auditRefunds):
    global _config
    global _civiDB
    global _awsLink
    global _stompLink
    global _logFile

    startTime = datetime.fromtimestamp(int(time.time()) - int(secondsToAudit), pytz.utc)
    endTime = datetime.fromtimestamp(int(time.time()) - int(gracePeriod), pytz.utc)
    print("AWS audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))

    # === Initialize the configuration file ===
    localdir = os.path.dirname(os.path.abspath(__file__))
    _config = SafeConfigParser()
    fileList = ["%s/amazon-audit.cfg" % localdir]
    if configFile is not None:
        fileList.append(configFile)
    _config.read(fileList)

    # === Open up ze STOMP ===
    _stompLink = DistStomp(_config.get('Stomp', 'server'), _config.getint('Stomp', 'port'))
    _stompLink.connect()
    
    # === Connection to Amazon ===
    _awsLink = Amazon(
        awsEndpoint=_config.get('AwsConfig', 'endpoint'),
        awsAccessKey=_config.get('AwsConfig', 'accessKey'),
        awsSecret=_config.get('AwsConfig', 'secretKey')
    )

    # === Connection to MySQL ===
    _civiDB = MySQL.connect(
        _config.get('MySQL', 'host'),
        _config.get('MySQL', 'user'),
        _config.get('MySQL', 'password'),
        _config.get('MySQL', 'schema')
    )

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    if historyFile and os.path.exists(historyFile):
        hfile = open(historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    if logFile:
        _logFile = open(logFile, 'a')
        _logFile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
         startTime = endTime
    
    # === Main Application ===
    awsTransactions = []

    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {
        'Success': 0,
        'Pending': 0,
        'Failed': 0,
        'Ignored': 0
    }
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            awsTransactions.append(json.loads(txn))
        hfile.close()

    # --- Obtain AWS history ---
    if auditPayments:
        print("Obtaining AWS payment transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
        print("Obtained %d transactions" % len(awsTransactions))

    if auditRefunds:
        print("Obtaining AWS refund transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Refund')
        print("Obtained %d transactions" % len(awsTransactions))
    
    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = dispatchTransaction(txn, auditPayments, auditRefunds)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Write the history file ---
    if historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics
    print("Processed %d AWS messages" % txncount)
    print(" ... of which %d messages were from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    _civiDB.close()
    _stompLink.disconnect()

    if hfile:
        hfile.close()
    if _logFile:
        _logFile.close()

    time.sleep(1)   # Let the STOMP library catch up
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile,
         auditPayments, auditRefunds):
    global _config
    global _civiDB
    global _awsLink
    global _stompLink
    global _logFile

    startTime = datetime.fromtimestamp(
        int(time.time()) - int(secondsToAudit), pytz.utc)
    endTime = datetime.fromtimestamp(
        int(time.time()) - int(gracePeriod), pytz.utc)
    print("AWS audit requested from %s to %s" %
          (startTime.isoformat(), endTime.isoformat()))

    # === Initialize the configuration file ===
    localdir = os.path.dirname(os.path.abspath(__file__))
    _config = SafeConfigParser()
    fileList = ["%s/amazon-audit.cfg" % localdir]
    if configFile is not None:
        fileList.append(configFile)
    _config.read(fileList)

    # === Open up ze STOMP ===
    _stompLink = DistStomp(_config.get('Stomp', 'server'),
                           _config.getint('Stomp', 'port'))
    _stompLink.connect()

    # === Connection to Amazon ===
    _awsLink = Amazon(awsEndpoint=_config.get('AwsConfig', 'endpoint'),
                      awsAccessKey=_config.get('AwsConfig', 'accessKey'),
                      awsSecret=_config.get('AwsConfig', 'secretKey'))

    # === Connection to MySQL ===
    _civiDB = MySQL.connect(_config.get('MySQL', 'host'),
                            _config.get('MySQL', 'user'),
                            _config.get('MySQL', 'password'),
                            _config.get('MySQL', 'schema'))

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    if historyFile and os.path.exists(historyFile):
        hfile = open(historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" %
                  (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    if logFile:
        _logFile = open(logFile, 'a')
        _logFile.write("!!! Starting run for dates %s -> %s\n" %
                       (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
        startTime = endTime

    # === Main Application ===
    awsTransactions = []

    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {'Success': 0, 'Pending': 0, 'Failed': 0, 'Ignored': 0}
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            awsTransactions.append(json.loads(txn))
        hfile.close()

    # --- Obtain AWS history ---
    if auditPayments:
        print("Obtaining AWS payment transactions for the period %s -> %s" %
              (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime,
                                                       endDate=endTime,
                                                       fpsOperation='Pay')
        print("Obtained %d transactions" % len(awsTransactions))

    if auditRefunds:
        print("Obtaining AWS refund transactions for the period %s -> %s" %
              (startTime.isoformat(), endTime.isoformat()))
        awsTransactions += _awsLink.getAccountActivity(startTime,
                                                       endDate=endTime,
                                                       fpsOperation='Refund')
        print("Obtained %d transactions" % len(awsTransactions))

    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = dispatchTransaction(txn, auditPayments, auditRefunds)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Write the history file ---
    if historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" %
                    (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(),
                     endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics
    print("Processed %d AWS messages" % txncount)
    print(" ... of which %d messages were from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    _civiDB.close()
    _stompLink.disconnect()

    if hfile:
        hfile.close()
    if _logFile:
        _logFile.close()

    time.sleep(1)  # Let the STOMP library catch up
Exemple #8
0
                      help='produce or consume NUMBER messages')

    options, args = parser.parse_args()

    if not options.host:
        print("Host name is required!")
        parser.print_help()
        sys.exit(1)
    if not options.port:
        print("Port is required!")
        parser.print_help()
        sys.exit(1)
    if not options.queue:
        print("Queue name is required!")
        parser.print_help()
        sys.exit(1)

    try:
        stomp = Stomp(options.host, options.port)
        # optional connect keyword args "username" and "password" like so:
        # stomp.connect(username="******", password="******")
        stomp.connect()
    except:
        print("Cannot connect")
        raise

    if options.produce:
        produce(options.queue, options.number)
    elif options.consume:
        consume(options.queue, options.number, options.callback)
def main():
    # === Extract options ===
    parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
    parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
    parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
    parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
    parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
    (options, args) = parser.parse_args()

    if len(args) != 1:
        parser.print_usage()
        exit()

    startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
    endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
    print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))

    # === Get the configuration options ===
    config = SafeConfigParser()
    fileList = ['./amazon-config.cfg']
    if options.configFile is not None:
        fileList.append(options.configFile)
    config.read(fileList)

    # === Open up ze STOMP ===
    sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
    sc.connect()

    # === Connection to Amazon ===
    aws = Amazon(
        awsEndpoint = config.get('AwsConfig', 'endpoint'),
        awsAccessKey = config.get('AwsConfig', 'accessKey'),
        awsSecret = config.get('AwsConfig', 'secretKey')
    )

    # === Connection to MySQL ===
    dbcon = MySQL.connect(
        config.get('MySQL', 'host'),
        config.get('MySQL', 'user'),
        config.get('MySQL', 'password'),
        config.get('MySQL', 'schema')
    )

    # === Open up the history and log files ===
    # If the history file exists, it will modify the start time of this script to be the end time of the
    # history file.
    hfile = None
    historyStart = startTime
    historyEnd = endTime
    if options.historyFile and os.path.exists(options.historyFile):
        hfile = open(options.historyFile, 'r')
        if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
            historyStart = dateutil.parser.parse(hfile.readline().strip())
            historyEnd = dateutil.parser.parse(hfile.readline().strip())
            startTime = historyEnd
            print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
    else:
        print('Not starting with a valid history file.')

    sfile = None
    if options.logFile:
        sfile = open(options.logFile, 'a')
        sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))

    # === Sanity checks ===
    if endTime < startTime:
         startTime = endTime

    # === Main Application ===
    # --- Process all previously pending transactions from the history file. If the transaction is still in some form
    #     of pending, add it back to the history list.
    historyCount = 0
    historyList = []
    historyStats = {
        'Success': 0,
        'Pending': 0,
        'Failed': 0,
        'Ignored': 0
    }
    if hfile:
        print("Processing history file")
        for txn in hfile:
            historyCount += 1
            txn = json.loads(txn)
            result = processTransaction(txn, dbcon, aws, sc, sfile, config)
            historyStats[result] += 1
            if result == 'Pending':
                historyList.append(txn)
        hfile.close()

    # --- Obtain AWS history ---
    print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
    awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
    print("Obtained %d transactions" % len(awsTransactions))

    # --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
    txncount = 0
    for txn in awsTransactions:
        txncount += 1
        result = processTransaction(txn, dbcon, aws, sc, sfile, config)
        historyStats[result] += 1
        if result == 'Pending':
            historyList.append(txn)

    print("\n--- Finished processing of messages. ---\n")

    # --- Prepare the history file for write ---
    if options.historyFile:
        print("Rewriting history file with %d transactions" % len(historyList))
        hfile = open(options.historyFile, 'w')
        hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
        for txn in historyList:
            hfile.write("%s\n" % json.dumps(txn))
        print("Flushing history file in preparation for main loop")
        hfile.flush()

    # --- Final statistics ---
    print("%d new AWS messages" % txncount)
    print(" Additionally %d messages were processed from history" % historyCount)
    print("This resulted in the following:")
    for entry in historyStats.items():
        print(" %s Messages: %d" % entry)

    # === Final Application Cleanup ===
    print("\nCleaning up.")
    dbcon.close()
    sc.disconnect()

    if hfile:
        hfile.close()
    if sfile:
        sfile.close()

    time.sleep(1)   # Let the STOMP library catch up