class WhenConnecting(DingusTestCase(Stomp)):
def setup(self):
super(WhenConnecting, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.sock = self.stomp.sock
self.frame = self.stomp.frame
self.stomp.connected = True
def should_set_socket_opts(self):
assert stompy.stomp.socket.calls('socket', DontCare, DontCare)
def should_connect(self):
self.stomp.connect()
assert self.frame.calls('connect', self.sock)
assert self.sock.calls('connect', (self.host, self.port))
def should_disconnect(self):
self.stomp.disconnect()
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'DISCONNECT'
assert self.frame.calls('send_frame', send_args.as_string())
assert self.stomp.sock.calls('shutdown', 0)
def should_raise_not_connected_error(self):
self.stomp.disconnect()
def setup_stomp():
global stomp
try:
stomp = Stomp("broker.bioconductor.org", 61613)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
class Stomp(object):
conn = None
def __init__(self):
if not config.no_effect:
self.conn = DistStomp(config.stomp.server, config.stomp.port)
self.conn.connect()
def __del__(self):
if self.conn:
self.conn.disconnect()
# Let the STOMP library catch up
time.sleep(1)
def send(self, queue_key, body):
if config.no_effect:
log.info("not queueing message. " + json.dumps(body))
return
self.conn.send(self.create_message(queue_key, body))
def create_message(self, queue_key, body):
msg = {
'destination': config.stomp.queues[queue_key],
'persistent': 'true',
}
msg.update(Stomp.source_meta())
if 'gateway' in body and 'gateway_txn_id' in body:
msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'], id=body['gateway_txn_id'])
msg.update({'body': json.dumps(body)})
return msg
@staticmethod
def source_meta():
return {
'source_name': os.path.basename(sys.argv[0]),
# FIXME: the controlling script should pass its own source_type
'source_type': 'audit',
'source_run_id': os.getpid(),
'source_version': process.version_stamp.source_revision(),
'source_enqueued_time': time.time(),
'source_host': socket.gethostname(),
}
def getOldStompConnection():
try:
log.debug("Attempting to open connection to ActiveMQ at '%s:%s'.",
stompHost,stompPort)
# Connect using the old model
stompClient = Stomp(stompHost, stompPort)
if (CONFIG_ENVIRONMENT == "production"):
log.debug("Not attempting authentication")
stompClient.connect()
else:
log.debug("Attempting authentication with user: '******'.", ACTIVEMQ_USER)
stompClient.connect(username=ACTIVEMQ_USER, password=ACTIVEMQ_PASS)
log.debug("Stomp connection established.")
except:
log.error("Cannot connect to Stomp at '%s:%s'.", stompHost, stompPort)
raise
return stompClient
import time
import tempfile
import os
import subprocess
import platform
import ConfigParser
import requests
import cookielib
import threading
import urllib
from stompy import Stomp
import mechanize
try:
stomp = Stomp("broker.bioconductor.org", 61613)
stomp.connect()
except:
print ("Cannot connect")
raise
stomp.subscribe({"destination": "/topic/builderevents", "ack": "client"})
global tracker_base_url
global build_counter
build_counter = {}
# FIXME - get this from canonical source (config.yaml)
hosts = ["zin2", "morelia", "moscato2"]
def handle_builder_event(obj):
global build_counter
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile, auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'), _config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(
awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
_civiDB = MySQL.connect(
_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile,
auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(
int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(
int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'),
_config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey'))
# === Connection to MySQL ===
_civiDB = MySQL.connect(_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema'))
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" %
(startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {'Success': 0, 'Pending': 0, 'Failed': 0, 'Ignored': 0}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" %
(AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(),
endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
help='produce or consume NUMBER messages')
options, args = parser.parse_args()
if not options.host:
print("Host name is required!")
parser.print_help()
sys.exit(1)
if not options.port:
print("Port is required!")
parser.print_help()
sys.exit(1)
if not options.queue:
print("Queue name is required!")
parser.print_help()
sys.exit(1)
try:
stomp = Stomp(options.host, options.port)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
if options.produce:
produce(options.queue, options.number)
elif options.consume:
consume(options.queue, options.number, options.callback)
def main():
# === Extract options ===
parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_usage()
exit()
startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Get the configuration options ===
config = SafeConfigParser()
fileList = ['./amazon-config.cfg']
if options.configFile is not None:
fileList.append(options.configFile)
config.read(fileList)
# === Open up ze STOMP ===
sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
sc.connect()
# === Connection to Amazon ===
aws = Amazon(
awsEndpoint = config.get('AwsConfig', 'endpoint'),
awsAccessKey = config.get('AwsConfig', 'accessKey'),
awsSecret = config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
dbcon = MySQL.connect(
config.get('MySQL', 'host'),
config.get('MySQL', 'user'),
config.get('MySQL', 'password'),
config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
historyEnd = endTime
if options.historyFile and os.path.exists(options.historyFile):
hfile = open(options.historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
sfile = None
if options.logFile:
sfile = open(options.logFile, 'a')
sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
txn = json.loads(txn)
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
hfile.close()
# --- Obtain AWS history ---
print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Prepare the history file for write ---
if options.historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(options.historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics ---
print("%d new AWS messages" % txncount)
print(" Additionally %d messages were processed from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
dbcon.close()
sc.disconnect()
if hfile:
hfile.close()
if sfile:
sfile.close()
time.sleep(1) # Let the STOMP library catch up