class WhenConnecting(DingusTestCase(Stomp)):
def setup(self):
super(WhenConnecting, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.sock = self.stomp.sock
self.frame = self.stomp.frame
self.stomp.connected = True
def should_set_socket_opts(self):
assert stompy.stomp.socket.calls('socket', DontCare, DontCare)
def should_connect(self):
self.stomp.connect()
assert self.frame.calls('connect', self.sock)
assert self.sock.calls('connect', (self.host, self.port))
def should_disconnect(self):
self.stomp.disconnect()
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'DISCONNECT'
assert self.frame.calls('send_frame', send_args.as_string())
assert self.stomp.sock.calls('shutdown', 0)
def should_raise_not_connected_error(self):
self.stomp.disconnect()
class WhenConsumingMessages(DingusTestCase(Stomp)):
def setup(self):
super(WhenConsumingMessages, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'destination': '/queue/nose_test',
'ack': 'client'}
def should_subscribe(self):
self.stomp.subscribe(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert self.stomp.subscribed
assert built_frame_args['command'] == 'SUBSCRIBE'
assert self.frame.calls('send_frame', send_args.as_string())
def should_poll(self):
this_frame = self.stomp.poll()
assert self.stomp.frame.calls('get_message', nb=True)
def should_receive_and_ack(self):
this_frame = self.stomp.receive_frame()
assert self.stomp.frame.calls('get_message')
self.stomp.ack(this_frame)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'ACK'
assert self.frame.calls('send_frame', send_args.as_string())
def should_register_callback(self):
callback_func = lambda message: 'pass'
this_frame = self.stomp.receive_frame(callback=callback_func)
assert self.stomp.frame.calls('get_message')
assert isinstance(self.stomp._callback, type(callback_func))
def should_unsubscribe(self):
self.stomp.unsubscribe({'destination': '/queue/nose-test'})
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'UNSUBSCRIBE'
assert self.frame.calls('send_frame', send_args.as_string())
assert not self.stomp.subscribed
def should_unsub_via_disco(self):
self.stomp._subscribed_to["/queue/nose-test"] = True
self.stomp.disconnect()
assert not self.stomp.subscribed
class Stomp(object):
conn = None
def __init__(self):
if not config.no_effect:
self.conn = DistStomp(config.stomp.server, config.stomp.port)
self.conn.connect()
def __del__(self):
if self.conn:
self.conn.disconnect()
# Let the STOMP library catch up
time.sleep(1)
def send(self, queue_key, body):
if config.no_effect:
log.info("not queueing message. " + json.dumps(body))
return
self.conn.send(self.create_message(queue_key, body))
def create_message(self, queue_key, body):
msg = {
'destination': config.stomp.queues[queue_key],
'persistent': 'true',
}
msg.update(Stomp.source_meta())
if 'gateway' in body and 'gateway_txn_id' in body:
msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'], id=body['gateway_txn_id'])
msg.update({'body': json.dumps(body)})
return msg
@staticmethod
def source_meta():
return {
'source_name': os.path.basename(sys.argv[0]),
# FIXME: the controlling script should pass its own source_type
'source_type': 'audit',
'source_run_id': os.getpid(),
'source_version': process.version_stamp.source_revision(),
'source_enqueued_time': time.time(),
'source_host': socket.gethostname(),
}
def should_set_disconnected_even_when_nc(self):
mystomp = Stomp('localhost', 99999)
mystomp.disconnect()
assert not mystomp.connected
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile, auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'), _config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(
awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
_civiDB = MySQL.connect(
_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
def handle_message(msg):
body = msg.get_body()
print("got this message: %s\n" % body)
sys.stdout.flush()
# spawn another script to handle notifications of new packages:
script_path = os.path.dirname(os.path.realpath(__file__))
arg = base64.b64encode(body)
cmd = "ruby %s/new_package_notifier.rb %s" % (script_path, arg)
result_code = subprocess.call(cmd, shell=True)
this_frame = stomp.send({'destination': "/topic/buildjobs",
'body': body,
'persistent': 'true'})
print("Receipt: %s" % this_frame.headers.get('receipt-id'))
sys.stdout.flush()
return True
while (True):
try:
m = q.read()
if (m != None):
result = handle_message(m)
if (result):
q.delete_message(m)
time.sleep(5)
except KeyboardInterrupt:
stomp.disconnect()
break
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile,
auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(
int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(
int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'),
_config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey'))
# === Connection to MySQL ===
_civiDB = MySQL.connect(_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema'))
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" %
(startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {'Success': 0, 'Pending': 0, 'Failed': 0, 'Ignored': 0}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" %
(AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(),
endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
def main():
# === Extract options ===
parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_usage()
exit()
startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Get the configuration options ===
config = SafeConfigParser()
fileList = ['./amazon-config.cfg']
if options.configFile is not None:
fileList.append(options.configFile)
config.read(fileList)
# === Open up ze STOMP ===
sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
sc.connect()
# === Connection to Amazon ===
aws = Amazon(
awsEndpoint = config.get('AwsConfig', 'endpoint'),
awsAccessKey = config.get('AwsConfig', 'accessKey'),
awsSecret = config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
dbcon = MySQL.connect(
config.get('MySQL', 'host'),
config.get('MySQL', 'user'),
config.get('MySQL', 'password'),
config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
historyEnd = endTime
if options.historyFile and os.path.exists(options.historyFile):
hfile = open(options.historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
sfile = None
if options.logFile:
sfile = open(options.logFile, 'a')
sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
txn = json.loads(txn)
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
hfile.close()
# --- Obtain AWS history ---
print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Prepare the history file for write ---
if options.historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(options.historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics ---
print("%d new AWS messages" % txncount)
print(" Additionally %d messages were processed from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
dbcon.close()
sc.disconnect()
if hfile:
hfile.close()
if sfile:
sfile.close()
time.sleep(1) # Let the STOMP library catch up