def should_raise_nc_when_sending(self):
mystomp = Stomp('localhost', 99999)
try:
mystomp.send({"body": "Vandelay Industries"})
except stompy.NotConnectedError, err:
assert True # Should raise not connected
return
def setup(self):
super(WhenConnecting, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.sock = self.stomp.sock
self.frame = self.stomp.frame
self.stomp.connected = True
def setup(self):
super(WhenUsingTransactions, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'transaction': 'nose_123'}
def setup_stomp():
global stomp
try:
stomp = Stomp("broker.bioconductor.org", 61613)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
def setup(self):
super(WhenProducingMessages, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'destination': '/queue/nose_test',
'body': 'test'}
def getOldStompConnection():
try:
log.debug("Attempting to open connection to ActiveMQ at '%s:%s'.",
stompHost,stompPort)
# Connect using the old model
stompClient = Stomp(stompHost, stompPort)
if (CONFIG_ENVIRONMENT == "production"):
log.debug("Not attempting authentication")
stompClient.connect()
else:
log.debug("Attempting authentication with user: '******'.", ACTIVEMQ_USER)
stompClient.connect(username=ACTIVEMQ_USER, password=ACTIVEMQ_PASS)
log.debug("Stomp connection established.")
except:
log.error("Cannot connect to Stomp at '%s:%s'.", stompHost, stompPort)
raise
return stompClient
class WhenProducingMessages(DingusTestCase(Stomp)):
def setup(self):
super(WhenProducingMessages, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'destination': '/queue/nose_test',
'body': 'test'}
def should_build_frame_and_send(self):
self.stomp.send(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'SEND'
assert self.frame.calls('send_frame', send_args.as_string())
class WhenConnecting(DingusTestCase(Stomp)):
def setup(self):
super(WhenConnecting, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.sock = self.stomp.sock
self.frame = self.stomp.frame
self.stomp.connected = True
def should_set_socket_opts(self):
assert stompy.stomp.socket.calls('socket', DontCare, DontCare)
def should_connect(self):
self.stomp.connect()
assert self.frame.calls('connect', self.sock)
assert self.sock.calls('connect', (self.host, self.port))
def should_disconnect(self):
self.stomp.disconnect()
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'DISCONNECT'
assert self.frame.calls('send_frame', send_args.as_string())
assert self.stomp.sock.calls('shutdown', 0)
def should_raise_not_connected_error(self):
self.stomp.disconnect()
class Stomp(object):
conn = None
def __init__(self):
if not config.no_effect:
self.conn = DistStomp(config.stomp.server, config.stomp.port)
self.conn.connect()
def __del__(self):
if self.conn:
self.conn.disconnect()
# Let the STOMP library catch up
time.sleep(1)
def send(self, queue_key, body):
if config.no_effect:
log.info("not queueing message. " + json.dumps(body))
return
self.conn.send(self.create_message(queue_key, body))
def create_message(self, queue_key, body):
msg = {
'destination': config.stomp.queues[queue_key],
'persistent': 'true',
}
msg.update(Stomp.source_meta())
if 'gateway' in body and 'gateway_txn_id' in body:
msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'], id=body['gateway_txn_id'])
msg.update({'body': json.dumps(body)})
return msg
@staticmethod
def source_meta():
return {
'source_name': os.path.basename(sys.argv[0]),
# FIXME: the controlling script should pass its own source_type
'source_type': 'audit',
'source_run_id': os.getpid(),
'source_version': process.version_stamp.source_revision(),
'source_enqueued_time': time.time(),
'source_host': socket.gethostname(),
}
class WhenUsingTransactions(DingusTestCase(Stomp)):
def setup(self):
super(WhenUsingTransactions, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'transaction': 'nose_123'}
def should_begin(self):
self.stomp.begin(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'BEGIN'
assert self.frame.calls('send_frame', send_args.as_string())
def should_commit(self):
self.stomp.commit(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'COMMIT'
assert self.frame.calls('send_frame', send_args.as_string())
def should_abort(self):
self.stomp.abort(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'ABORT'
assert self.frame.calls('send_frame', send_args.as_string())
def __init__(self):
if not config.no_effect:
self.conn = DistStomp(config.stomp.server, config.stomp.port)
self.conn.connect()
help='produce or consume NUMBER messages')
options, args = parser.parse_args()
if not options.host:
print("Host name is required!")
parser.print_help()
sys.exit(1)
if not options.port:
print("Port is required!")
parser.print_help()
sys.exit(1)
if not options.queue:
print("Queue name is required!")
parser.print_help()
sys.exit(1)
try:
stomp = Stomp(options.host, options.port)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
if options.produce:
produce(options.queue, options.number)
elif options.consume:
consume(options.queue, options.number, options.callback)
def should_fail_connect_with_timeout(self):
import socket
socket.setdefaulttimeout(.5)
self.stomp = Stomp('10.10.0.0', 99999)
self.failUnlessRaises(self.stomp.ConnectionTimeoutError,
self.stomp.connect)
def should_fail_connect(self):
self.stomp = Stomp('localhost', 99999)
self.failUnlessRaises(self.stomp.ConnectionError, self.stomp.connect)
def should_set_disconnected_even_when_nc(self):
mystomp = Stomp('localhost', 99999)
mystomp.disconnect()
assert not mystomp.connected
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile, auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'), _config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(
awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
_civiDB = MySQL.connect(
_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime, endDate=endTime, fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
def should_fail_to_send(self):
mystomp = Stomp('localhost', 99999)
self.failUnlessRaises(stompy.NotConnectedError, mystomp.send,
{"body": "f"})
def should_set_sub(self):
mystomp = Stomp('localhost', 99999)
mystomp._subscribed_to['/queue/nose_test'] = True
assert mystomp.subscribed is not None
class WhenConsumingMessages(DingusTestCase(Stomp)):
def setup(self):
super(WhenConsumingMessages, self).setup()
self.host = 'localhost'
self.port = 61613
self.stomp = Stomp(self.host, self.port)
self.frame = self.stomp.frame
self.sock = self.stomp.sock
self.stomp.connected = True
self.headers = {'destination': '/queue/nose_test',
'ack': 'client'}
def should_subscribe(self):
self.stomp.subscribe(self.headers)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert self.stomp.subscribed
assert built_frame_args['command'] == 'SUBSCRIBE'
assert self.frame.calls('send_frame', send_args.as_string())
def should_poll(self):
this_frame = self.stomp.poll()
assert self.stomp.frame.calls('get_message', nb=True)
def should_receive_and_ack(self):
this_frame = self.stomp.receive_frame()
assert self.stomp.frame.calls('get_message')
self.stomp.ack(this_frame)
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'ACK'
assert self.frame.calls('send_frame', send_args.as_string())
def should_register_callback(self):
callback_func = lambda message: 'pass'
this_frame = self.stomp.receive_frame(callback=callback_func)
assert self.stomp.frame.calls('get_message')
assert isinstance(self.stomp._callback, type(callback_func))
def should_unsubscribe(self):
self.stomp.unsubscribe({'destination': '/queue/nose-test'})
built_frame = self.frame.calls('build_frame', DontCare).one()
built_frame_args = built_frame.args[0]
send_args = built_frame[3]
assert built_frame_args['command'] == 'UNSUBSCRIBE'
assert self.frame.calls('send_frame', send_args.as_string())
assert not self.stomp.subscribed
def should_unsub_via_disco(self):
self.stomp._subscribed_to["/queue/nose-test"] = True
self.stomp.disconnect()
assert not self.stomp.subscribed
help='produce or consume NUMBER messages')
options, args = parser.parse_args()
if not options.host:
print("Host name is required!")
parser.print_help()
sys.exit(1)
if not options.port:
print("Port is required!")
parser.print_help()
sys.exit(1)
if not options.queue:
print("Queue name is required!")
parser.print_help()
sys.exit(1)
try:
stomp = Stomp(options.host, options.port)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
if options.produce:
produce(options.queue, options.number)
elif options.consume:
consume(options.queue, options.number, options.callback)
import json
import time
import tempfile
import os
import subprocess
import platform
import ConfigParser
import requests
import cookielib
import threading
import urllib
from stompy import Stomp
import mechanize
try:
stomp = Stomp("broker.bioconductor.org", 61613)
stomp.connect()
except:
print ("Cannot connect")
raise
stomp.subscribe({"destination": "/topic/builderevents", "ack": "client"})
global tracker_base_url
global build_counter
build_counter = {}
# FIXME - get this from canonical source (config.yaml)
hosts = ["zin2", "morelia", "moscato2"]
def handle_builder_event(obj):
obj['svn_url'] = url
obj['repository'] = 'scratch'
now = pacific.localize(now0)
timestamp1 = now.strftime("%Y%m%d%H%M%S")
timestamp2 = now.strftime("%a %b %d %Y %H:%M:%S")
timestamp2 = timestamp2 + " GMT-%s (%s)" % (offset, tzname)
obj['job_id'] = "%s_%s" % (pkgname_bare, timestamp1)
obj['time'] = timestamp2
obj['client_id'] = "single_package_builder_autobuild:%s:%s" % (issue_id, pkgname)
json = json.dumps(obj)
#print(json)
try:
stomp = Stomp("broker.bioconductor.org", 61613)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
this_frame = stomp.send({'destination': "/topic/buildjobs",
'body': json,
'persistent': 'true'})
print("Receipt: %s" % this_frame.headers.get('receipt-id'))
sys.stdout.flush()
segs.pop()
path = "/".join(segs)
sys.path.append(path)
os.environ['DJANGO_SETTINGS_MODULE'] = 'spb_history.settings'
# now you can do stuff like this:
#from spb_history.viewhistory.models import Package
#print Package.objects.count()
completed_jobs = []
from spb_history.viewhistory.models import Job
from spb_history.viewhistory.models import Package
from spb_history.viewhistory.models import Build
from spb_history.viewhistory.models import Message
try:
stomp = Stomp("broker.bioconductor.org", 61613)
# optional connect keyword args "username" and "password" like so:
# stomp.connect(username="******", password="******")
stomp.connect()
except:
print("Cannot connect")
raise
# do we want acks?
#stomp.subscribe({'destination': "/topic/buildjobs", 'ack': 'client'})
stomp.subscribe({'destination': "/topic/builderevents", 'ack': 'client'})
def get_build_obj(obj):
print("in get_build_obj(), obj['job_id']==%s, obj['builder_id']==%s\n"\
def main(secondsToAudit, configFile, gracePeriod, historyFile, logFile,
auditPayments, auditRefunds):
global _config
global _civiDB
global _awsLink
global _stompLink
global _logFile
startTime = datetime.fromtimestamp(
int(time.time()) - int(secondsToAudit), pytz.utc)
endTime = datetime.fromtimestamp(
int(time.time()) - int(gracePeriod), pytz.utc)
print("AWS audit requested from %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
# === Initialize the configuration file ===
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/amazon-audit.cfg" % localdir]
if configFile is not None:
fileList.append(configFile)
_config.read(fileList)
# === Open up ze STOMP ===
_stompLink = DistStomp(_config.get('Stomp', 'server'),
_config.getint('Stomp', 'port'))
_stompLink.connect()
# === Connection to Amazon ===
_awsLink = Amazon(awsEndpoint=_config.get('AwsConfig', 'endpoint'),
awsAccessKey=_config.get('AwsConfig', 'accessKey'),
awsSecret=_config.get('AwsConfig', 'secretKey'))
# === Connection to MySQL ===
_civiDB = MySQL.connect(_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
_config.get('MySQL', 'schema'))
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
if historyFile and os.path.exists(historyFile):
hfile = open(historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" %
(startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
if logFile:
_logFile = open(logFile, 'a')
_logFile.write("!!! Starting run for dates %s -> %s\n" %
(startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
awsTransactions = []
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {'Success': 0, 'Pending': 0, 'Failed': 0, 'Ignored': 0}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
awsTransactions.append(json.loads(txn))
hfile.close()
# --- Obtain AWS history ---
if auditPayments:
print("Obtaining AWS payment transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
if auditRefunds:
print("Obtaining AWS refund transactions for the period %s -> %s" %
(startTime.isoformat(), endTime.isoformat()))
awsTransactions += _awsLink.getAccountActivity(startTime,
endDate=endTime,
fpsOperation='Refund')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = dispatchTransaction(txn, auditPayments, auditRefunds)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Write the history file ---
if historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(historyFile, 'w')
hfile.write("%s\n%s\n%s\n" %
(AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(),
endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics
print("Processed %d AWS messages" % txncount)
print(" ... of which %d messages were from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
_civiDB.close()
_stompLink.disconnect()
if hfile:
hfile.close()
if _logFile:
_logFile.close()
time.sleep(1) # Let the STOMP library catch up
def main():
# === Extract options ===
parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_usage()
exit()
startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Get the configuration options ===
config = SafeConfigParser()
fileList = ['./amazon-config.cfg']
if options.configFile is not None:
fileList.append(options.configFile)
config.read(fileList)
# === Open up ze STOMP ===
sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
sc.connect()
# === Connection to Amazon ===
aws = Amazon(
awsEndpoint = config.get('AwsConfig', 'endpoint'),
awsAccessKey = config.get('AwsConfig', 'accessKey'),
awsSecret = config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
dbcon = MySQL.connect(
config.get('MySQL', 'host'),
config.get('MySQL', 'user'),
config.get('MySQL', 'password'),
config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
historyEnd = endTime
if options.historyFile and os.path.exists(options.historyFile):
hfile = open(options.historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
sfile = None
if options.logFile:
sfile = open(options.logFile, 'a')
sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
txn = json.loads(txn)
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
hfile.close()
# --- Obtain AWS history ---
print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Prepare the history file for write ---
if options.historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(options.historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics ---
print("%d new AWS messages" % txncount)
print(" Additionally %d messages were processed from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
dbcon.close()
sc.disconnect()
if hfile:
hfile.close()
if sfile:
sfile.close()
time.sleep(1) # Let the STOMP library catch up
import ConfigParser
from stompy import Stomp
print("Starting monitor...\n")
sys.stdout.flush()
config = ConfigParser.ConfigParser()
config.read('/home/biocadmin/packagebuilder/spb_history/aws.ini')
access_key = config.get("aws", "access_key_id")
secret_key = config.get("aws", "secret_key")
conn = SQSConnection(access_key, secret_key)
q = conn.get_queue("packagesubmitted")
try:
stomp = Stomp("broker.bioconductor.org", 61613)
stomp.connect()
except:
print("Cannot connect")
raise
def handle_message(msg):
body = msg.get_body()
print("got this message: %s\n" % body)
sys.stdout.flush()
# spawn another script to handle notifications of new packages:
script_path = os.path.dirname(os.path.realpath(__file__))
arg = base64.b64encode(body)
cmd = "ruby %s/new_package_notifier.rb %s" % (script_path, arg)
