def __init__(self, qpath, save_msgs, backend, host, port, db, user, pwd,
pidfile):
'''
Set up the database details.
'''
self._db_host = host
self._db_port = port
self._db_name = db
self._db_username = user
self._db_pwd = pwd
# Only here (and in the imports) would we change if we needed
# to change the DB class used.
self._apeldb = apel.db.ApelDb(backend, self._db_host, self._db_port,
self._db_username, self._db_pwd,
self._db_name)
# Check the connection while we're setting up.
self._apeldb.test_connection()
inqpath = os.path.join(qpath, "incoming")
rejectqpath = os.path.join(qpath, "reject")
self._inq = Queue(inqpath, schema=QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=REJECT_SCHEMA)
self._save_msgs = save_msgs
if self._save_msgs:
acceptqpath = os.path.join(qpath, "accept")
self._acceptq = Queue(acceptqpath, schema=QSCHEMA)
# Create a message factory
self._rf = RecordFactory()
self._pidfile = pidfile
log.info("Loader created.")
def check_dir(root):
'''
Check the directory for incoming, outgoing, reject
or accept directories. If they exist, check them for
messages.
'''
print '\nStarting message status script.'
print 'Root directory: %s\n' % root
queues = []
incoming = os.path.join(root, 'incoming')
if os.path.isdir(incoming):
queues.append(Queue(incoming, schema=QSCHEMA))
outgoing = os.path.join(root, 'outgoing')
# outgoing uses QueueSimple, not Queue
if os.path.isdir(outgoing):
queues.append(QueueSimple(outgoing))
reject = os.path.join(root, 'reject')
if os.path.isdir(reject):
queues.append(Queue(reject, schema=QSCHEMA))
accept = os.path.join(root, 'accept')
if os.path.isdir(accept):
queues.append(Queue(accept, schema=QSCHEMA))
for q in queues:
msgs, locked = check_queue(q)
#check_empty_dirs(q)
print ' Messages: %s' % msgs
print ' Locked: %s\n' % locked
if locked > 0:
question = 'Unlock %s messages?' % locked
if ask_user(question):
clear_locks(q)
def test1_init(self):
""" QueueSet.__init__() """
self.failUnlessRaises(TypeError, QueueSet, ('a'))
self.failUnlessRaises(TypeError, QueueSet,
(Queue(self.q1, schema={'data': 'string'}), 'a'))
self.failUnlessRaises(
TypeError, QueueSet,
([Queue(self.q1, schema={'data': 'string'}), 'a']))
self.failUnlessRaises(TypeError, QueueSet, ([1, 2]))
self.failUnlessRaises(TypeError, QueueSet, ((1, 2)))
def test3_firstnext(self):
""" QueueSet.first()/next() """
q1 = Queue(self.q1, schema={'data': 'string'})
q2 = Queue(self.q2, schema={'data': 'string'})
for i in range(10):
q1.add({'data': '%i A\n' % i})
q2.add({'data': '%i A\n' % i})
qs = QueueSet([q1, q2])
e = qs.first()
assert isinstance(e, tuple)
assert isinstance(e[0], Queue)
assert isinstance(e[1], str)
while e[0]:
e = qs.next()
def test2_addremove(self):
""" QueueSet.add()/remove() """
q1 = Queue(self.q1, schema={'data': 'string'})
q2 = Queue(self.q2, schema={'data': 'string'})
q3 = Queue(self.q3, schema={'data': 'string'})
q4 = Queue(self.q4, schema={'data': 'string'})
for i in range(10):
q1.add({'data': '%i A\n' % i})
q2.add({'data': '%i A\n' % i})
q3.add({'data': '%i A\n' % i})
q4.add({'data': '%i A\n' % i})
qs = QueueSet([q1, q2])
qs.add(q3)
qs.remove(q1)
qs.add(q1, q4)
self.failUnlessRaises(QueueError, qs.add, ([q1]))
def purge_queue():
settings = Settings()
try:
queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
return queue.purge()
except OSError, error:
raise JensMessagingError("Failed to purge Queue object (%s)" % error)
def count_pending_hints():
settings = Settings()
try:
queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
return queue.count()
except OSError, error:
raise JensMessagingError("Failed to create Queue object (%s)" % error)
def test1init(self):
""" Queue.__init__() """
path = self.tempdir + '/aaa/bbb/ccc'
umask = None
maxelts = 10
good_schemas = [
{
'a': 'binary'
},
{
'a': 'binary*'
},
{
'a': 'string'
},
{
'a': 'string*'
},
{
'a': 'table'
},
{
'a': 'binary?',
'b': 'binary'
},
{
'a': 'binary?',
'b': 'binary*'
},
{
'a': 'string',
'b': 'binary?*'
},
]
for schema in good_schemas:
try:
Queue(path, umask=umask, maxelts=maxelts, schema=schema)
except Exception:
error = sys.exc_info()[1]
self.fail("Shouldn't have failed. Exception: %s" % error)
bad_schemas = [['foo'], {'foo': 1}, {'a': 'binary?'}]
for schema in bad_schemas:
self.failUnlessRaises(QueueError,
Queue,
path,
umask=umask,
maxelts=maxelts,
schema=schema)
bad_schemas = [{'a': 'strings'}, {'a': 'table??'}]
for schema in bad_schemas:
self.failUnlessRaises(QueueError,
Queue,
path,
umask=umask,
maxelts=maxelts,
schema=schema)
def test2copy(self):
""" Queue.copy(). """
q = Queue(self.qdir, schema={'a': 'string'})
q1 = q.copy()
q.foo = 1
try:
q1.foo
except AttributeError:
pass
else:
self.fail('Not a copy, but reference.')
def test3_is_locked(self):
""" Queue._is_locked_*() """
q = Queue(self.qdir, schema={'a': 'string'})
assert q._is_locked_nlink('') is False
assert q._is_locked_nlink('not_there') is False
os.mkdir(self.qdir + '/a')
assert q._is_locked_nlink('a') is False
os.mkdir(self.qdir + '/a/%s' % queue.LOCKED_DIRECTORY)
assert q._is_locked_nlink('a') is True
time.sleep(1)
assert q._is_locked_nlink('a', time.time()) is True
assert q._is_locked_nonlink('') is False
assert q._is_locked_nonlink('not_there') is False
os.mkdir(self.qdir + '/b')
assert q._is_locked_nonlink('b') is False
os.mkdir(self.qdir + '/b/%s' % queue.LOCKED_DIRECTORY)
assert q._is_locked_nonlink('b') is True
time.sleep(1)
assert q._is_locked_nonlink('b', time.time()) is True
def _fetch_all_messages(settings):
try:
queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
except OSError, error:
raise JensMessagingError("Failed to create Queue object (%s)" % error)
# number of queues
QUEUES = 4
print("*** Setup & populate queues")
# generate paths
paths = []
for i in range(QUEUES):
paths.append(working_dir + '/q-%i' % i)
COUNT = 5
print("creating %i initial queues. adding %i elements into each." %
(QUEUES, COUNT))
queues = []
queue_num = 0
while queue_num < QUEUES:
queue = Queue(paths[queue_num], maxelts=5, schema={'body': 'string'})
print("adding %d elements to the queue %s" % (COUNT, paths[queue_num]))
element = {}
done = 0
while not COUNT or done < COUNT:
done += 1
element['body'] = 'Queue %i. Element %i' % (queue_num, done)
queue.add(element)
queues.append(queue)
queue_num += 1
print("done.")
print("*** Browse")
i = 2
queue_set = QueueSet(queues[0:i])
queue_set_count = queue_set.count()
def add_msg_to_queue(settings, msg):
queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
queue.enqueue(msg)
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None,
path_type='dirq',
protocol=STOMP_MESSAGING,
project=None,
token=''):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# Used to differentiate between STOMP and AMS methods
self._protocol = protocol
# Used when interacting with an Argo Messaging Service
self._project = project
self._token = token
if self._protocol == Ssm2.AMS_MESSAGING:
self._ams = ArgoMessagingService(endpoint=self._brokers[0],
token=self._token,
cert=self._cert,
key=self._key,
project=self._project)
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
# Determine what sort of outgoing structure to make
if path_type == 'dirq':
if QueueSimple is None:
raise ImportError("dirq path_type requested but the dirq "
"module wasn't found.")
self._outq = QueueSimple(qpath)
elif path_type == 'directory':
self._outq = MessageDirectory(qpath)
else:
raise Ssm2Exception('Unsupported path_type variable.')
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
# Receivers must use the dirq module, so make a quick sanity check
# that dirq is installed.
if Queue is None:
raise ImportError("Receiving SSMs must use dirq, but the dirq "
"module wasn't found.")
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired or will expire '
'within a day.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired or will expire '
'within a day. Please obtain the new one from the final '
'server receiving your messages.' % enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see entries from stomp.py and connectionpool at WARNING and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see entries from stomp.py and connectionpool at INFO above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.INFO)
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
self._outq = QueueSimple(qpath)
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s' % enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
self._outq = QueueSimple(qpath)
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired. Please obtain the '
'new one from the final server receiving your messages.' %
enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see log entries from stomp.py at the warning level and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see log entries from stomp.py at the info level and above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
def _queue_item(item):
settings = Settings()
try:
queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA)
except OSError, error:
raise JensMessagingError("Failed to create Queue object (%s)" % error)
"""Tool to convert unloaded messages to loadable ones."""
from dirq.QueueSimple import QueueSimple
from dirq.queue import Queue
inpath = '/root/iris/messages'
outpath = '/root/iris/messages/incoming'
OUTQSCHEMA = {"body": "string", "signer": "string",
"empaid": "string?", "error": "string?"}
inq = QueueSimple(inpath)
outq = Queue(outpath, schema=OUTQSCHEMA)
for name in inq:
if not inq.lock(name):
continue
data = inq.get(name)
outq.add({'body': data, 'signer': 'iris', 'empaid': 'local'})
inq.remove(name)
import sys
import tempfile
# total number of elements
COUNT = 9
# queue head directory
path = tempfile.mkdtemp()
# max elements per elements directory
maxelts = 3
# element's schema
schema = {'body': 'string', 'header': 'table?'}
# ========
# PRODUCER
print("*** PRODUCER")
dirq_p = Queue(path, maxelts=maxelts, schema=schema)
print("adding %d elements to the queue at %s" % (COUNT, path))
done = 1
while done <= COUNT:
element = {}
try:
element['body'] = ('Élément %i \u263A\n' % done).decode("utf-8")
except AttributeError:
element['body'] = 'Élément %i \u263A\n' % done
if done % 2: # header only for odd sequential elements
element['header'] = dict(os.environ)
name = dirq_p.enqueue(element)
# name = dirq_p.add(element) # same
print("added %.2i: %s" % (done, name))
done += 1
def post(self, request, format=None):
"""
Submit Cloud Accounting Records.
.../api/v1/cloud/record
Will save Cloud Accounting Records for later loading.
"""
try:
empaid = request.META['HTTP_EMPA_ID']
except KeyError:
empaid = 'noid'
self.logger.info("Received message. ID = %s", empaid)
try:
signer = request.META['SSL_CLIENT_S_DN']
except KeyError:
self.logger.error("No DN supplied in header")
return Response(status=401)
# authorise DNs here
if not self._signer_is_valid(signer):
self.logger.error("%s not a valid provider", signer)
return Response(status=403)
if "_content" in request.POST.dict():
# then POST likely to come via the rest api framework
# hence use the content of request.POST as message
body = request.POST.get('_content')
else:
# POST likely to comes through a browser client or curl
# hence use request.body as message
body = request.body
self.logger.debug("Message body received: %s", body)
for header in request.META:
self.logger.debug("%s: %s", header, request.META[header])
# taken from ssm2
QSCHEMA = {'body': 'string',
'signer': 'string',
'empaid': 'string?'}
inqpath = os.path.join(settings.QPATH, 'incoming')
inq = Queue(inqpath, schema=QSCHEMA)
try:
name = inq.add({'body': body,
'signer': signer,
'empaid': empaid})
except QueueError as err:
self.logger.error("Could not save %s/%s: %s", inqpath, name, err)
response = "Data could not be saved to disk, please try again."
return Response(response, status=500)
self.logger.info("Message saved to in queue as %s/%s", inqpath, name)
response = "Data successfully saved for future loading."
return Response(response, status=202)
