class ResourceRegistryStandalone(object):
"""
Resource Registry service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name, config=config, scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def create(self, object=None, actor_id=None, lcstate=None):
if object is None:
raise BadRequest("Object not present")
if not "type_" in object:
raise BadRequest("Object is not an IonObject")
cur_time = get_ion_ts()
object['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
object['ts_created'] = cur_time
object['ts_updated'] = cur_time
new_res_id = create_unique_resource_id()
res_id, rev = self.datastore.create_doc(object, new_res_id)
if actor_id and actor_id != 'anonymous':
self.create_association(res_id, "hasOwner", actor_id)
return res
def create_mult(self, res_list, lcstate=None):
cur_time = get_ion_ts()
for resobj in res_list:
resobj['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
resobj['ts_created'] = cur_time
resobj['ts_updated'] = cur_time
id_list = [create_unique_resource_id() for i in xrange(len(res_list))]
res = self.datastore.create_doc_mult(res_list, id_list)
res_list = [(rid,rrv) for success,rid,rrv in res]
return res_list
def read(self, object_id='', rev_id=''):
if not object_id:
raise BadRequest("The object_id parameter is an empty string")
return self.datastore.read_doc(object_id, rev_id)
def read_mult(self, object_ids=None):
if not object_ids:
raise BadRequest("The object_ids parameter is empty")
return self.datastore.read_doc_mult(object_ids)
def create_association(self, subject=None, predicate=None, obj=None, assoc_type='H2H'):
"""
Create an association between two IonObjects with a given predicate
"""
if not subject or not predicate or not obj:
raise BadRequest("Association must have all elements set")
if type(subject) is str:
subject_id = subject
subject = self.read(subject_id)
else:
if "_id" not in subject or "_rev" not in subject:
raise BadRequest("Subject id or rev not available")
subject_id = subject._id
st = type(subject).__name__
if type(obj) is str:
object_id = obj
obj = self.read(object_id)
else:
if "_id" not in obj or "_rev" not in obj:
raise BadRequest("Object id or rev not available")
object_id = obj._id
ot = type(obj).__name__
assoc_type = assoc_type or 'H2H'
if not assoc_type in AT:
raise BadRequest("Unsupported assoc_type: %s" % assoc_type)
# Check that subject and object type are permitted by association definition
# Note: Need import here, so that import orders are not screwed up
from pyon.core.registry import getextends
from pyon.ion.resource import Predicates
from pyon.core.bootstrap import IonObject
assoc = dict(type_="Association",
at=assoc_type,
s=subject_id, st=st, srv=subject._rev,
p=predicate,
o=object_id, ot=ot, orv=obj._rev,
ts=get_ion_ts())
return self.datastore.create_doc(assoc, create_unique_association_id())
def find_by_type(self, restype, id_only=False, **kwargs):
start_key = [restype]
end_key = [restype]
res = self.datastore.find_docs_by_view('resource', 'by_type',
start_key=start_key, end_key=end_key, id_only=id_only, **kwargs)
if id_only:
match = [docid for docid, indexkey, doc in res]
else:
match = [doc for docid, indexkey, doc in res]
return match
class ResourceRegistryStandalone(object):
"""
Resource Registry service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name,
config=config,
scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def create(self, object=None, actor_id=None, lcstate=None):
if object is None:
raise BadRequest("Object not present")
if not "type_" in object:
raise BadRequest("Object is not an IonObject")
cur_time = get_ion_ts()
object['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
object['ts_created'] = cur_time
object['ts_updated'] = cur_time
new_res_id = create_unique_resource_id()
res_id, rev = self.datastore.create_doc(object, new_res_id)
if actor_id and actor_id != 'anonymous':
self.create_association(res_id, "hasOwner", actor_id)
return res
def create_mult(self, res_list, lcstate=None):
cur_time = get_ion_ts()
for resobj in res_list:
resobj['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
resobj['ts_created'] = cur_time
resobj['ts_updated'] = cur_time
id_list = [create_unique_resource_id() for i in xrange(len(res_list))]
res = self.datastore.create_doc_mult(res_list, id_list)
res_list = [(rid, rrv) for success, rid, rrv in res]
return res_list
def read(self, object_id='', rev_id=''):
if not object_id:
raise BadRequest("The object_id parameter is an empty string")
return self.datastore.read_doc(object_id, rev_id)
def read_mult(self, object_ids=None):
if not object_ids:
raise BadRequest("The object_ids parameter is empty")
return self.datastore.read_doc_mult(object_ids)
def create_association(self,
subject=None,
predicate=None,
obj=None,
assoc_type='H2H'):
"""
Create an association between two IonObjects with a given predicate
"""
if not subject or not predicate or not obj:
raise BadRequest("Association must have all elements set")
if type(subject) is str:
subject_id = subject
subject = self.read(subject_id)
else:
if "_id" not in subject or "_rev" not in subject:
raise BadRequest("Subject id or rev not available")
subject_id = subject._id
st = type(subject).__name__
if type(obj) is str:
object_id = obj
obj = self.read(object_id)
else:
if "_id" not in obj or "_rev" not in obj:
raise BadRequest("Object id or rev not available")
object_id = obj._id
ot = type(obj).__name__
assoc_type = assoc_type or 'H2H'
if not assoc_type in AT:
raise BadRequest("Unsupported assoc_type: %s" % assoc_type)
# Check that subject and object type are permitted by association definition
# Note: Need import here, so that import orders are not screwed up
from pyon.core.registry import getextends
from pyon.ion.resource import Predicates
from pyon.core.bootstrap import IonObject
assoc = dict(type_="Association",
at=assoc_type,
s=subject_id,
st=st,
srv=subject._rev,
p=predicate,
o=object_id,
ot=ot,
orv=obj._rev,
ts=get_ion_ts())
return self.datastore.create_doc(assoc, create_unique_association_id())
def find_by_type(self, restype, id_only=False, **kwargs):
start_key = [restype]
end_key = [restype]
res = self.datastore.find_docs_by_view('resource',
'by_type',
start_key=start_key,
end_key=end_key,
id_only=id_only,
**kwargs)
if id_only:
match = [docid for docid, indexkey, doc in res]
else:
match = [doc for docid, indexkey, doc in res]
return match
class QueueBlame(Plugin):
name = 'queueblame'
def __init__(self):
Plugin.__init__(self)
import uuid
self.ds_name = "queueblame-%s" % str(uuid.uuid4())[0:6]
from collections import defaultdict
self.queues_by_test = defaultdict(lambda: defaultdict(dict))
def options(self, parser, env):
super(QueueBlame, self).options(parser, env=env)
parser.add_option('--queueblame-by-queue',
action='store_true',
dest='queueblame_by_queue',
help='Show output by queue instead of by test',
default=False)
parser.add_option(
'--queueblame-full',
action='store_true',
dest='queueblame_full',
help='Display ALL queues, not just queues with consumers/msgs')
parser.add_option(
'--queueblame-no-trim',
action='store_false',
dest='queueblame_trim',
help=
'Trim output so that repeated test names/queue names are omitted for brevity. Human readable but not easily machine readable.',
default=True)
def configure(self, options, conf):
"""Configure the plugin and system, based on selected options."""
super(QueueBlame, self).configure(options, conf)
self._queueblame_by_queue = options.queueblame_by_queue
self._queueblame_full = options.queueblame_full
self._queueblame_trim = options.queueblame_trim
def begin(self):
from pyon.datastore.couchdb.couchdb_standalone import CouchDataStore
from pyon.public import CFG
self.ds = CouchDataStore(datastore_name=self.ds_name, config=CFG)
def finalize(self, result):
self.ds.delete_datastore()
self.ds.close()
def beforeTest(self, test):
import os
os.environ['QUEUE_BLAME'] = "%s,%s" % (self.ds_name, test.id())
def afterTest(self, test):
from pyon.net.transport import NameTrio, TransportError
from pyon.net.channel import RecvChannel
import os
import sys
# need a connection to node to get queue stats
from pyon.net.messaging import make_node
node, ioloop = make_node()
os.environ.pop('QUEUE_BLAME')
tid = test.id()
# grab raw data from database
obj_ids = self.ds.list_objects()
objs = self.ds.read_doc_mult(obj_ids)
for x in objs:
queue = x['queue_name']
if 'accesses' in self.queues_by_test[tid][queue]:
self.queues_by_test[tid][queue]['accesses'] += 1
else:
# grab intel from channel
ch = node.channel(RecvChannel)
ch._recv_name = NameTrio(queue.split('.')[0], queue)
try:
msgs, consumers = ch.get_stats()
exists = True
#print >>sys.stderr, "LOG ME", queue, msgs, consumers
except TransportError:
msgs = 0
consumers = 0
exists = False
finally:
ch.close()
self.queues_by_test[tid][queue] = {
'exists': exists,
'msgs': msgs,
'consumers': consumers,
'accesses': 1
}
# must also check all the queues from previous tests, to capture bleed
bleed_queues = set()
for test, testqueues in self.queues_by_test.iteritems():
if test != tid:
map(bleed_queues.add, testqueues.iterkeys())
# don't test anything we already just tested
bleed_queues.difference_update(self.queues_by_test[tid].iterkeys())
for queue in bleed_queues:
ch = node.channel(RecvChannel)
ch._recv_name = NameTrio(queue.split('.')[0], queue)
try:
msgs, consumers = ch.get_stats()
exists = True
except TransportError:
msgs = 0
consumers = 0
exists = False
# drain the queue!
if exists and msgs > 0 and consumers == 0:
print >> sys.stderr, "DRAIN QUEUE:", queue
ch.start_consume()
for x in xrange(msgs):
m, h, d = ch.recv()
print >> sys.stderr, h
ch.ack(d)
ch.close()
self.queues_by_test[tid][queue] = {
'exists': exists,
'msgs': msgs,
'consumers': consumers,
'accesses': 0
} # 0 is special here, indicates a bleed check
# empty the database for next test use
self.ds.delete_datastore()
self.ds.create_datastore(create_indexes=False)
node.stop_node()
ioloop.join(timeout=5)
def report(self, stream):
# format report
table = []
self.total_count = 0
def is_interesting(qd):
"""
Helper method, returns if a row is interesting based on msgs, consumers and queueblame_full flag.
"""
return self._queueblame_full or (not self._queueblame_full and
(qd['msgs'] > 0
or qd['consumers'] > 0))
def add_row(first, second, queuedict):
"""
Can be called with queue/test or test/queue first, hence generic name.
Returns bool indicated row was added or not.
"""
self.total_count += 1
if is_interesting(queuedict):
acc = ' '
if queuedict['accesses'] > 1:
acc = queuedict['accesses']
elif queuedict['accesses'] == 0:
acc = 'PREV'
exists = 'T' if queuedict['exists'] else 'F'
table.append([
str(x) for x in [
first, second, acc, exists, queuedict['msgs'],
queuedict['consumers']
]
])
return True
return False
# create tests by queue, used in a few places below
from collections import defaultdict
tests_by_queue = defaultdict(list)
for test, queues in self.queues_by_test.iteritems():
for queue, queuedict in queues.iteritems():
tests_by_queue[queue].append(dict(queuedict, test=test))
# list of queues that are tagged as PREV, we keep tabs on it but only use it later if correct conditions
prev_list = set()
# build output table
if not self._queueblame_by_queue:
for test, queues in self.queues_by_test.iteritems():
for queue, queuedict in queues.iteritems():
ret = add_row(test, queue, queuedict)
if ret and queuedict['accesses'] == 0:
prev_list.add(queue)
else:
for queue, calls in tests_by_queue.iteritems():
for call in calls:
ret = add_row(queue, call['test'], call)
if ret and queuedict['accesses'] == 0:
prev_list.add(queue)
# generate prev_list table if it is interesting
prev_list_table = []
if not self._queueblame_full:
for prev in prev_list:
# cut down dict
prev_accesses = [
qd['test'] for qd in tests_by_queue[prev]
if qd['accesses'] > 0
]
for pa in prev_accesses:
prev_list_table.append([prev, pa])
# sort by first col
table.sort(cmp=lambda x, y: cmp(x[0], y[0]))
prev_list_table.sort(cmp=lambda x, y: cmp(x[0], y[0]))
# do we trim?
if self._queueblame_trim:
last = ""
for i, x in enumerate(table):
if x[0] != last:
last = x[0]
else:
table[i][0] = ""
last = ""
for i, x in enumerate(prev_list_table):
if x[0] != last:
last = x[0]
else:
prev_list_table[i][0] = ""
if self._queueblame_by_queue:
table.insert(
0, ['Queue', 'Test', '# Acc >1', 'Ex?', '# Msgs', '# Cnsmrs'])
else:
table.insert(
0, ['Test', 'Queue', '# Acc >1', 'Ex?', '# Msgs', '# Cnsmrs'])
if len(prev_list_table) > 0:
prev_list_table.insert(0, ['Queue', 'Test'])
# format prev_table too
widths = [
max([len(row[x]) for row in prev_list_table])
for x in xrange(len(prev_list_table[0]))
]
prev_fmt_out = [
" ".join([x.ljust(widths[i]) for i, x in enumerate(row)])
for row in prev_list_table
]
prev_fmt_out.insert(
1, " ".join(
[''.ljust(widths[i], '=') for i in xrange(len(widths))]))
else:
prev_fmt_out = []
# calculate widths
widths = [
max([len(row[x]) for row in table]) for x in xrange(len(table[0]))
]
fmt_out = [
" ".join([x.ljust(widths[i]) for i, x in enumerate(row)])
for row in table
]
# insert col separation row
fmt_out.insert(
1,
" ".join([''.ljust(widths[i], '=') for i in xrange(len(widths))]))
stream.write("\n" + "=" * len(fmt_out[0]) + "\n\n")
stream.write(
"Queue blame report (DB: %s, full: %s, by_queue: %s)\n" %
(self.ds_name, self._queueblame_full, self._queueblame_by_queue))
stream.write(
"If 'PREV' in accesses column, indicates queue was not accessed during this test and could indicate bleed between tests.\n"
)
if not self._queueblame_full and len(table) > 1:
stream.write(
"\n*** The following queues still have messages or consumers! ***\n"
)
stream.write("\n")
stream.write("\n".join(fmt_out))
stream.write("\n" + "=" * len(fmt_out[0]) + "\n")
stream.write("%d shown of %d total\n" %
(len(table) - 1, self.total_count))
stream.write("\n")
if len(prev_fmt_out) > 0:
stream.write(
"\n\nThe following queues were accessed by the associated tests, inspect them for proper cleanup of subscribers!\n\n"
)
stream.write("\n".join(prev_fmt_out))
stream.write("\n" + "=" * len(prev_fmt_out[0]) + "\n")
stream.write("\n")
class QueueBlame(Plugin):
name = 'queueblame'
def __init__(self):
Plugin.__init__(self)
import uuid
self.ds_name = "queueblame-%s" % str(uuid.uuid4())[0:6]
from collections import defaultdict
self.queues_by_test = defaultdict(lambda: defaultdict(dict))
def options(self, parser, env):
super(QueueBlame, self).options(parser, env=env)
parser.add_option('--queueblame-by-queue', action='store_true', dest='queueblame_by_queue', help='Show output by queue instead of by test', default=False)
parser.add_option('--queueblame-full', action='store_true', dest='queueblame_full', help='Display ALL queues, not just queues with consumers/msgs')
parser.add_option('--queueblame-no-trim', action='store_false', dest='queueblame_trim', help='Trim output so that repeated test names/queue names are omitted for brevity. Human readable but not easily machine readable.', default=True)
def configure(self, options, conf):
"""Configure the plugin and system, based on selected options."""
super(QueueBlame, self).configure(options, conf)
self._queueblame_by_queue = options.queueblame_by_queue
self._queueblame_full = options.queueblame_full
self._queueblame_trim = options.queueblame_trim
def begin(self):
from pyon.datastore.couchdb.couchdb_standalone import CouchDataStore
from pyon.public import CFG
self.ds = CouchDataStore(datastore_name=self.ds_name, config=CFG)
def finalize(self, result):
self.ds.delete_datastore()
self.ds.close()
def beforeTest(self, test):
import os
os.environ['QUEUE_BLAME'] = "%s,%s" % (self.ds_name, test.id())
def afterTest(self, test):
from pyon.net.transport import NameTrio, TransportError
from pyon.net.channel import RecvChannel
import os
import sys
# need a connection to node to get queue stats
from pyon.net.messaging import make_node
node, ioloop = make_node()
os.environ.pop('QUEUE_BLAME')
tid = test.id()
# grab raw data from database
obj_ids = self.ds.list_objects()
objs = self.ds.read_doc_mult(obj_ids)
for x in objs:
queue = x['queue_name']
if 'accesses' in self.queues_by_test[tid][queue]:
self.queues_by_test[tid][queue]['accesses'] += 1
else:
# grab intel from channel
ch = node.channel(RecvChannel)
ch._recv_name = NameTrio(queue.split('.')[0], queue)
try:
msgs, consumers = ch.get_stats()
exists = True
#print >>sys.stderr, "LOG ME", queue, msgs, consumers
except TransportError:
msgs = 0
consumers = 0
exists = False
finally:
ch.close()
self.queues_by_test[tid][queue] = { 'exists': exists,
'msgs': msgs,
'consumers' : consumers,
'accesses' : 1 }
# must also check all the queues from previous tests, to capture bleed
bleed_queues = set()
for test, testqueues in self.queues_by_test.iteritems():
if test != tid:
map(bleed_queues.add, testqueues.iterkeys())
# don't test anything we already just tested
bleed_queues.difference_update(self.queues_by_test[tid].iterkeys())
for queue in bleed_queues:
ch = node.channel(RecvChannel)
ch._recv_name = NameTrio(queue.split('.')[0], queue)
try:
msgs, consumers = ch.get_stats()
exists = True
except TransportError:
msgs = 0
consumers = 0
exists = False
# drain the queue!
if exists and msgs > 0 and consumers == 0:
print >>sys.stderr, "DRAIN QUEUE:", queue
ch.start_consume()
for x in xrange(msgs):
m, h, d = ch.recv()
print >>sys.stderr, h
ch.ack(d)
ch.close()
self.queues_by_test[tid][queue] = { 'exists': exists,
'msgs': msgs,
'consumers': consumers,
'accesses' : 0 } # 0 is special here, indicates a bleed check
# empty the database for next test use
self.ds.delete_datastore()
self.ds.create_datastore(create_indexes=False)
node.stop_node()
ioloop.join(timeout=5)
def report(self, stream):
# format report
table = []
self.total_count = 0
def is_interesting(qd):
"""
Helper method, returns if a row is interesting based on msgs, consumers and queueblame_full flag.
"""
return self._queueblame_full or (not self._queueblame_full and (qd['msgs'] > 0 or qd['consumers'] > 0))
def add_row(first, second, queuedict):
"""
Can be called with queue/test or test/queue first, hence generic name.
Returns bool indicated row was added or not.
"""
self.total_count += 1
if is_interesting(queuedict):
acc = ' '
if queuedict['accesses'] > 1:
acc = queuedict['accesses']
elif queuedict['accesses'] == 0:
acc = 'PREV'
exists = 'T' if queuedict['exists'] else 'F'
table.append([str(x) for x in [first, second, acc, exists, queuedict['msgs'], queuedict['consumers']]])
return True
return False
# create tests by queue, used in a few places below
from collections import defaultdict
tests_by_queue = defaultdict(list)
for test, queues in self.queues_by_test.iteritems():
for queue, queuedict in queues.iteritems():
tests_by_queue[queue].append(dict(queuedict, test=test))
# list of queues that are tagged as PREV, we keep tabs on it but only use it later if correct conditions
prev_list = set()
# build output table
if not self._queueblame_by_queue:
for test, queues in self.queues_by_test.iteritems():
for queue, queuedict in queues.iteritems():
ret = add_row(test, queue, queuedict)
if ret and queuedict['accesses'] == 0:
prev_list.add(queue)
else:
for queue, calls in tests_by_queue.iteritems():
for call in calls:
ret = add_row(queue, call['test'], call)
if ret and queuedict['accesses'] == 0:
prev_list.add(queue)
# generate prev_list table if it is interesting
prev_list_table = []
if not self._queueblame_full:
for prev in prev_list:
# cut down dict
prev_accesses = [qd['test'] for qd in tests_by_queue[prev] if qd['accesses'] > 0]
for pa in prev_accesses:
prev_list_table.append([prev, pa])
# sort by first col
table.sort(cmp=lambda x,y: cmp(x[0], y[0]))
prev_list_table.sort(cmp=lambda x,y: cmp(x[0], y[0]))
# do we trim?
if self._queueblame_trim:
last = ""
for i, x in enumerate(table):
if x[0] != last:
last = x[0]
else:
table[i][0] = ""
last = ""
for i, x in enumerate(prev_list_table):
if x[0] != last:
last = x[0]
else:
prev_list_table[i][0] = ""
if self._queueblame_by_queue:
table.insert(0, ['Queue', 'Test', '# Acc >1', 'Ex?', '# Msgs', '# Cnsmrs'])
else:
table.insert(0, ['Test', 'Queue', '# Acc >1', 'Ex?', '# Msgs', '# Cnsmrs'])
if len(prev_list_table) > 0:
prev_list_table.insert(0, ['Queue', 'Test'])
# format prev_table too
widths = [max([len(row[x]) for row in prev_list_table]) for x in xrange(len(prev_list_table[0]))]
prev_fmt_out = [" ".join([x.ljust(widths[i]) for i, x in enumerate(row)]) for row in prev_list_table]
prev_fmt_out.insert(1, " ".join([''.ljust(widths[i], '=') for i in xrange(len(widths))]))
else:
prev_fmt_out = []
# calculate widths
widths = [max([len(row[x]) for row in table]) for x in xrange(len(table[0]))]
fmt_out = [" ".join([x.ljust(widths[i]) for i, x in enumerate(row)]) for row in table]
# insert col separation row
fmt_out.insert(1, " ".join([''.ljust(widths[i], '=') for i in xrange(len(widths))]))
stream.write("\n" + "=" * len(fmt_out[0]) + "\n\n")
stream.write("Queue blame report (DB: %s, full: %s, by_queue: %s)\n" % (self.ds_name, self._queueblame_full, self._queueblame_by_queue))
stream.write("If 'PREV' in accesses column, indicates queue was not accessed during this test and could indicate bleed between tests.\n")
if not self._queueblame_full and len(table) > 1:
stream.write("\n*** The following queues still have messages or consumers! ***\n")
stream.write("\n")
stream.write("\n".join(fmt_out))
stream.write("\n" + "=" * len(fmt_out[0]) + "\n")
stream.write("%d shown of %d total\n" % (len(table)-1, self.total_count))
stream.write("\n")
if len(prev_fmt_out) > 0:
stream.write("\n\nThe following queues were accessed by the associated tests, inspect them for proper cleanup of subscribers!\n\n")
stream.write("\n".join(prev_fmt_out))
stream.write("\n" + "=" * len(prev_fmt_out[0]) + "\n")
stream.write("\n")
