def _expand_range(cls, data):
"""
Return an iterator on multiple lines based on the ranges the provided
data contains.
"""
fmt = '' # Format string
rng_list = [] # List of ranges found
while data.find('[') >= 0:
pfx, sfx = data.split('[', 1)
rng, data = sfx.split(']', 1)
rng_list.append(rng)
fmt += "%s%%s" % pfx
fmt += data
# If no range is found, no more work is needed
if not rng_list:
return iter([data])
# Create range iterators for all ranges found.
rangesets = [RangeSet(rng) for rng in rng_list]
# Verify that all ranges have the same length
minsize = min([len(rng) for rng in rangesets])
maxsize = max([len(rng) for rng in rangesets])
if minsize != maxsize:
raise ModelFileValueError("Range size mismatch %d != %d" %
(minsize, maxsize))
# Need striter() to build padded strings if present
rangesets = [list(rng.striter()) for rng in rangesets]
# Generate the new lines based from the rangeset
return (fmt % tpl for tpl in zip(*rangesets))
def _testRS(self, r, exc):
try:
rset = RangeSet(r)
print rset
except RangeSetParseError, e:
self.assertEqual(RangeSetParseError, exc)
return
def _testRS(self, r, exc):
try:
rset = RangeSet(r)
except RangeSetParseError as e:
self.assertEqual(RangeSetParseError, exc)
return
except:
raise
self.assert_(0, "error not detected/no exception raised")
def _strfold(self):
"""Convert to string grouping elements using ranges when possible."""
for pattern, indexes in self._fold_range():
if indexes:
rngs = []
for elem in indexes:
rng = RangeSet(elem[0])
rng.padding = elem[1]
if len(rng) > 1:
rngs.append('[%s]' % rng)
else:
rngs.append(str(rng))
yield pattern % tuple(rngs)
else:
yield pattern
def check_rangeset(option, opt, value):
try:
return RangeSet(value)
except RangeSetParseError:
raise OptionValueError(
"option %s: invalid rangeset value: %s" % (opt, value))
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
group_resolver.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(options.regroup) + int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=options.autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not options.list:
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, options.autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__,
options.autostep)
else:
xset.intersection_update(
class_set(nodes, autostep=options.autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__,
options.autostep)
else:
xset.difference_update(class_set(nodes, autostep=options.autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
options.autostep)
else:
xset.symmetric_difference_update(class_set(nodes, \
autostep=options.autostep))
# Finish xset computing from args
compute_nodeset(xset, args, options.autostep)
# The list command has a special handling
if options.list > 0:
return command_list(options, xset)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__": None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
format = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((format % s for s in x.striter()))
elif options.fold:
xsubres = lambda x: format % x
elif options.regroup:
xsubres = lambda x: format % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: format % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
group_resolver.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(bool(options.listall)) + int(options.regroup) + \
int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.axis and (not options.fold or options.rangeset):
parser.error("--axis option is only supported when folding nodeset")
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
autostep = options.autostep
# Do not use autostep for computation when a percentage or the special
# value 'auto' is specified. Real autostep value is set post-process.
if type(autostep) is float or autostep == 'auto':
autostep = None
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not (options.list or options.listall):
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__, autostep)
else:
xset.intersection_update(class_set(nodes, autostep=autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__, autostep)
else:
xset.difference_update(class_set(nodes, autostep=autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
autostep)
else:
xset.symmetric_difference_update(
class_set(nodes, autostep=autostep))
# Finish xset computing from args
compute_nodeset(xset, args, autostep)
# The list command has a special handling
if options.list > 0 or options.listall > 0:
return command_list(options, xset, group_resolver)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__": None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
if options.autostep == 'auto':
# Simple implementation of --autostep=auto
# if we have at least 3 nodes, all index should be foldable as a-b/n
xset.autostep = max(3, len(xset))
elif type(options.autostep) is float:
# at least % of nodes should be foldable as a-b/n
autofactor = float(options.autostep)
xset.autostep = int(math.ceil(float(len(xset)) * autofactor))
# user-specified nD-nodeset fold axis
if options.axis:
if not options.axis.startswith('-'):
# axis are 1-indexed in nodeset CLI (0 ignored)
xset.fold_axis = tuple(x - 1 for x in RangeSet(options.axis)
if x > 0)
else:
# negative axis index (only single number supported)
xset.fold_axis = [int(options.axis)]
fmt = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((fmt % s for s in x.striter()))
elif options.fold:
# Special case when folding using NodeSet and format is set (#277)
if class_set is NodeSet and fmt != '%s':
# Create a new set after format has been applied to each node
xset = class_set._fromlist1((fmt % xnodestr for xnodestr in xset),
autostep=xset.autostep)
xsubres = lambda x: x
else:
xsubres = lambda x: fmt % x
elif options.regroup:
xsubres = lambda x: fmt % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: fmt % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
def route_stream(self, request_iterator, context):
#FIXME: Refactor this
#First, see from the header message if we are part of the recipients
init_msg = next(request_iterator)
stream_local = False
if self._vmid in RangeSet(
init_msg.destinations.encode('ascii', 'ignore')):
# If we are part of the recipients, use a tee to get a
# local copy of the stream while forwarding it
local_iter, forward_iter = mt_tee(request_iterator)
# Unpack the header message to initialize the stream and
# find out how to handle the next messages
req = getattr(agent_pb2, init_msg.args.TypeName())()
init_msg.args.Unpack(req)
input_handler, output_handler, ret = self._stream_init_handler(
init_msg.name, req, context)
if isinstance(ret, agent_pb2.GenericError):
ret_msg = agent_pb2.RouteMessageResult(source=self._vmid,
error=ret)
# If the header handling resulted in error, we stop
# the stream handling on this node
stream_local = False
else:
stream_local = True
ret_msg = agent_pb2.RouteMessageResult(source=self._vmid)
ret_msg.result.Pack(ret)
logging.debug("Tbon: %d returning first reply for stream",
self._vmid)
yield ret_msg
if stream_local:
# Build input and output handlers for the following
# messages based on the callbacks received from handling
# the header message
logging.debug("Tbon: %d continuing in local+relay mode",
self._vmid)
def get_output():
# Get messages from the generator for this stream
for output_msg in output_handler(context):
if isinstance(output_msg, agent_pb2.GenericError):
ret_msg = agent_pb2.RouteMessageResult(
source=self._vmid, error=output_msg)
else:
ret_msg = agent_pb2.RouteMessageResult(
source=self._vmid)
ret_msg.result.Pack(output_msg)
yield ret_msg
def send_input():
# Push everything from the local iter to the handler
# for this stream
for input_msg in local_iter:
req = getattr(agent_pb2, input_msg.args.TypeName())()
input_msg.args.Unpack(req)
input_handler(input_msg.name, req, context)
#Create a dedicated thread to block and push on the local iterator
thread = threading.Thread(target=send_input)
thread.start()
# Forward the header message + following messages to the
# next hops and yield everything they send us + what we
# produce locally
def new_iterin():
yield init_msg
for i in forward_iter:
yield i
for e in mt_chain(
[get_output(),
self._relay.route_stream(new_iterin())]):
yield e
else:
# We are not part of the recipients so just forward the
# whole stream to the next hops and yield everything they
# send us
def new_iterin():
yield init_msg
for i in request_iterator:
yield i
logging.debug("Tbon: %d continuing stream in relay mode",
self._vmid)
for e in self._relay.route_stream(new_iterin()):
logging.debug("Node %d ouputing message %s from children rpcs",
self._vmid, e)
yield e
logging.debug("Tbon: %d finished with stream", self._vmid)