def _create_channel(channel, node, default_content_type=None):
device = channel.get('device')
if has_control_chars(device):
raise ClusterConfigParsingError(
_('Bad device name: %s in %s') % (device, node.name))
path = parse_location(channel.get('path'))
if not device:
raise ClusterConfigParsingError(
_('Must specify device for file in %s') % node.name)
access = DEVICE_MAP.get(device, -1)
mode = channel.get('mode', None)
meta = channel.get('meta', {})
content_type = channel.get('content_type',
default_content_type if path else 'text/html')
if access & ACCESS_READABLE and path:
if not is_swift_path(path):
raise ClusterConfigParsingError(
_('Readable device must be a swift object'))
if not path.account or not path.container:
raise ClusterConfigParsingError(
_('Invalid path %s in %s') % (path.url, node.name))
return ZvmChannel(device,
access,
path=path,
content_type=content_type,
meta_data=meta,
mode=mode)
def _create_channel(channel, node, default_content_type=None):
device = channel.get('device')
if has_control_chars(device):
raise ClusterConfigParsingError(_('Bad device name: %s in %s') % (device, node.name))
path = parse_location(channel.get('path'))
if not device:
raise ClusterConfigParsingError(_('Must specify device for file in %s') % node.name)
access = DEVICE_MAP.get(device, -1)
mode = channel.get('mode', None)
meta = channel.get('meta', {})
content_type = channel.get('content_type', default_content_type if path else 'text/html')
if access & ACCESS_READABLE and path:
if not is_swift_path(path):
raise ClusterConfigParsingError(_('Readable device must be a swift object'))
if not path.account or not path.container:
raise ClusterConfigParsingError(_('Invalid path %s in %s')
% (path.url, node.name))
return ZvmChannel(device, access, path=path,
content_type=content_type, meta_data=meta, mode=mode)
def _create_node(node_config):
name = node_config.get('name')
if not name:
raise ClusterConfigParsingError(_('Must specify node name'))
if has_control_chars(name):
raise ClusterConfigParsingError(_('Invalid node name'))
nexe = node_config.get('exec')
if not nexe:
raise ClusterConfigParsingError(_('Must specify exec stanza for %s') % name)
exe = parse_location(nexe.get('path'))
if not exe:
raise ClusterConfigParsingError(_('Must specify executable path for %s') % name)
if is_zvm_path(exe):
raise ClusterConfigParsingError(_('Executable path cannot be a zvm path in %s') % name)
args = nexe.get('args')
env = nexe.get('env')
if has_control_chars('%s %s %s' % (exe.url, args, env)):
raise ClusterConfigParsingError(_('Invalid nexe property for %s') % name)
replicate = node_config.get('replicate', 1)
return ZvmNode(0, name, exe, args, env, replicate)
def _create_node(node_config):
name = node_config.get('name')
if not name:
raise ClusterConfigParsingError(_('Must specify node name'))
if has_control_chars(name):
raise ClusterConfigParsingError(_('Invalid node name'))
nexe = node_config.get('exec')
if not nexe:
raise ClusterConfigParsingError(
_('Must specify exec stanza for %s') % name)
exe = parse_location(nexe.get('path'))
if not exe:
raise ClusterConfigParsingError(
_('Must specify executable path for %s') % name)
if is_zvm_path(exe):
raise ClusterConfigParsingError(
_('Executable path cannot be a zvm path in %s') % name)
args = nexe.get('args')
env = nexe.get('env')
if has_control_chars('%s %s %s' % (exe.url, args, env)):
raise ClusterConfigParsingError(
_('Invalid nexe property for %s') % name)
replicate = node_config.get('replicate', 1)
return ZvmNode(0, name, exe, args, env, replicate)
def parse(self, cluster_config, add_user_image, account_name=None, replica_count=1, **kwargs):
"""
Parse deserialized config and build separate job configs per node
:param cluster_config: deserialized JSON cluster map
:param add_user_image: True if we need to add user image channel to all nodes
:param **kwargs: optional arguments for list_container, list_account callbacks
:raises ClusterConfigParsingError: on all errors
"""
self.nodes = {}
self.node_id = 1
self.node_list = []
try:
connect_devices = {}
for node in cluster_config:
zvm_node = _create_node(node)
node_count = node.get('count', 1)
if isinstance(node_count, int) and node_count > 0:
pass
else:
raise ClusterConfigParsingError(_('Invalid node count: %s') % str(node_count))
file_list = node.get('file_list')
read_list = []
write_list = []
other_list = []
if file_list:
for f in file_list:
channel = _create_channel(
f, zvm_node,
default_content_type=self.default_content_type)
if is_zvm_path(channel.path):
_add_connected_device(connect_devices, channel, zvm_node)
continue
if channel.access < 0:
if self.is_sysimage_device(channel.device):
other_list.append(channel)
continue
raise ClusterConfigParsingError(_('Unknown device %s in %s')
% (channel.device, zvm_node.name))
if channel.access & ACCESS_READABLE:
read_list.insert(0, channel)
elif channel.access & ACCESS_CDR:
read_list.append(channel)
elif channel.access & ACCESS_WRITABLE:
write_list.append(channel)
else:
other_list.append(channel)
read_group = False
for chan in read_list:
if '*' in chan.path.path:
read_group = True
object_list = self.find_objects(chan.path, **kwargs)
read_mask = re.escape(chan.path.path).replace('\\*', '(.*)')
read_mask = re.compile(read_mask)
node_count = len(object_list)
for i in range(node_count):
new_path = object_list[i]
new_node = self._add_new_channel(zvm_node, chan, index=(i + 1), path=new_path)
new_node.store_wildcards(new_path, read_mask)
else:
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node, chan, index=i)
else:
self._add_new_channel(zvm_node, chan)
for chan in write_list:
if chan.path and '*' in chan.path.url:
if read_group:
for i in range(1, node_count + 1):
new_node = self.nodes.get(_create_node_name(zvm_node.name, i))
new_url = _extract_stored_wildcards(chan.path, new_node)
new_node.add_channel(channel=chan,
path=parse_location(new_url))
else:
for i in range(1, node_count + 1):
new_name = _create_node_name(zvm_node.name, i)
new_url = chan.path.url.replace('*', new_name)
new_node = self._add_new_channel(zvm_node, chan, index=i,
path=parse_location(new_url))
new_node.wildcards = [new_name] * chan.path.url.count('*')
elif chan.path:
if node_count > 1:
raise ClusterConfigParsingError(_('Single path %s for multiple node '
'definition: %s, please use wildcard')
% (chan.path.url, zvm_node.name))
self._add_new_channel(zvm_node, chan)
else:
if 'stdout' not in chan.device \
and 'stderr' not in chan.device:
raise ClusterConfigParsingError(_('Immediate response is not available '
'for device %s') % chan.device)
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node, chan, index=i)
else:
self._add_new_channel(zvm_node, chan)
for chan in other_list:
if self.is_sysimage_device(chan.device):
chan.access = ACCESS_RANDOM | ACCESS_READABLE
else:
if not chan.path:
raise ClusterConfigParsingError(_('Path is required for device: %s') % chan.device)
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node, chan, index=i)
else:
self._add_new_channel(zvm_node, chan)
except ClusterConfigParsingError:
raise
except Exception:
print traceback.format_exc()
raise ClusterConfigParsingError('Config parser internal error')
for node in cluster_config:
connection_list = node.get('connect')
node_name = node.get('name')
src_devices = connect_devices.get(node_name, None)
if not connection_list:
if src_devices:
connection_list = [connected_node for connected_node in src_devices.iterkeys()]
else:
continue
self._add_all_connections(node_name, connection_list, src_devices)
for node_name in sorted(self.nodes.keys()):
self.node_list.append(self.nodes[node_name])
if add_user_image:
for node in self.node_list:
node.add_new_channel('image', ACCESS_CDR, removable='yes')
if account_name:
self.resolve_path_info(account_name, replica_count)
self.total_count = 0
for n in self.node_list:
self.total_count += n.replicate
def parse(self,
cluster_config,
add_user_image,
account_name=None,
replica_count=1,
**kwargs):
"""
Parse deserialized config and build separate job configs per node
:param cluster_config: deserialized JSON cluster map
:param add_user_image: True if we need to add user image channel to all nodes
:param **kwargs: optional arguments for list_container, list_account callbacks
:raises ClusterConfigParsingError: on all errors
"""
self.nodes = {}
self.node_id = 1
self.node_list = []
try:
connect_devices = {}
for node in cluster_config:
zvm_node = _create_node(node)
node_count = node.get('count', 1)
if isinstance(node_count, int) and node_count > 0:
pass
else:
raise ClusterConfigParsingError(
_('Invalid node count: %s') % str(node_count))
file_list = node.get('file_list')
read_list = []
write_list = []
other_list = []
if file_list:
for f in file_list:
channel = _create_channel(
f,
zvm_node,
default_content_type=self.default_content_type)
if is_zvm_path(channel.path):
_add_connected_device(connect_devices, channel,
zvm_node)
continue
if channel.access < 0:
if self.is_sysimage_device(channel.device):
other_list.append(channel)
continue
raise ClusterConfigParsingError(
_('Unknown device %s in %s') %
(channel.device, zvm_node.name))
if channel.access & ACCESS_READABLE:
read_list.insert(0, channel)
elif channel.access & ACCESS_CDR:
read_list.append(channel)
elif channel.access & ACCESS_WRITABLE:
write_list.append(channel)
else:
other_list.append(channel)
read_group = False
for chan in read_list:
if '*' in chan.path.path:
read_group = True
object_list = self.find_objects(
chan.path, **kwargs)
read_mask = re.escape(chan.path.path).replace(
'\\*', '(.*)')
read_mask = re.compile(read_mask)
node_count = len(object_list)
for i in range(node_count):
new_path = object_list[i]
new_node = self._add_new_channel(zvm_node,
chan,
index=(i + 1),
path=new_path)
new_node.store_wildcards(new_path, read_mask)
else:
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node,
chan,
index=i)
else:
self._add_new_channel(zvm_node, chan)
for chan in write_list:
if chan.path and '*' in chan.path.url:
if read_group:
for i in range(1, node_count + 1):
new_node = self.nodes.get(
_create_node_name(zvm_node.name, i))
new_url = _extract_stored_wildcards(
chan.path, new_node)
new_node.add_channel(
channel=chan,
path=parse_location(new_url))
else:
for i in range(1, node_count + 1):
new_name = _create_node_name(
zvm_node.name, i)
new_url = chan.path.url.replace(
'*', new_name)
new_node = self._add_new_channel(
zvm_node,
chan,
index=i,
path=parse_location(new_url))
new_node.wildcards = [
new_name
] * chan.path.url.count('*')
elif chan.path:
if node_count > 1:
raise ClusterConfigParsingError(
_('Single path %s for multiple node '
'definition: %s, please use wildcard') %
(chan.path.url, zvm_node.name))
self._add_new_channel(zvm_node, chan)
else:
if 'stdout' not in chan.device \
and 'stderr' not in chan.device:
raise ClusterConfigParsingError(
_('Immediate response is not available '
'for device %s') % chan.device)
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node,
chan,
index=i)
else:
self._add_new_channel(zvm_node, chan)
for chan in other_list:
if self.is_sysimage_device(chan.device):
chan.access = ACCESS_RANDOM | ACCESS_READABLE
else:
if not chan.path:
raise ClusterConfigParsingError(
_('Path is required for device: %s') %
chan.device)
if node_count > 1:
for i in range(1, node_count + 1):
self._add_new_channel(zvm_node, chan, index=i)
else:
self._add_new_channel(zvm_node, chan)
except ClusterConfigParsingError:
raise
except Exception:
print traceback.format_exc()
raise ClusterConfigParsingError('Config parser internal error')
for node in cluster_config:
connection_list = node.get('connect')
node_name = node.get('name')
src_devices = connect_devices.get(node_name, None)
if not connection_list:
if src_devices:
connection_list = [
connected_node
for connected_node in src_devices.iterkeys()
]
else:
continue
self._add_all_connections(node_name, connection_list, src_devices)
for node_name in sorted(self.nodes.keys()):
self.node_list.append(self.nodes[node_name])
if add_user_image:
for node in self.node_list:
node.add_new_channel('image', ACCESS_CDR, removable='yes')
if account_name:
self.resolve_path_info(account_name, replica_count)
self.total_count = 0
for n in self.node_list:
self.total_count += n.replicate