Exemple #1
0
def _create_channel(channel, node, default_content_type=None):
    device = channel.get('device')
    if has_control_chars(device):
        raise ClusterConfigParsingError(
            _('Bad device name: %s in %s') % (device, node.name))
    path = parse_location(channel.get('path'))
    if not device:
        raise ClusterConfigParsingError(
            _('Must specify device for file in %s') % node.name)
    access = DEVICE_MAP.get(device, -1)
    mode = channel.get('mode', None)
    meta = channel.get('meta', {})
    content_type = channel.get('content_type',
                               default_content_type if path else 'text/html')
    if access & ACCESS_READABLE and path:
        if not is_swift_path(path):
            raise ClusterConfigParsingError(
                _('Readable device must be a swift object'))
        if not path.account or not path.container:
            raise ClusterConfigParsingError(
                _('Invalid path %s in %s') % (path.url, node.name))
    return ZvmChannel(device,
                      access,
                      path=path,
                      content_type=content_type,
                      meta_data=meta,
                      mode=mode)
Exemple #2
0
def _create_channel(channel, node, default_content_type=None):
    device = channel.get('device')
    if has_control_chars(device):
        raise ClusterConfigParsingError(_('Bad device name: %s in %s') % (device, node.name))
    path = parse_location(channel.get('path'))
    if not device:
        raise ClusterConfigParsingError(_('Must specify device for file in %s') % node.name)
    access = DEVICE_MAP.get(device, -1)
    mode = channel.get('mode', None)
    meta = channel.get('meta', {})
    content_type = channel.get('content_type', default_content_type if path else 'text/html')
    if access & ACCESS_READABLE and path:
        if not is_swift_path(path):
            raise ClusterConfigParsingError(_('Readable device must be a swift object'))
        if not path.account or not path.container:
            raise ClusterConfigParsingError(_('Invalid path %s in %s')
                                            % (path.url, node.name))
    return ZvmChannel(device, access, path=path,
                      content_type=content_type, meta_data=meta, mode=mode)
Exemple #3
0
def _create_node(node_config):
    name = node_config.get('name')
    if not name:
        raise ClusterConfigParsingError(_('Must specify node name'))
    if has_control_chars(name):
        raise ClusterConfigParsingError(_('Invalid node name'))
    nexe = node_config.get('exec')
    if not nexe:
        raise ClusterConfigParsingError(_('Must specify exec stanza for %s') % name)
    exe = parse_location(nexe.get('path'))
    if not exe:
        raise ClusterConfigParsingError(_('Must specify executable path for %s') % name)
    if is_zvm_path(exe):
        raise ClusterConfigParsingError(_('Executable path cannot be a zvm path in %s') % name)
    args = nexe.get('args')
    env = nexe.get('env')
    if has_control_chars('%s %s %s' % (exe.url, args, env)):
        raise ClusterConfigParsingError(_('Invalid nexe property for %s') % name)
    replicate = node_config.get('replicate', 1)
    return ZvmNode(0, name, exe, args, env, replicate)
Exemple #4
0
def _create_node(node_config):
    name = node_config.get('name')
    if not name:
        raise ClusterConfigParsingError(_('Must specify node name'))
    if has_control_chars(name):
        raise ClusterConfigParsingError(_('Invalid node name'))
    nexe = node_config.get('exec')
    if not nexe:
        raise ClusterConfigParsingError(
            _('Must specify exec stanza for %s') % name)
    exe = parse_location(nexe.get('path'))
    if not exe:
        raise ClusterConfigParsingError(
            _('Must specify executable path for %s') % name)
    if is_zvm_path(exe):
        raise ClusterConfigParsingError(
            _('Executable path cannot be a zvm path in %s') % name)
    args = nexe.get('args')
    env = nexe.get('env')
    if has_control_chars('%s %s %s' % (exe.url, args, env)):
        raise ClusterConfigParsingError(
            _('Invalid nexe property for %s') % name)
    replicate = node_config.get('replicate', 1)
    return ZvmNode(0, name, exe, args, env, replicate)
Exemple #5
0
    def parse(self, cluster_config, add_user_image, account_name=None, replica_count=1, **kwargs):
        """
        Parse deserialized config and build separate job configs per node

        :param cluster_config: deserialized JSON cluster map
        :param add_user_image: True if we need to add user image channel to all nodes
        :param **kwargs: optional arguments for list_container, list_account callbacks

        :raises ClusterConfigParsingError: on all errors
        """
        self.nodes = {}
        self.node_id = 1
        self.node_list = []
        try:
            connect_devices = {}
            for node in cluster_config:
                zvm_node = _create_node(node)
                node_count = node.get('count', 1)
                if isinstance(node_count, int) and node_count > 0:
                    pass
                else:
                    raise ClusterConfigParsingError(_('Invalid node count: %s') % str(node_count))
                file_list = node.get('file_list')
                read_list = []
                write_list = []
                other_list = []

                if file_list:
                    for f in file_list:
                        channel = _create_channel(
                            f, zvm_node,
                            default_content_type=self.default_content_type)
                        if is_zvm_path(channel.path):
                            _add_connected_device(connect_devices, channel, zvm_node)
                            continue
                        if channel.access < 0:
                            if self.is_sysimage_device(channel.device):
                                other_list.append(channel)
                                continue
                            raise ClusterConfigParsingError(_('Unknown device %s in %s')
                                                            % (channel.device, zvm_node.name))
                        if channel.access & ACCESS_READABLE:
                            read_list.insert(0, channel)
                        elif channel.access & ACCESS_CDR:
                            read_list.append(channel)
                        elif channel.access & ACCESS_WRITABLE:
                            write_list.append(channel)
                        else:
                            other_list.append(channel)

                    read_group = False
                    for chan in read_list:
                        if '*' in chan.path.path:
                            read_group = True
                            object_list = self.find_objects(chan.path, **kwargs)
                            read_mask = re.escape(chan.path.path).replace('\\*', '(.*)')
                            read_mask = re.compile(read_mask)
                            node_count = len(object_list)
                            for i in range(node_count):
                                new_path = object_list[i]
                                new_node = self._add_new_channel(zvm_node, chan, index=(i + 1), path=new_path)
                                new_node.store_wildcards(new_path, read_mask)
                        else:
                            if node_count > 1:
                                for i in range(1, node_count + 1):
                                    self._add_new_channel(zvm_node, chan, index=i)
                            else:
                                self._add_new_channel(zvm_node, chan)

                    for chan in write_list:
                        if chan.path and '*' in chan.path.url:
                            if read_group:
                                for i in range(1, node_count + 1):
                                    new_node = self.nodes.get(_create_node_name(zvm_node.name, i))
                                    new_url = _extract_stored_wildcards(chan.path, new_node)
                                    new_node.add_channel(channel=chan,
                                                         path=parse_location(new_url))
                            else:
                                for i in range(1, node_count + 1):
                                    new_name = _create_node_name(zvm_node.name, i)
                                    new_url = chan.path.url.replace('*', new_name)
                                    new_node = self._add_new_channel(zvm_node, chan, index=i,
                                                                     path=parse_location(new_url))
                                    new_node.wildcards = [new_name] * chan.path.url.count('*')
                        elif chan.path:
                            if node_count > 1:
                                raise ClusterConfigParsingError(_('Single path %s for multiple node '
                                                                  'definition: %s, please use wildcard')
                                                                % (chan.path.url, zvm_node.name))
                            self._add_new_channel(zvm_node, chan)
                        else:
                            if 'stdout' not in chan.device \
                                and 'stderr' not in chan.device:
                                raise ClusterConfigParsingError(_('Immediate response is not available '
                                                                  'for device %s') % chan.device)
                            if node_count > 1:
                                for i in range(1, node_count + 1):
                                    self._add_new_channel(zvm_node, chan, index=i)
                            else:
                                self._add_new_channel(zvm_node, chan)
                    for chan in other_list:
                        if self.is_sysimage_device(chan.device):
                            chan.access = ACCESS_RANDOM | ACCESS_READABLE
                        else:
                            if not chan.path:
                                raise ClusterConfigParsingError(_('Path is required for device: %s') % chan.device)
                        if node_count > 1:
                            for i in range(1, node_count + 1):
                                self._add_new_channel(zvm_node, chan, index=i)
                        else:
                            self._add_new_channel(zvm_node, chan)
        except ClusterConfigParsingError:
            raise
        except Exception:
            print traceback.format_exc()
            raise ClusterConfigParsingError('Config parser internal error')

        for node in cluster_config:
            connection_list = node.get('connect')
            node_name = node.get('name')
            src_devices = connect_devices.get(node_name, None)
            if not connection_list:
                if src_devices:
                    connection_list = [connected_node for connected_node in src_devices.iterkeys()]
                else:
                    continue
            self._add_all_connections(node_name, connection_list, src_devices)
        for node_name in sorted(self.nodes.keys()):
            self.node_list.append(self.nodes[node_name])
        if add_user_image:
            for node in self.node_list:
                    node.add_new_channel('image', ACCESS_CDR, removable='yes')
        if account_name:
            self.resolve_path_info(account_name, replica_count)
        self.total_count = 0
        for n in self.node_list:
            self.total_count += n.replicate
Exemple #6
0
    def parse(self,
              cluster_config,
              add_user_image,
              account_name=None,
              replica_count=1,
              **kwargs):
        """
        Parse deserialized config and build separate job configs per node

        :param cluster_config: deserialized JSON cluster map
        :param add_user_image: True if we need to add user image channel to all nodes
        :param **kwargs: optional arguments for list_container, list_account callbacks

        :raises ClusterConfigParsingError: on all errors
        """
        self.nodes = {}
        self.node_id = 1
        self.node_list = []
        try:
            connect_devices = {}
            for node in cluster_config:
                zvm_node = _create_node(node)
                node_count = node.get('count', 1)
                if isinstance(node_count, int) and node_count > 0:
                    pass
                else:
                    raise ClusterConfigParsingError(
                        _('Invalid node count: %s') % str(node_count))
                file_list = node.get('file_list')
                read_list = []
                write_list = []
                other_list = []

                if file_list:
                    for f in file_list:
                        channel = _create_channel(
                            f,
                            zvm_node,
                            default_content_type=self.default_content_type)
                        if is_zvm_path(channel.path):
                            _add_connected_device(connect_devices, channel,
                                                  zvm_node)
                            continue
                        if channel.access < 0:
                            if self.is_sysimage_device(channel.device):
                                other_list.append(channel)
                                continue
                            raise ClusterConfigParsingError(
                                _('Unknown device %s in %s') %
                                (channel.device, zvm_node.name))
                        if channel.access & ACCESS_READABLE:
                            read_list.insert(0, channel)
                        elif channel.access & ACCESS_CDR:
                            read_list.append(channel)
                        elif channel.access & ACCESS_WRITABLE:
                            write_list.append(channel)
                        else:
                            other_list.append(channel)

                    read_group = False
                    for chan in read_list:
                        if '*' in chan.path.path:
                            read_group = True
                            object_list = self.find_objects(
                                chan.path, **kwargs)
                            read_mask = re.escape(chan.path.path).replace(
                                '\\*', '(.*)')
                            read_mask = re.compile(read_mask)
                            node_count = len(object_list)
                            for i in range(node_count):
                                new_path = object_list[i]
                                new_node = self._add_new_channel(zvm_node,
                                                                 chan,
                                                                 index=(i + 1),
                                                                 path=new_path)
                                new_node.store_wildcards(new_path, read_mask)
                        else:
                            if node_count > 1:
                                for i in range(1, node_count + 1):
                                    self._add_new_channel(zvm_node,
                                                          chan,
                                                          index=i)
                            else:
                                self._add_new_channel(zvm_node, chan)

                    for chan in write_list:
                        if chan.path and '*' in chan.path.url:
                            if read_group:
                                for i in range(1, node_count + 1):
                                    new_node = self.nodes.get(
                                        _create_node_name(zvm_node.name, i))
                                    new_url = _extract_stored_wildcards(
                                        chan.path, new_node)
                                    new_node.add_channel(
                                        channel=chan,
                                        path=parse_location(new_url))
                            else:
                                for i in range(1, node_count + 1):
                                    new_name = _create_node_name(
                                        zvm_node.name, i)
                                    new_url = chan.path.url.replace(
                                        '*', new_name)
                                    new_node = self._add_new_channel(
                                        zvm_node,
                                        chan,
                                        index=i,
                                        path=parse_location(new_url))
                                    new_node.wildcards = [
                                        new_name
                                    ] * chan.path.url.count('*')
                        elif chan.path:
                            if node_count > 1:
                                raise ClusterConfigParsingError(
                                    _('Single path %s for multiple node '
                                      'definition: %s, please use wildcard') %
                                    (chan.path.url, zvm_node.name))
                            self._add_new_channel(zvm_node, chan)
                        else:
                            if 'stdout' not in chan.device \
                                and 'stderr' not in chan.device:
                                raise ClusterConfigParsingError(
                                    _('Immediate response is not available '
                                      'for device %s') % chan.device)
                            if node_count > 1:
                                for i in range(1, node_count + 1):
                                    self._add_new_channel(zvm_node,
                                                          chan,
                                                          index=i)
                            else:
                                self._add_new_channel(zvm_node, chan)
                    for chan in other_list:
                        if self.is_sysimage_device(chan.device):
                            chan.access = ACCESS_RANDOM | ACCESS_READABLE
                        else:
                            if not chan.path:
                                raise ClusterConfigParsingError(
                                    _('Path is required for device: %s') %
                                    chan.device)
                        if node_count > 1:
                            for i in range(1, node_count + 1):
                                self._add_new_channel(zvm_node, chan, index=i)
                        else:
                            self._add_new_channel(zvm_node, chan)
        except ClusterConfigParsingError:
            raise
        except Exception:
            print traceback.format_exc()
            raise ClusterConfigParsingError('Config parser internal error')

        for node in cluster_config:
            connection_list = node.get('connect')
            node_name = node.get('name')
            src_devices = connect_devices.get(node_name, None)
            if not connection_list:
                if src_devices:
                    connection_list = [
                        connected_node
                        for connected_node in src_devices.iterkeys()
                    ]
                else:
                    continue
            self._add_all_connections(node_name, connection_list, src_devices)
        for node_name in sorted(self.nodes.keys()):
            self.node_list.append(self.nodes[node_name])
        if add_user_image:
            for node in self.node_list:
                node.add_new_channel('image', ACCESS_CDR, removable='yes')
        if account_name:
            self.resolve_path_info(account_name, replica_count)
        self.total_count = 0
        for n in self.node_list:
            self.total_count += n.replicate