コード例 #1
0
 def ensure_dict(self, data, key):
     """
     Ensure that data is a dict.
     """
     if data is None:
         raise SocketValidationError('Missing value for "{}".'.format(key))
     if not isinstance(data, dict):
         raise SocketValidationError('Wrong format for "{}". Expected object.'.format(key), data.line)
コード例 #2
0
 def ensure_string(self, data, key, optional=False):
     """
     Ensure that data is a string.
     """
     if optional and data is None:
         return
     elif data is None:
         raise SocketValidationError('Missing value for "{}".'.format(key))
     if not isinstance(data, str):
         raise SocketValidationError('Wrong format for "{}". Expected string.'.format(key), data.line)
コード例 #3
0
    def process_script_dependency(self, name, spec, path, allow_empty=False):  # noqa: C901
        """
        Process script dependency from YAML. Returns dependency dict.
        """
        dependency = {'type': 'script', 'config': {'allow_full_access': True}, 'path': '<YAML:{}>'.format(path),
                      'runtime_name': self.runtime['name'], 'line': name.line}

        if isinstance(spec, str):
            dependency['source'] = spec
        else:
            self.ensure_dict(spec, 'script')
            dependency['config']['timeout'] = spec.get('timeout')

            self.validate_min_max_value(spec, 'cache', 0, settings.SOCKETS_MAX_CACHE_TIME, lineno=name.line)
            self.validate_min_max_value(spec, 'timeout', 0, settings.SOCKETS_MAX_TIMEOUT, lineno=name.line)

            if 'source' in spec:
                dependency['source'] = spec.pop('source')
            else:
                # Fallback to endpoint name + extension
                file_path = spec.pop('file', None)
                if file_path is not None:
                    allow_empty = False
                    self.ensure_string(file_path, 'file')
                    if len(file_path) > self.max_path_length:
                        raise SocketValidationError('Source file path is too long.', name.line)
                    if not VALID_PATH_REGEX.match(file_path):
                        raise SocketValidationError('Source file path contains invalid characters.', name.line)
                else:
                    file_path = '{}.{}'.format(path[path.find('/') + 1:], self.runtime['ext'])

                dependency['path'] = file_path
                self.files_processed.add(file_path)
                try:
                    dependency['source'] = self.zip_handler.read_file(file_path)
                except SocketMissingFile:
                    # If we are dealing with partial update and file in path is already installed - skip it
                    # and use old checksum
                    if file_path not in self.socket.file_list:
                        if allow_empty:
                            return None
                        raise
                    file_info = self.socket.file_list[file_path]
                    # If file was a helper before, remove helper=True flag
                    if 'helper' in file_info:
                        del file_info['helper']

                    dependency['checksum'] = file_info['checksum']
                    return dependency

        dependency['source'] = force_text(dependency['source'], errors='ignore')
        dependency['checksum'] = md5(force_bytes(dependency['source'], errors='ignore')).hexdigest()

        self.add_size(len(dependency['source']) - self.socket.file_list.get(dependency['path'], {}).get('size', 0))
        return dependency
コード例 #4
0
    def map_script_endpoint(self, name, spec):
        endpoint = {'name': name, 'type': 'endpoint', 'acl': {}, 'calls': []}
        dependencies = []

        try:
            script_dep = self.process_script_dependency(
                name, spec, path='endpoints/{}'.format(name), allow_empty=True)
        except SocketValidationError as ex:
            raise SocketValidationError(
                'Endpoint "{name}": {message}'.format(name=name,
                                                      message=str(ex)),
                ex.lineno)

        if isinstance(spec, dict):
            endpoint['acl'] = spec.pop('acl', {})
            # Remaining info in endpoint spec are definitions per http method or metadata
            defined_methods = set(spec.keys())
        else:
            defined_methods = set()

        # Check if default script dependency is defined
        if script_dep:
            dependencies.append(script_dep)

            unused_methods = self.http_methods - defined_methods
            if unused_methods == self.http_methods:
                unused_methods = ['*']

            default = {
                'type': 'script',
                'path': script_dep['path'],
                'methods': list(unused_methods),
            }

            self.map_endpoint_settings(default, spec)
            endpoint['calls'].append(default)

        elif not defined_methods.intersection(self.http_methods):
            raise SocketValidationError(
                'No calls defined for endpoint: "{}".'.format(name), name.line)

        # Process per key specs (per HTTP method) if any are defined
        metadata = {}
        if isinstance(spec, dict):
            calls, deps, metadata = self.map_script_endpoint_method(name, spec)
            endpoint['calls'] += calls
            dependencies += deps
        return endpoint, dependencies, metadata
コード例 #5
0
    def process_socket(self):
        """
        Process socket that we have initialized with and return endpoints along with dependencies.

        :returns: dependencies, is_partial=False
        """
        socket = self.socket
        socket.size = sum(f['size'] for f in socket.file_list.values())
        previous_socket_spec = socket.file_list.get(settings.SOCKETS_YAML)

        try:
            socket_spec_raw = self.zip_handler.get_socket_spec()
        except SocketMissingFile:
            # If socket is missing yaml but it is already installed - process partial update
            if previous_socket_spec:
                return self.partial_process_socket(), True
            raise

        socket_spec_raw = force_bytes(socket_spec_raw)
        socket_spec_checksum = md5(socket_spec_raw).hexdigest()

        self.add_size(len(socket_spec_raw) - self.socket.file_list.get(settings.SOCKETS_YAML, {}).get('size', 0))

        # Socket YAML is the main default dependency
        dependencies = [{'type': 'spec', 'source': socket_spec_raw, 'checksum': socket_spec_checksum}]
        dependency_proc = (
            ('endpoints', self.process_endpoint_dependency),
            ('classes', self.process_class_dependency),
            ('hosting', self.process_hosting_dependency),
            ('event_handlers', self.process_event_handlers_dependency)
        )

        # Load socket YAML and validate
        socket_spec = self.load_socket_spec(socket_spec_raw)
        self.validate_socket_spec(socket_spec)

        data = {
            'description': socket_spec.pop('description', ''),
            'version': self.process_version(socket_spec.pop('version', None))
        }
        self.process_runtime(socket_spec.pop('runtime', None))

        # Process all installable dependencies
        for dep_key, dep_func in dependency_proc:
            if dep_key in socket_spec:
                dependencies += dep_func(socket_spec.pop(dep_key))
        dependencies += self.process_helpers()

        # Put remaining info as metadata.
        data['metadata'] = socket_spec
        serializer = SocketSerializer(socket, data=data, partial=True)

        if not serializer.is_valid():
            raise SocketValidationError('Field {}'.format(format_error(serializer.errors)))

        data.update(serializer.validated_data)
        for attr, value in data.items():
            setattr(socket, attr, value)

        return dependencies, False
コード例 #6
0
    def process_event_handlers_dependency(self, handlers_spec):
        """
        Process event handlers section dependency from YAML. Returns dependency list.
        """
        self.ensure_dict(handlers_spec, 'event_handlers')
        event_handlers = []
        for eh_name, script_spec in handlers_spec.items():
            eh_dep = self.parse_event_handler_name(eh_name)

            try:
                script_dep = self.process_script_dependency(eh_name, script_spec,
                                                            path='event_handlers/{}'.format(eh_name))
            except SocketValidationError as ex:
                raise SocketValidationError(
                    'Event handler "{name}": {message}'.format(
                        name=eh_name,
                        message=str(ex)),
                    ex.lineno
                )

            script_dep.update(eh_dep)

            metadata = {}
            # Put remaining keys into metadata
            if isinstance(script_spec, dict):
                metadata = script_spec
            script_dep['metadata'] = metadata
            event_handlers.append(script_dep)
        return event_handlers
コード例 #7
0
    def validate_common_settings(self, spec, lineno=None):
        self.validate_min_max_value(spec,
                                    'cache',
                                    0.0,
                                    settings.SOCKETS_MAX_CACHE_TIME,
                                    lineno=lineno)
        self.validate_min_max_value(spec,
                                    'timeout',
                                    0.0,
                                    settings.SOCKETS_MAX_TIMEOUT,
                                    lineno=lineno)
        self.validate_min_max_value(spec,
                                    'async',
                                    0,
                                    settings.SOCKETS_MAX_ASYNC,
                                    lineno=lineno)
        self.validate_min_max_value(spec,
                                    'mcpu',
                                    0,
                                    settings.SOCKETS_MAX_MCPU,
                                    lineno=lineno)

        if ('async' in spec or 'mcpu' in spec) and not self.is_trusted:
            raise SocketValidationError(
                'Cannot set Async/MCPU on this account. Contact administrator.',
                lineno=lineno)
コード例 #8
0
    def validate_socket_spec(self, socket_spec):
        """
        Do some initial validation on socket spec.
        """

        self.ensure_dict(socket_spec, 'socket')
        self.ensure_string(socket_spec.get('description'), 'description', optional=True)
        if len(socket_spec) > self.max_number_of_keys:
            raise SocketValidationError('Too many properties defined.')
コード例 #9
0
 def process_version(self, version):
     """
     Process and validate version from YAML.
     """
     version_str = version or self.default_version
     if not isinstance(version, str):
         version_str = str(version_str)
     if not VERSION_REGEX.match(version_str):
         raise SocketValidationError('Incorrect version value.', getattr(version, 'line', None))
     return version_str
コード例 #10
0
    def map_channel_endpoint(self, name, spec):
        channel = spec.pop('channel')

        self.ensure_string(channel, 'channel')
        if not CHANNEL_REGEX.match(channel):
            raise SocketValidationError('Wrong format for channel of endpoint: "{}".'.format(name), channel.line)

        call = {'type': 'channel', 'channel': channel, 'methods': ['GET']}
        self.map_endpoint_settings(call, spec)
        return {'name': name, 'type': 'endpoint', 'acl': {}, 'calls': [call]}
コード例 #11
0
    def parse_event_handler_name(self, name):  # noqa: C901
        """
        Parse and validate event handler name. Return event handler dependency created from it.
        """
        self.ensure_string(name, 'name')
        eh_parts = name.split('.', 3)
        eh_type = eh_parts[0]
        eh_dep = {
            'type': 'event_handler_{}'.format(eh_type),
            'handler_name': name
        }
        lineno = name.line

        if eh_type == 'data':
            # Parse data.* event_handlers. E.g. data.user.create
            if len(eh_parts) != 3:
                raise SocketValidationError(
                    'Wrong format for data event handler.', lineno)
            eh_dep['class'] = eh_parts[1]
            eh_dep['signal'] = eh_parts[2]

        elif eh_type == 'schedule':
            # Parse schedule.* event_handlers. E.g. schedule.interval.5_minutes and schedule.crontab.*/5 * * * *
            if len(eh_parts) != 3:
                raise SocketValidationError(
                    'Wrong format for schedule event handler.', lineno)
            schedule_format = eh_parts[1]
            if schedule_format == 'crontab':
                eh_dep['crontab'] = eh_parts[2]
            elif schedule_format == 'interval':
                interval_match = INTERVAL_REGEX.match(eh_parts[2])
                if not interval_match:
                    raise SocketValidationError(
                        'Wrong format for schedule interval.', lineno)
                interval_dict = interval_match.groupdict(0)
                eh_dep['interval'] = int(interval_dict['hours']) * 60 * 60 + int(interval_dict['minutes']) * 60 + \
                    int(interval_dict['seconds'])
            else:
                raise SocketValidationError(
                    'Wrong type of schedule event handler.', lineno)

        elif eh_type == 'events':
            # Parse events.* event_handlers. E.g. events.data_processed, events.socket1.data_processed
            if len(eh_parts) == 2:
                # Prefix signal name if necessary
                eh_dep['signal'] = '{}.{}'.format(self.socket.name,
                                                  eh_parts[1])
                eh_dep['handler_name'] = 'events.{}'.format(eh_dep['signal'])
            elif len(eh_parts) == 3:
                eh_dep['signal'] = '{}.{}'.format(eh_parts[1], eh_parts[2])
            else:
                raise SocketValidationError('Wrong format for event handler.',
                                            lineno)
        else:
            raise SocketValidationError(
                'Unsupported event handler type: "{}".'.format(eh_type),
                lineno)
        return eh_dep
コード例 #12
0
    def validate_min_max_value(self, spec, key, min_val, max_val, lineno=None):
        if key not in spec or spec[key] is None:
            return

        try:
            v = float(spec[key])
            if v <= min_val or v > max_val:
                raise ValueError
            spec[key] = v
        except (ValueError, KeyError, TypeError):
            raise SocketValidationError(
                'Invalid {key} value. Must be higher than {min} and lower than or equal to {max}.'.format(
                    key=key, min=min_val, max=max_val,
                ), lineno=lineno)
コード例 #13
0
    def map_script_endpoint_method(self, name, spec):
        """
        Map YAML endpoint methods spec to endpoint calls and dependencies.
        Returns endpoint calls, dependencies lists and metadata info.
        """
        dependencies = []
        calls = []
        metadata = {}
        for method in list(spec.keys()):
            if method not in self.http_methods:
                continue

            method_spec = spec.pop(method)

            try:
                script_dep = self.process_script_dependency(method, method_spec,
                                                            path='endpoints/{}/{}'.format(name, method))
            except SocketValidationError as ex:
                raise SocketValidationError(
                    'Endpoint "{name}", method: "{method}": {message}'.format(
                        name=name,
                        method=method,
                        message=str(ex)),
                    ex.lineno
                )
            dependencies.append(script_dep)
            call = {
                'type': 'script',
                'path': script_dep['path'],
                'runtime': self.runtime['name'],
                'methods': [method],
            }

            # Use global spec first and override with method_spec settings
            self.map_endpoint_settings(call, spec)
            self.map_endpoint_settings(call, method_spec)
            calls.append(call)

            # Add rest to metadata
            if isinstance(method_spec, dict) and method_spec:
                metadata[method] = method_spec
        return calls, dependencies, metadata
コード例 #14
0
    def process_runtime(self, runtime):
        """
        Process and validate script runtime from YAML.
        """
        possible_runtimes = self.possible_runtimes
        if not self.socket.is_new_format:
            possible_runtimes = self.old_possible_runtimes

        # Default runtime is the first possible.
        self.runtime = possible_runtimes[0][1]

        if not runtime:
            return

        self.ensure_string(runtime, 'runtime')
        for alias, runtime_data in possible_runtimes:
            if runtime in alias:
                self.runtime = runtime_data
                return
        raise SocketValidationError('Incorrect runtime value.', runtime.line)