def check_command_type_struct_field( ctxt: IDLCompatibilityContext, type_name: str, old_field: syntax.Field, new_field: syntax.Field, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new type struct field.""" # pylint: disable=too-many-arguments if new_field.unstable: ctxt.add_new_command_type_field_unstable_error(cmd_name, type_name, new_field.name, new_idl_file_path) if old_field.optional and not new_field.optional: ctxt.add_new_command_type_field_required_error(cmd_name, type_name, new_field.name, new_idl_file_path) check_param_or_type_validator(ctxt, old_field, new_field, cmd_name, new_idl_file_path, type_name, is_command_parameter=False) old_field_type = get_field_type(old_field, old_idl_file, old_idl_file_path) new_field_type = get_field_type(new_field, new_idl_file, new_idl_file_path) check_command_type(ctxt, old_field_type, new_field_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path)
def check_command_params_or_type_struct_fields( ctxt: IDLCompatibilityContext, old_struct: syntax.Struct, new_struct: syntax.Struct, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str, is_command_parameter: bool): """Check compatibility between old and new parameters or command type fields.""" # pylint: disable=too-many-arguments for old_field in old_struct.fields or []: new_field_exists = False for new_field in new_struct.fields or []: if new_field.name == old_field.name: new_field_exists = True check_command_param_or_type_struct_field( ctxt, old_field, new_field, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, old_struct.name, is_command_parameter) break if not new_field_exists and not old_field.unstable: ctxt.add_new_param_or_command_type_field_missing_error( cmd_name, old_field.name, old_idl_file_path, old_struct.name, is_command_parameter) # Check if a new field has been added to the parameters or type struct. # If so, it must be optional. for new_field in new_struct.fields or []: newly_added = True for old_field in old_struct.fields or []: if new_field.name == old_field.name: newly_added = False if newly_added and not new_field.optional and not new_field.unstable: ctxt.add_new_param_or_command_type_field_added_required_error( cmd_name, new_field.name, new_idl_file_path, new_struct.name, is_command_parameter)
def check_subset(ctxt: IDLCompatibilityContext, cmd_name: str, field_name: str, type_name: str, sub_list: List[Union[str, syntax.EnumValue]], super_list: List[Union[str, syntax.EnumValue]], file_path: str): # pylint: disable=too-many-arguments """Check if sub_list is a subset of the super_list and log an error if not.""" if not set(sub_list).issubset(super_list): ctxt.add_command_not_subset_error(cmd_name, field_name, type_name, file_path)
def check_command_type_struct_fields( ctxt: IDLCompatibilityContext, old_type: syntax.Struct, new_type: syntax.Struct, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new type fields.""" # pylint: disable=too-many-arguments for old_field in old_type.fields or []: if old_field.unstable: continue new_field_exists = False for new_field in new_type.fields or []: if new_field.name == old_field.name: new_field_exists = True check_command_type_struct_field(ctxt, old_type.name, old_field, new_field, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) break if not new_field_exists: ctxt.add_new_command_type_field_missing_error( cmd_name, old_type.name, old_field.name, old_idl_file_path)
def check_command_parameters(ctxt: IDLCompatibilityContext, old_cmd: syntax.Command, new_cmd: syntax.Command, cmd_name: str, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new command parameters.""" # pylint: disable=too-many-arguments for old_param in old_cmd.fields: new_param_exists = False for new_param in new_cmd.fields: if new_param.name == old_param.name: new_param_exists = True check_command_parameter(ctxt, old_param, new_param, cmd_name, old_idl_file_path) break if not new_param_exists and not old_param.unstable: ctxt.add_command_parameter_removed_error(old_cmd.command_name, old_param.name, old_idl_file_path) # Check if a new parameter has been added to the command. # If so, it must be optional. for new_param in new_cmd.fields: newly_added = True for old_param in old_cmd.fields: if new_param.name == old_param.name: newly_added = False if newly_added and not new_param.optional and not new_param.unstable: ctxt.add_new_command_parameter_required_error(new_cmd.name, new_param.name, new_idl_file_path)
def check_param_or_command_type( ctxt: IDLCompatibilityContext, old_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], new_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str, param_name: Optional[str], is_command_parameter: bool): """Check compatibility between old and new command parameter type or command type.""" # pylint: disable=too-many-arguments,too-many-branches if check_array_type( ctxt, "command_parameter" if is_command_parameter else "command_namespace", old_type, new_type, cmd_name, param_name if is_command_parameter else "type", old_idl_file_path, new_idl_file_path): old_type = old_type.element_type new_type = new_type.element_type if old_type is None: ctxt.add_command_or_param_type_invalid_error(cmd_name, old_idl_file_path, param_name, is_command_parameter) ctxt.errors.dump_errors() sys.exit(1) if new_type is None: ctxt.add_command_or_param_type_invalid_error(cmd_name, new_idl_file_path, param_name, is_command_parameter) ctxt.errors.dump_errors() sys.exit(1) if isinstance(old_type, syntax.Type): check_param_or_command_type_recursive(ctxt, old_type, new_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, param_name, is_command_parameter) elif isinstance(old_type, syntax.Enum): if isinstance(new_type, syntax.Enum): check_superset(ctxt, cmd_name, new_type.name, new_type.values, old_type.values, new_idl_file_path, param_name, is_command_parameter) else: ctxt.add_new_command_or_param_type_not_enum_error( cmd_name, new_type.name, old_type.name, new_idl_file_path, param_name, is_command_parameter) elif isinstance(old_type, syntax.Struct): if isinstance(new_type, syntax.Struct): check_command_params_or_type_struct_fields( ctxt, old_type, new_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, is_command_parameter) else: ctxt.add_new_command_or_param_type_not_struct_error( cmd_name, new_type.name, old_type.name, new_idl_file_path, param_name, is_command_parameter)
def check_reply_field(ctxt: IDLCompatibilityContext, old_field: syntax.Field, new_field: syntax.Field, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new reply field.""" # pylint: disable=too-many-arguments if new_field.unstable: ctxt.add_new_reply_field_unstable_error(cmd_name, new_field.name, new_idl_file_path) if new_field.optional and not old_field.optional: ctxt.add_new_reply_field_optional_error(cmd_name, new_field.name, new_idl_file_path) if old_field.validator: # Not implemented. ctxt.add_reply_field_contains_validator_error(cmd_name, old_field.name, old_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) if new_field.validator: # Not implemented. ctxt.add_reply_field_contains_validator_error(cmd_name, new_field.name, new_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) old_field_type = get_field_type(old_field, old_idl_file, old_idl_file_path) new_field_type = get_field_type(new_field, new_idl_file, new_idl_file_path) check_reply_field_type(ctxt, old_field_type, new_field_type, cmd_name, old_field.name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path)
def check_reply_field(ctxt: IDLCompatibilityContext, old_field: syntax.Field, new_field: syntax.Field, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new reply field.""" # pylint: disable=too-many-arguments if new_field.unstable: ctxt.add_new_reply_field_unstable_error(cmd_name, new_field.name, new_idl_file_path) if new_field.optional and not old_field.optional: ctxt.add_new_reply_field_optional_error(cmd_name, new_field.name, new_idl_file_path) if new_field.validator: if old_field.validator: if new_field.validator != old_field.validator: ctxt.add_reply_field_validators_not_equal_error( cmd_name, new_field.name, new_idl_file_path) else: ctxt.add_reply_field_contains_validator_error( cmd_name, new_field.name, new_idl_file_path) old_field_type = get_field_type(old_field, old_idl_file, old_idl_file_path) new_field_type = get_field_type(new_field, new_idl_file, new_idl_file_path) if check_array_type(ctxt, "reply_field", old_field_type, new_field_type, cmd_name, 'type', old_idl_file_path, new_idl_file_path): old_field_type = old_field_type.element_type new_field_type = new_field_type.element_type check_reply_field_type(ctxt, old_field_type, new_field_type, cmd_name, old_field.name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path)
def check_param_or_type_validator(ctxt: IDLCompatibilityContext, old_field: syntax.Field, new_field: syntax.Field, cmd_name: str, new_idl_file_path: str, type_name: Optional[str], is_command_parameter: bool): """ Check compatibility between old and new validators. Check compatibility between old and new validators in command parameters and command type struct fields. """ # pylint: disable=too-many-arguments if new_field.validator: if old_field.validator: if new_field.validator != old_field.validator: if is_command_parameter: ctxt.add_command_parameter_validators_not_equal_error( cmd_name, new_field.name, new_idl_file_path) else: ctxt.add_command_type_validators_not_equal_error( cmd_name, type_name, new_field.name, new_idl_file_path) else: if is_command_parameter: ctxt.add_command_parameter_contains_validator_error( cmd_name, new_field.name, new_idl_file_path) else: ctxt.add_command_type_contains_validator_error( cmd_name, type_name, new_field.name, new_idl_file_path)
def get_new_commands( ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: List[str] ) -> Tuple[Dict[str, syntax.Command], Dict[str, syntax.IDLParsedSpec], Dict[ str, str]]: """Get new IDL commands and check validity.""" new_commands: Dict[str, syntax.Command] = dict() new_command_file: Dict[str, syntax.IDLParsedSpec] = dict() new_command_file_path: Dict[str, str] = dict() for dirpath, _, filenames in os.walk(new_idl_dir): for new_filename in filenames: if not new_filename.endswith( '.idl') or new_filename in SKIPPED_FILES: continue new_idl_file_path = os.path.join(dirpath, new_filename) with open(new_idl_file_path) as new_file: new_idl_file = parser.parse( new_file, new_idl_file_path, CompilerImportResolver(import_directories + [new_idl_dir])) if new_idl_file.errors: new_idl_file.errors.dump_errors() raise ValueError(f"Cannot parse {new_idl_file_path}") for new_cmd in new_idl_file.spec.symbols.commands: # Ignore imported commands as they will be processed in their own file. if new_cmd.api_version == "" or new_cmd.imported: continue if new_cmd.api_version != "1": # We're not ready to handle future API versions yet. ctxt.add_command_invalid_api_version_error( new_cmd.command_name, new_cmd.api_version, new_idl_file_path) continue if new_cmd.command_name in new_commands: ctxt.add_duplicate_command_name_error( new_cmd.command_name, new_idl_dir, new_idl_file_path) continue new_commands[new_cmd.command_name] = new_cmd new_command_file[new_cmd.command_name] = new_idl_file new_command_file_path[ new_cmd.command_name] = new_idl_file_path return new_commands, new_command_file, new_command_file_path
def check_compatibility(old_idl_dir: str, new_idl_dir: str, import_directories: List[str]) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new IDL commands.""" # pylint: disable=too-many-locals ctxt = IDLCompatibilityContext(old_idl_dir, new_idl_dir, IDLCompatibilityErrorCollection()) new_commands, new_command_file, new_command_file_path = get_new_commands( ctxt, new_idl_dir, import_directories) # Check new commands' compatibility with old ones. # Note, a command can be added to V1 at any time, it's ok if a # new command has no corresponding old command. old_commands: Dict[str, syntax.Command] = dict() for dirpath, _, filenames in os.walk(old_idl_dir): for old_filename in filenames: if not old_filename.endswith('.idl'): continue old_idl_file_path = os.path.join(dirpath, old_filename) with open(old_idl_file_path) as old_file: old_idl_file = parser.parse( old_file, old_idl_file_path, CompilerImportResolver(import_directories + [old_idl_dir])) if old_idl_file.errors: old_idl_file.errors.dump_errors() raise ValueError(f"Cannot parse {old_idl_file_path}") for old_cmd in old_idl_file.spec.symbols.commands: if old_cmd.api_version == "": continue if old_cmd.api_version != "1": # We're not ready to handle future API versions yet. ctxt.add_command_invalid_api_version_error( old_cmd.command_name, old_cmd.api_version, old_idl_file_path) continue if old_cmd.command_name in old_commands: ctxt.add_duplicate_command_name_error(old_cmd.command_name, old_idl_dir, old_idl_file_path) continue old_commands[old_cmd.command_name] = old_cmd if old_cmd.command_name not in new_commands: # Can't remove a command from V1 ctxt.add_command_removed_error(old_cmd.command_name, old_idl_file_path) continue new_cmd = new_commands[old_cmd.command_name] new_idl_file = new_command_file[old_cmd.command_name] new_idl_file_path = new_command_file_path[old_cmd.command_name] check_reply_fields(ctxt, old_cmd, new_cmd, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) ctxt.errors.dump_errors() return ctxt.errors
def check_namespace(ctxt: IDLCompatibilityContext, old_cmd: syntax.Command, new_cmd: syntax.Command, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new namespace.""" # pylint: disable=too-many-arguments old_namespace = old_cmd.namespace new_namespace = new_cmd.namespace # IDL parser already checks that namespace must be one of these 4 types. if old_namespace == common.COMMAND_NAMESPACE_IGNORED: if new_namespace != common.COMMAND_NAMESPACE_IGNORED: ctxt.add_new_namespace_incompatible_error(old_cmd.command_name, old_namespace, new_namespace, new_idl_file_path) elif old_namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB_OR_UUID: if new_namespace not in ( common.COMMAND_NAMESPACE_IGNORED, common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB_OR_UUID): ctxt.add_new_namespace_incompatible_error(old_cmd.command_name, old_namespace, new_namespace, new_idl_file_path) elif old_namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: if new_namespace == common.COMMAND_NAMESPACE_TYPE: ctxt.add_new_namespace_incompatible_error(old_cmd.command_name, old_namespace, new_namespace, new_idl_file_path) elif old_namespace == common.COMMAND_NAMESPACE_TYPE: old_type = get_field_type(old_cmd, old_idl_file, old_idl_file_path) if new_namespace == common.COMMAND_NAMESPACE_TYPE: new_type = get_field_type(new_cmd, new_idl_file, new_idl_file_path) check_param_or_command_type(ctxt, old_type, new_type, old_cmd.command_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, param_name=None, is_command_parameter=False) # If old type is "namespacestring", the new namespace can be changed to any # of the other namespace types. elif old_type.name != "namespacestring": # Otherwise, the new namespace can only be changed to "ignored". if new_namespace != common.COMMAND_NAMESPACE_IGNORED: ctxt.add_new_namespace_incompatible_error( old_cmd.command_name, old_namespace, new_namespace, new_idl_file_path) else: assert False, 'unrecognized namespace option'
def check_reply_field_type(ctxt: IDLCompatibilityContext, old_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], new_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, field_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new reply field type.""" # pylint: disable=too-many-arguments,too-many-branches if check_array_type(ctxt, "reply_field", old_field_type, new_field_type, cmd_name, 'type', old_idl_file_path, new_idl_file_path): old_field_type = old_field_type.element_type new_field_type = new_field_type.element_type if old_field_type is None: ctxt.add_reply_field_type_invalid_error(cmd_name, field_name, old_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) if new_field_type is None: ctxt.add_reply_field_type_invalid_error(cmd_name, field_name, new_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) if isinstance(old_field_type, syntax.Type): check_reply_field_type_recursive(ctxt, old_field_type, new_field_type, cmd_name, field_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) elif isinstance(old_field_type, syntax.Enum): if isinstance(new_field_type, syntax.Enum): check_subset(ctxt, cmd_name, field_name, new_field_type.name, new_field_type.values, old_field_type.values, new_idl_file_path) else: ctxt.add_new_reply_field_type_not_enum_error( cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path) elif isinstance(old_field_type, syntax.Struct): if isinstance(new_field_type, syntax.Struct): check_reply_fields(ctxt, old_field_type, new_field_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) else: ctxt.add_new_reply_field_type_not_struct_error( cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path)
def check_array_type(ctxt: IDLCompatibilityContext, symbol: str, old_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], new_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, symbol_name: str, old_idl_file_path: str, new_idl_file_path: str) -> bool: """Check compatibility between old and new ArrayTypes.""" # pylint: disable=too-many-arguments,too-many-branches old_is_array = isinstance(old_type, syntax.ArrayType) new_is_array = isinstance(new_type, syntax.ArrayType) if not old_is_array and not new_is_array: return False if not old_is_array or not new_is_array: ctxt.add_type_not_array_error(symbol, cmd_name, symbol_name, new_type.name, old_type.name, new_idl_file_path if old_is_array else old_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) return True
def check_command_param_or_type_struct_field( ctxt: IDLCompatibilityContext, old_field: syntax.Field, new_field: syntax.Field, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str, type_name: Optional[str], is_command_parameter: bool): """Check compatibility between the old and new command parameter or command type struct field.""" # pylint: disable=too-many-arguments if not old_field.unstable and new_field.unstable: ctxt.add_new_param_or_command_type_field_unstable_error( cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter) # If old field is unstable and new field is stable, the new field should either be optional or # have a default value. if old_field.unstable and not new_field.unstable and not new_field.optional and new_field.default is None: ctxt.add_new_param_or_command_type_field_stable_required_no_default_error( cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter) if old_field.optional and not new_field.optional: ctxt.add_new_param_or_command_type_field_required_error( cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter) check_param_or_type_validator(ctxt, old_field, new_field, cmd_name, new_idl_file_path, type_name, is_command_parameter) old_field_type = get_field_type(old_field, old_idl_file, old_idl_file_path) new_field_type = get_field_type(new_field, new_idl_file, new_idl_file_path) check_param_or_command_type(ctxt, old_field_type, new_field_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, old_field.name, is_command_parameter)
def check_command_parameter(ctxt: IDLCompatibilityContext, old_param: syntax.Field, new_param: syntax.Field, cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between the old and new command parameter.""" # pylint: disable=too-many-arguments if not old_param.unstable and new_param.unstable: ctxt.add_command_parameter_unstable_error(cmd_name, old_param.name, old_idl_file_path) if old_param.unstable and not new_param.optional and not new_param.unstable: ctxt.add_command_parameter_stable_required_error( cmd_name, old_param.name, old_idl_file_path) if old_param.optional and not new_param.optional: ctxt.add_command_parameter_required_error(cmd_name, old_param.name, old_idl_file_path) check_param_or_type_validator(ctxt, old_param, new_param, cmd_name, new_idl_file_path, type_name=None, is_command_parameter=True) old_parameter_type = get_field_type(old_param, old_idl_file, old_idl_file_path) new_parameter_type = get_field_type(new_param, new_idl_file, new_idl_file_path) check_command_parameter_type(ctxt, old_parameter_type, new_parameter_type, cmd_name, old_param.name, old_idl_file_path, new_idl_file_path)
def check_error_reply(old_basic_types_path: str, new_basic_types_path: str, import_directories: List[str]) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new ErrorReply.""" old_idl_dir = os.path.dirname(old_basic_types_path) new_idl_dir = os.path.dirname(new_basic_types_path) ctxt = IDLCompatibilityContext(old_idl_dir, new_idl_dir, IDLCompatibilityErrorCollection()) with open(old_basic_types_path) as old_file: old_idl_file = parser.parse(old_file, old_basic_types_path, CompilerImportResolver(import_directories)) if old_idl_file.errors: old_idl_file.errors.dump_errors() raise ValueError(f"Cannot parse {old_basic_types_path}") old_error_reply_struct = old_idl_file.spec.symbols.get_struct("ErrorReply") if old_error_reply_struct is None: ctxt.add_missing_error_reply_error(old_basic_types_path) else: with open(new_basic_types_path) as new_file: new_idl_file = parser.parse(new_file, new_basic_types_path, CompilerImportResolver(import_directories)) if new_idl_file.errors: new_idl_file.errors.dump_errors() raise ValueError(f"Cannot parse {new_basic_types_path}") new_error_reply_struct = new_idl_file.spec.symbols.get_struct("ErrorReply") if new_error_reply_struct is None: ctxt.add_missing_error_reply_error(new_basic_types_path) else: check_reply_fields(ctxt, old_error_reply_struct, new_error_reply_struct, "n/a", old_idl_file, new_idl_file, old_basic_types_path, new_basic_types_path) ctxt.errors.dump_errors() return ctxt.errors
def check_security_access_check(ctxt: IDLCompatibilityContext, old_access_checks: syntax.AccessChecks, new_access_checks: syntax.AccessChecks, cmd_name: str, new_idl_file_path: str) -> None: """Check the compatibility between security access checks of the old and new command.""" if old_access_checks is not None and new_access_checks is not None: old_simple_check = old_access_checks.simple new_simple_check = new_access_checks.simple if old_simple_check is not None and new_simple_check is not None: if old_simple_check.check != new_simple_check.check: ctxt.add_check_not_equal_error(cmd_name, old_simple_check.check, new_simple_check.check, new_idl_file_path) else: old_privilege = old_simple_check.privilege new_privilege = new_simple_check.privilege if old_privilege is not None and new_privilege is not None: if old_privilege.resource_pattern != new_privilege.resource_pattern: ctxt.add_resource_pattern_not_equal_error( cmd_name, old_privilege.resource_pattern, new_privilege.resource_pattern, new_idl_file_path) if not set(new_privilege.action_type).issubset( old_privilege.action_type): ctxt.add_new_action_types_not_subset_error( cmd_name, new_idl_file_path)
def check_reply_fields(ctxt: IDLCompatibilityContext, old_cmd: syntax.Command, new_cmd: syntax.Command, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new reply fields.""" # pylint: disable=too-many-arguments old_reply = old_idl_file.spec.symbols.get_struct(old_cmd.reply_type) new_reply = new_idl_file.spec.symbols.get_struct(new_cmd.reply_type) for old_field in old_reply.fields or []: if old_field.unstable: continue new_field_exists = False for new_field in new_reply.fields or []: if new_field.name == old_field.name: new_field_exists = True check_reply_field(ctxt, old_field, new_field, old_cmd.name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) break if not new_field_exists: ctxt.add_new_reply_field_missing_error(new_cmd.command_name, old_field.name, old_idl_file_path)
def check_command_parameter(ctxt: IDLCompatibilityContext, old_param: syntax.Field, new_param: syntax.Field, cmd_name: str, old_idl_file_path: str): """Check compatibility between the old and new command parameter.""" if not old_param.unstable and new_param.unstable: ctxt.add_command_parameter_unstable_error(cmd_name, old_param.name, old_idl_file_path) if old_param.unstable and not new_param.optional and not new_param.unstable: ctxt.add_command_parameter_stable_required_error(cmd_name, old_param.name, old_idl_file_path) if old_param.optional and not new_param.optional: ctxt.add_command_parameter_required_error(cmd_name, old_param.name, old_idl_file_path)
def check_compatibility(old_idl_dir: str, new_idl_dir: str, import_directories: List[str]) -> IDLCompatibilityErrorCollection: """Check IDL compatibility between old and new IDL commands.""" # pylint: disable=too-many-locals ctxt = IDLCompatibilityContext(old_idl_dir, new_idl_dir, IDLCompatibilityErrorCollection()) new_commands, new_command_file, new_command_file_path = get_new_commands( ctxt, new_idl_dir, import_directories) # Check new commands' compatibility with old ones. # Note, a command can be added to V1 at any time, it's ok if a # new command has no corresponding old command. old_commands: Dict[str, syntax.Command] = dict() for dirpath, _, filenames in os.walk(old_idl_dir): for old_filename in filenames: if not old_filename.endswith('.idl') or old_filename in SKIPPED_FILES: continue old_idl_file_path = os.path.join(dirpath, old_filename) with open(old_idl_file_path) as old_file: old_idl_file = parser.parse( old_file, old_idl_file_path, CompilerImportResolver(import_directories + [old_idl_dir])) if old_idl_file.errors: old_idl_file.errors.dump_errors() raise ValueError(f"Cannot parse {old_idl_file_path}") for old_cmd in old_idl_file.spec.symbols.commands: # Ignore imported commands as they will be processed in their own file. if old_cmd.api_version == "" or old_cmd.imported: continue if old_cmd.api_version != "1": # We're not ready to handle future API versions yet. ctxt.add_command_invalid_api_version_error( old_cmd.command_name, old_cmd.api_version, old_idl_file_path) continue if old_cmd.command_name in old_commands: ctxt.add_duplicate_command_name_error(old_cmd.command_name, old_idl_dir, old_idl_file_path) continue old_commands[old_cmd.command_name] = old_cmd if old_cmd.command_name not in new_commands: # Can't remove a command from V1 ctxt.add_command_removed_error(old_cmd.command_name, old_idl_file_path) continue new_cmd = new_commands[old_cmd.command_name] new_idl_file = new_command_file[old_cmd.command_name] new_idl_file_path = new_command_file_path[old_cmd.command_name] if not old_cmd.strict and new_cmd.strict: ctxt.add_command_strict_true_error(new_cmd.command_name, new_idl_file_path) # Check compatibility of command's parameters. check_command_params_or_type_struct_fields( ctxt, old_cmd, new_cmd, old_cmd.command_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, is_command_parameter=True) check_namespace(ctxt, old_cmd, new_cmd, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) old_reply = old_idl_file.spec.symbols.get_struct(old_cmd.reply_type) new_reply = new_idl_file.spec.symbols.get_struct(new_cmd.reply_type) check_reply_fields(ctxt, old_reply, new_reply, old_cmd.command_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) check_security_access_checks(ctxt, old_cmd.access_check, new_cmd.access_check, old_cmd, new_idl_file_path) # TODO (SERVER-55203): Remove error_skipped logic. ctxt.errors.remove_skipped_errors_and_dump_all_errors("Commands", old_idl_dir, new_idl_dir) return ctxt.errors
def check_reply_field_type_recursive( ctxt: IDLCompatibilityContext, old_field_type: syntax.Type, new_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, field_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str) -> None: # pylint: disable=too-many-arguments,too-many-branches """Check compatibility between old and new reply field type if old field type is a syntax.Type instance.""" if not isinstance(new_field_type, syntax.Type): ctxt.add_new_reply_field_type_enum_or_struct_error( cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path) return if "any" in old_field_type.bson_serialization_type: ctxt.add_old_reply_field_bson_any_error(cmd_name, field_name, old_field_type.name, old_idl_file_path) return if "any" in new_field_type.bson_serialization_type: ctxt.add_new_reply_field_bson_any_error(cmd_name, field_name, new_field_type.name, new_idl_file_path) return if isinstance(old_field_type, syntax.VariantType): # If the new type is not variant just check the single type. new_variant_types = new_field_type.variant_types if isinstance( new_field_type, syntax.VariantType) else [new_field_type] old_variant_types = old_field_type.variant_types # Check that new variant types are a subset of old variant types. for new_variant_type in new_variant_types: old_variant_type_exists = False for old_variant_type in old_variant_types: if old_variant_type.name == new_variant_type.name: old_variant_type_exists = True # Check that the old and new version of each variant type is also compatible. check_reply_field_type_recursive( ctxt, old_variant_type, new_variant_type, cmd_name, field_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) if not old_variant_type_exists: ctxt.add_new_reply_field_variant_type_not_subset_error( cmd_name, field_name, new_field_type.name, new_idl_file_path) # If new type is variant and has a struct as a variant type, compare old and new variant_struct_type. # Since enums can't be part of variant types, we don't explicitly check for enums. if isinstance(new_field_type, syntax.VariantType) and new_field_type.variant_struct_type is not None: if old_field_type.variant_struct_type is None: ctxt.add_new_reply_field_variant_type_not_subset_error( cmd_name, field_name, new_field_type.variant_struct_type.name, new_idl_file_path) else: check_reply_fields(ctxt, old_field_type.variant_struct_type, new_field_type.variant_struct_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) else: if isinstance(new_field_type, syntax.VariantType): ctxt.add_new_reply_field_variant_type_error(cmd_name, field_name, old_field_type.name, new_field_type.name, new_idl_file_path) else: check_subset(ctxt, cmd_name, field_name, new_field_type.name, new_field_type.bson_serialization_type, old_field_type.bson_serialization_type, new_idl_file_path)
def check_security_access_checks(ctxt: IDLCompatibilityContext, old_access_checks: syntax.AccessChecks, new_access_checks: syntax.AccessChecks, cmd: syntax.Command, new_idl_file_path: str) -> None: """Check the compatibility between security access checks of the old and new command.""" # pylint:disable=too-many-locals,too-many-branches,too-many-nested-blocks cmd_name = cmd.command_name if old_access_checks is not None and new_access_checks is not None: old_access_check_type = old_access_checks.get_access_check_type() new_access_check_type = new_access_checks.get_access_check_type() if old_access_check_type != new_access_check_type: ctxt.add_access_check_type_not_equal_error(cmd_name, old_access_check_type, new_access_check_type, new_idl_file_path) else: old_simple_check = old_access_checks.simple new_simple_check = new_access_checks.simple if old_simple_check is not None and new_simple_check is not None: if old_simple_check.check != new_simple_check.check: ctxt.add_check_not_equal_error(cmd_name, old_simple_check.check, new_simple_check.check, new_idl_file_path) else: old_privilege = old_simple_check.privilege new_privilege = new_simple_check.privilege if old_privilege is not None and new_privilege is not None: if old_privilege.resource_pattern != new_privilege.resource_pattern: ctxt.add_resource_pattern_not_equal_error( cmd_name, old_privilege.resource_pattern, new_privilege.resource_pattern, new_idl_file_path) if not set(new_privilege.action_type).issubset(old_privilege.action_type): ctxt.add_new_action_types_not_subset_error(cmd_name, new_idl_file_path) old_complex_checks = old_access_checks.complex new_complex_checks = new_access_checks.complex if old_complex_checks is not None and new_complex_checks is not None: if len(new_complex_checks) > len(old_complex_checks): ctxt.add_new_additional_complex_access_check_error(cmd_name, new_idl_file_path) else: old_checks, old_privileges = split_complex_checks(old_complex_checks) new_checks, new_privileges = split_complex_checks(new_complex_checks) if not set(new_checks).issubset(old_checks): ctxt.add_new_complex_checks_not_subset_error(cmd_name, new_idl_file_path) if len(new_privileges) > len(old_privileges): ctxt.add_new_complex_privileges_not_subset_error( cmd_name, new_idl_file_path) else: # Check that each new_privilege matches an old_privilege (the resource_pattern is # equal and the action_types are a subset of the old action_types). for new_privilege in new_privileges: for old_privilege in old_privileges: if (new_privilege.resource_pattern == old_privilege.resource_pattern and set(new_privilege.action_type).issubset( old_privilege.action_type)): old_privileges.remove(old_privilege) break else: ctxt.add_new_complex_privileges_not_subset_error( cmd_name, new_idl_file_path) elif new_access_checks is None and old_access_checks is not None: ctxt.add_removed_access_check_field_error(cmd_name, new_idl_file_path) elif old_access_checks is None and new_access_checks is not None and cmd.api_version == '1': ctxt.add_added_access_check_field_error(cmd_name, new_idl_file_path)
def check_command_type(ctxt: IDLCompatibilityContext, old_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], new_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str): """Check compatibility between old and new command type.""" # pylint: disable=too-many-arguments,too-many-branches if old_type is None: ctxt.add_command_type_invalid_error(cmd_name, old_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) if new_type is None: ctxt.add_command_type_invalid_error(cmd_name, new_idl_file_path) ctxt.errors.dump_errors() sys.exit(1) if isinstance(old_type, syntax.Type): if isinstance(new_type, syntax.Type): if "any" in old_type.bson_serialization_type: ctxt.add_old_command_type_bson_any_error(cmd_name, old_type.name, old_idl_file_path) elif "any" in new_type.bson_serialization_type: ctxt.add_new_command_type_bson_any_error(cmd_name, new_type.name, new_idl_file_path) else: check_type_superset(ctxt, cmd_name, new_type.name, old_type.bson_serialization_type, new_type.bson_serialization_type, new_idl_file_path) else: ctxt.add_new_command_type_enum_or_struct_error(cmd_name, new_type.name, old_type.name, new_idl_file_path) elif isinstance(old_type, syntax.Enum): if isinstance(new_type, syntax.Enum): check_type_superset(ctxt, cmd_name, new_type.name, old_type.values, new_type.values, new_idl_file_path) else: ctxt.add_new_command_type_not_enum_error(cmd_name, new_type.name, old_type.name, new_idl_file_path) elif isinstance(old_type, syntax.Struct): if isinstance(new_type, syntax.Struct): check_command_type_struct_fields(ctxt, old_type, new_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) else: ctxt.add_new_command_type_not_struct_error(cmd_name, new_type.name, old_type.name, new_idl_file_path)
def check_reply_field_type_recursive( ctxt: IDLCompatibilityContext, old_field_type: syntax.Type, new_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, field_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str) -> None: # pylint: disable=too-many-arguments,too-many-branches """Check compatibility between old and new reply field type if old field type is a syntax.Type instance.""" if not isinstance(new_field_type, syntax.Type): ctxt.add_new_reply_field_type_enum_or_struct_error( cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path) return # If bson_serialization_type switches from 'any' to non-any type. if "any" in old_field_type.bson_serialization_type and "any" not in new_field_type.bson_serialization_type: ctxt.add_old_reply_field_bson_any_error(cmd_name, field_name, old_field_type.name, old_idl_file_path) return # If bson_serialization_type switches from non-any to 'any' type. if "any" not in old_field_type.bson_serialization_type and "any" in new_field_type.bson_serialization_type: ctxt.add_new_reply_field_bson_any_error(cmd_name, field_name, old_field_type.name, new_idl_file_path) return allow_name: str = cmd_name + "-reply-" + field_name if "any" in old_field_type.bson_serialization_type: # If 'any' is not explicitly allowed as the bson_serialization_type. if allow_name not in ALLOW_ANY_TYPE_LIST: ctxt.add_reply_field_bson_any_not_allowed_error( cmd_name, field_name, old_field_type.name, old_idl_file_path) return if old_field_type.cpp_type != new_field_type.cpp_type: ctxt.add_reply_field_cpp_type_not_equal_error( cmd_name, field_name, new_field_type.name, new_idl_file_path) if isinstance(old_field_type, syntax.VariantType): # If the new type is not variant just check the single type. new_variant_types = new_field_type.variant_types if isinstance( new_field_type, syntax.VariantType) else [new_field_type] old_variant_types = old_field_type.variant_types # Check that new variant types are a subset of old variant types. for new_variant_type in new_variant_types: for old_variant_type in old_variant_types: if old_variant_type.name == new_variant_type.name: # Check that the old and new version of each variant type is also compatible. check_reply_field_type_recursive( ctxt, old_variant_type, new_variant_type, cmd_name, field_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) break else: # new_variant_type was not found in old_variant_types. ctxt.add_new_reply_field_variant_type_not_subset_error( cmd_name, field_name, new_variant_type.name, new_idl_file_path) # If new type is variant and has a struct as a variant type, compare old and new variant_struct_type. # Since enums can't be part of variant types, we don't explicitly check for enums. if isinstance(new_field_type, syntax.VariantType ) and new_field_type.variant_struct_type is not None: if old_field_type.variant_struct_type is None: ctxt.add_new_reply_field_variant_type_not_subset_error( cmd_name, field_name, new_field_type.variant_struct_type.name, new_idl_file_path) else: check_reply_fields(ctxt, old_field_type.variant_struct_type, new_field_type.variant_struct_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path) else: if isinstance(new_field_type, syntax.VariantType): ctxt.add_new_reply_field_variant_type_error( cmd_name, field_name, old_field_type.name, new_idl_file_path) else: check_subset(ctxt, cmd_name, field_name, new_field_type.name, new_field_type.bson_serialization_type, old_field_type.bson_serialization_type, new_idl_file_path)
def check_param_or_command_type_recursive( ctxt: IDLCompatibilityContext, old_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], new_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec, old_idl_file_path: str, new_idl_file_path: str, param_name: Optional[str], is_command_parameter: bool): # pylint: disable=too-many-arguments,too-many-branches """ Check compatibility between old and new command or param type recursively. If the old type is a syntax.Type instance, check the compatibility between the old and new command type or parameter type recursively. """ if not isinstance(new_type, syntax.Type): ctxt.add_new_command_or_param_type_enum_or_struct_error( cmd_name, new_type.name, old_type.name, new_idl_file_path, param_name, is_command_parameter) return # If bson_serialization_type switches from 'any' to non-any type. if "any" in old_type.bson_serialization_type and "any" not in new_type.bson_serialization_type: ctxt.add_old_command_or_param_type_bson_any_error( cmd_name, old_type.name, old_idl_file_path, param_name, is_command_parameter) return # If bson_serialization_type switches from non-any to 'any' type. if "any" not in old_type.bson_serialization_type and "any" in new_type.bson_serialization_type: ctxt.add_new_command_or_param_type_bson_any_error( cmd_name, new_type.name, new_idl_file_path, param_name, is_command_parameter) return allow_name: str = cmd_name + "-param-" + param_name if is_command_parameter else cmd_name if "any" in old_type.bson_serialization_type: # If 'any' is not explicitly allowed as the bson_serialization_type. if allow_name not in ALLOW_ANY_TYPE_LIST: ctxt.add_command_or_param_type_bson_any_not_allowed_error( cmd_name, old_type.name, old_idl_file_path, param_name, is_command_parameter) return if old_type.cpp_type != new_type.cpp_type: ctxt.add_command_or_param_cpp_type_not_equal_error( cmd_name, new_type.name, new_idl_file_path, param_name, is_command_parameter) if isinstance(old_type, syntax.VariantType): if not isinstance(new_type, syntax.VariantType): ctxt.add_new_command_or_param_type_not_variant_type_error( cmd_name, new_type.name, new_idl_file_path, param_name, is_command_parameter) else: new_variant_types = new_type.variant_types old_variant_types = old_type.variant_types # Check that new variant types are a superset of old variant types. for old_variant_type in old_variant_types: for new_variant_type in new_variant_types: if old_variant_type.name == new_variant_type.name: # Check that the old and new version of each variant type is also compatible. check_param_or_command_type_recursive( ctxt, old_variant_type, new_variant_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, param_name, is_command_parameter) break else: # old_variant_type was not found in new_variant_types. ctxt.add_new_command_or_param_variant_type_not_superset_error( cmd_name, old_variant_type.name, new_idl_file_path, param_name, is_command_parameter) # If old and new types both have a struct as a variant type, compare old and new variant_struct_type. # Since enums can't be part of variant types, we don't explicitly check for enums. if old_type.variant_struct_type is not None: if new_type.variant_struct_type is not None: check_command_params_or_type_struct_fields( ctxt, old_type.variant_struct_type, new_type.variant_struct_type, cmd_name, old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path, is_command_parameter) # If old type has a variant struct type and new type does not have a variant struct type. else: ctxt.add_new_command_or_param_variant_type_not_superset_error( cmd_name, old_type.variant_struct_type.name, new_idl_file_path, param_name, is_command_parameter) else: check_superset(ctxt, cmd_name, new_type.name, new_type.bson_serialization_type, old_type.bson_serialization_type, new_idl_file_path, param_name, is_command_parameter)