Exemple #1
0
def MoeProjectConfigFromJson(config_json, filename=''):
    """Create a MoeProjectConfig from a config JSON object."""
    config_utils.CheckJsonKeys('project', config_json, _PROJECT_CONFIG_KEYS)
    project_name = config_json[u'name']
    project = MoeProjectConfig(project_name)
    project.empty = False
    project.filename = config_json.get(u'filename') or filename

    project.translators = config_json.get(u'translators', [])

    project.internal_repository_config = MakeRepositoryConfig(
        config_json[u'internal_repository'],
        repository_name=project_name + '_internal',
        project_space=base.INTERNAL_STR)

    project.public_repository_config = MakeRepositoryConfig(
        config_json[u'public_repository'],
        repository_name=project_name + '_public',
        project_space=base.PUBLIC_STR)

    project.noisy_files_re = config_json.get('noisy_files_re')
    project.moe_db_url = config_json.get('moe_db_url')
    project.owners = config_json.get('owners', [])
    project.manual_equivalence_deltas = config_json.get(
        'manual_equivalence_deltas')
    project.import_strategy = _MigrationStrategyFromJson(
        config_json.get(u'import_strategy'),
        default_strategy=project.import_strategy)
    project.export_strategy = _MigrationStrategyFromJson(
        config_json.get(u'export_strategy'),
        default_strategy=project.export_strategy)

    project.config_json = config_json
    return project
Exemple #2
0
    def __init__(self,
                 usernames_file=None,
                 publishable_usernames=None,
                 scrubbable_usernames=None,
                 scrub_unknown_users=False):
        """Create a UsernameFilter.

    Args:
      usernames_file: str, filename of usernames to publish/scrub
      publishable_usernames: set, usernames to publish
      scrubbable_usernames: set, usernames to scrub
      scrub_unknown_users: bool, whether usernames not specified as publishable
                           or scrubbable should be scrubbed or reported as
                           errors
    """
        publishable_usernames = set(publishable_usernames or [])
        scrubbable_usernames = set(scrubbable_usernames or [])

        self._scrub_unknown_users = scrub_unknown_users

        if usernames_file:
            usernames = config_utils.ReadConfigFile(usernames_file)
            config_utils.CheckJsonKeys(
                'usernames config', usernames,
                [u'publishable_usernames', u'scrubbable_usernames'])
            publishable_usernames.update(
                usernames.get(u'publishable_usernames', []))
            scrubbable_usernames.update(
                usernames.get(u'scrubbable_usernames', []))

        self._publishable_usernames = publishable_usernames
        self._scrubbable_usernames = scrubbable_usernames
Exemple #3
0
def CommentsFromJson(comments_json, filename_substitutions=None):
    """Generate a list of Comment objects from a JSON list."""
    filename_substitutions = filename_substitutions or {}
    result = []
    for comment_json in comments_json:
        config_utils.CheckJsonKeys(
            'comment', comment_json,
            [u'filename', u'line', u'text', u'char_index'])
        filename = comment_json.get(u'filename', '')
        result.append(
            Comment(filename_substitutions.get(filename, filename),
                    comment_json.get(u'line', 0),
                    comment_json.get(u'char_index', 0),
                    comment_json.get(u'text', '')))
    return result
Exemple #4
0
def _MigrationStrategyFromJson(strategy_json, default_strategy=None):
    if strategy_json is None:
        return default_strategy
    config_utils.CheckJsonKeys('strategy', strategy_json, [
        u'merge_strategy', u'commit_strategy', u'separate_revisions',
        u'copy_metadata', u'preapprove_public_changelogs'
    ])

    return MigrationStrategy(
        merge_strategy=strategy_json.get(u'merge_strategy'),
        commit_strategy=strategy_json.get(u'commit_strategy'),
        separate_revisions=strategy_json.get(u'separate_revisions'),
        copy_metadata=strategy_json.get(u'copy_metadata'),
        preapprove_public_changelogs=strategy_json.get(
            u'preapprove_public_changelogs'),
        default_strategy=default_strategy)
Exemple #5
0
 def assertCheckSucceeds(self):
   config_utils.CheckJsonKeys('test', self._dict, self._keys)
Exemple #6
0
def ScrubberConfigFromJson(codebase,
                           input_files,
                           config_json,
                           extension_to_scrubber_map=None,
                           default_scrubbers=None,
                           modify=False,
                           output_tar='',
                           temp_dir='',
                           **unused_kwargs):
    """Generate a ScrubberConfig object from a ScrubberConfig JSON object."""
    def SetOption(key, func=None):
        """Set an option in the config from JSON, using the enclosing scope.

    Args:
      key: unicode; the key in the JSON config and corresponding config
           attribute name.
      func: An optional transformation to apply to the JSON value before storing
            in the config.
    """
        if key in config_json:
            value = config_json[key]
            if func is not None:
                value = func(value)
            setattr(config, str(key), value)

    config_utils.CheckJsonKeys('scrubber config', config_json,
                               _SCRUBBER_CONFIG_KEYS)
    config = ScrubberConfig(codebase, input_files, extension_to_scrubber_map,
                            default_scrubbers, modify, output_tar, temp_dir)

    # General options.
    SetOption(u'ignore_files_re', func=re.compile)
    SetOption(u'do_not_scrub_files_re', func=re.compile)
    SetOption(u'sensitive_words')
    config.sensitive_words = config_json.get(u'sensitive_words', [])
    SetOption(u'extension_map',
              func=lambda m: [(re.compile(r), e) for r, e in m])
    SetOption(u'sensitive_res')
    sensitive_string_file = config_json.get(u'sensitive_string_file')
    if sensitive_string_file:
        sensitive_string_json = config_utils.ReadConfigFile(
            sensitive_string_file)
        config_utils.CheckJsonKeys('sensitive string config',
                                   sensitive_string_json,
                                   [u'sensitive_words', u'sensitive_res'])
        config.sensitive_words.extend(
            sensitive_string_json.get(u'sensitive_words', []))
        config.sensitive_res.extend(
            sensitive_string_json.get(u'sensitive_res', []))

    whitelist_entries = []
    for entry in config_json.get(u'whitelist', []):
        config_utils.CheckJsonKeys('whitelist entry', entry,
                                   [u'filter', u'trigger', u'filename'])
        whitelist_entries.append(
            (entry.get(u'filter',
                       ''), entry.get(u'trigger',
                                      ''), entry.get(u'filename', '')))
    config.whitelist = whitelist.Whitelist(whitelist_entries)
    SetOption(u'scrub_sensitive_comments')
    SetOption(u'rearranging_config')
    SetOption(u'string_replacements')
    SetOption(u'regex_replacements')
    SetOption(u'scrub_non_documentation_comments')
    SetOption(u'scrub_all_comments')

    # User options.
    # TODO(dborowitz): Make the scrubbers pass unicode to the UsernameFilter.
    # TODO(dborowitz): Make these names consistent so we can use SetOption.
    strs = lambda us: [str(u) for u in us]
    if u'usernames_to_publish' in config_json:
        config.publishable_usernames = strs(
            config_json[u'usernames_to_publish'])
    if u'usernames_to_scrub' in config_json:
        config.scrubbable_usernames = strs(config_json[u'usernames_to_scrub'])
    SetOption(u'usernames_file')
    SetOption(u'scrub_unknown_users')
    SetOption(u'scrub_authors')
    SetOption(u'scrub_proto_comments')

    # C/C++-specific options.
    SetOption(u'c_includes_config_file')

    # Java-specific options.
    action_map = {
        'IGNORE': base.ACTION_IGNORE,
        'DELETE': base.ACTION_DELETE,
        'ERROR': base.ACTION_ERROR,
    }
    SetOption(u'empty_java_file_action', func=lambda a: action_map[a])
    SetOption(u'maximum_blank_lines')
    SetOption(u'scrub_java_testsize_annotations')
    config.java_renames = []
    for rename in config_json.get(u'java_renames', []):
        config_utils.CheckJsonKeys('java rename', rename,
                                   [u'internal_package', u'public_package'])
        config.java_renames.append(
            java_scrubber.JavaRenameScrubber(rename[u'internal_package'],
                                             rename[u'public_package']))

    # Javascript-specific options.
    # TODO(user): Remove js_directory_rename after all config files have been
    # migrated to use js_directory_renames.
    js_directory_rename = config_json.get(u'js_directory_rename')
    if js_directory_rename is not None:
        config_utils.CheckJsonKeys(
            'JS directory rename', js_directory_rename,
            [u'internal_directory', u'public_directory'])
        config.js_directory_renames.append(
            line_scrubber.JsDirectoryRename(
                js_directory_rename[u'internal_directory'],
                js_directory_rename[u'public_directory']))

    js_directory_renames = config_json.get(u'js_directory_renames', [])
    for js_directory_rename in js_directory_renames:
        config_utils.CheckJsonKeys(
            'JS directory rename', js_directory_rename,
            [u'internal_directory', u'public_directory'])
        config.js_directory_renames.append(
            line_scrubber.JsDirectoryRename(
                js_directory_rename[u'internal_directory'],
                js_directory_rename[u'public_directory']))

    # Python-specific options.
    config.python_module_renames = []
    for rename in config_json.get(u'python_module_renames', []):
        config_utils.CheckJsonKeys(
            'python module rename', rename,
            [u'internal_module', u'public_module', u'as_name'])
        config.python_module_renames.append(
            python_scrubber.PythonModuleRename(rename[u'internal_module'],
                                               rename[u'public_module'],
                                               as_name=rename.get(u'as_name')))

    # TODO(dborowitz): Find out why these are singleton protobufs; possibly
    # flatten them.
    config.python_module_removes = []
    for remove in config_json.get(u'python_module_removes', []):
        config_utils.CheckJsonKeys('python module removal', remove,
                                   [u'import_module'])
        config.python_module_removes.append(
            python_scrubber.PythonModuleRemove(remove[u'import_module']))

    python_shebang_replace = config_json.get(u'python_shebang_replace')
    if python_shebang_replace is not None:
        config_utils.CheckJsonKeys('python shebang replacement',
                                   python_shebang_replace, [u'shebang_line'])
        config.python_shebang_replace = python_scrubber.PythonShebangReplace(
            python_shebang_replace[u'shebang_line'])

    # GWT-specific options.
    SetOption(u'scrub_gwt_inherits')

    config.ResetScrubbers(extension_to_scrubber_map, default_scrubbers)
    return config