def MoeProjectConfigFromJson(config_json, filename=''):
"""Create a MoeProjectConfig from a config JSON object."""
config_utils.CheckJsonKeys('project', config_json, _PROJECT_CONFIG_KEYS)
project_name = config_json[u'name']
project = MoeProjectConfig(project_name)
project.empty = False
project.filename = config_json.get(u'filename') or filename
project.translators = config_json.get(u'translators', [])
project.internal_repository_config = MakeRepositoryConfig(
config_json[u'internal_repository'],
repository_name=project_name + '_internal',
project_space=base.INTERNAL_STR)
project.public_repository_config = MakeRepositoryConfig(
config_json[u'public_repository'],
repository_name=project_name + '_public',
project_space=base.PUBLIC_STR)
project.noisy_files_re = config_json.get('noisy_files_re')
project.moe_db_url = config_json.get('moe_db_url')
project.owners = config_json.get('owners', [])
project.manual_equivalence_deltas = config_json.get(
'manual_equivalence_deltas')
project.import_strategy = _MigrationStrategyFromJson(
config_json.get(u'import_strategy'),
default_strategy=project.import_strategy)
project.export_strategy = _MigrationStrategyFromJson(
config_json.get(u'export_strategy'),
default_strategy=project.export_strategy)
project.config_json = config_json
return project
def __init__(self,
usernames_file=None,
publishable_usernames=None,
scrubbable_usernames=None,
scrub_unknown_users=False):
"""Create a UsernameFilter.
Args:
usernames_file: str, filename of usernames to publish/scrub
publishable_usernames: set, usernames to publish
scrubbable_usernames: set, usernames to scrub
scrub_unknown_users: bool, whether usernames not specified as publishable
or scrubbable should be scrubbed or reported as
errors
"""
publishable_usernames = set(publishable_usernames or [])
scrubbable_usernames = set(scrubbable_usernames or [])
self._scrub_unknown_users = scrub_unknown_users
if usernames_file:
usernames = config_utils.ReadConfigFile(usernames_file)
config_utils.CheckJsonKeys(
'usernames config', usernames,
[u'publishable_usernames', u'scrubbable_usernames'])
publishable_usernames.update(
usernames.get(u'publishable_usernames', []))
scrubbable_usernames.update(
usernames.get(u'scrubbable_usernames', []))
self._publishable_usernames = publishable_usernames
self._scrubbable_usernames = scrubbable_usernames
def CommentsFromJson(comments_json, filename_substitutions=None):
"""Generate a list of Comment objects from a JSON list."""
filename_substitutions = filename_substitutions or {}
result = []
for comment_json in comments_json:
config_utils.CheckJsonKeys(
'comment', comment_json,
[u'filename', u'line', u'text', u'char_index'])
filename = comment_json.get(u'filename', '')
result.append(
Comment(filename_substitutions.get(filename, filename),
comment_json.get(u'line', 0),
comment_json.get(u'char_index', 0),
comment_json.get(u'text', '')))
return result
def _MigrationStrategyFromJson(strategy_json, default_strategy=None):
if strategy_json is None:
return default_strategy
config_utils.CheckJsonKeys('strategy', strategy_json, [
u'merge_strategy', u'commit_strategy', u'separate_revisions',
u'copy_metadata', u'preapprove_public_changelogs'
])
return MigrationStrategy(
merge_strategy=strategy_json.get(u'merge_strategy'),
commit_strategy=strategy_json.get(u'commit_strategy'),
separate_revisions=strategy_json.get(u'separate_revisions'),
copy_metadata=strategy_json.get(u'copy_metadata'),
preapprove_public_changelogs=strategy_json.get(
u'preapprove_public_changelogs'),
default_strategy=default_strategy)
def assertCheckSucceeds(self):
config_utils.CheckJsonKeys('test', self._dict, self._keys)
def ScrubberConfigFromJson(codebase,
input_files,
config_json,
extension_to_scrubber_map=None,
default_scrubbers=None,
modify=False,
output_tar='',
temp_dir='',
**unused_kwargs):
"""Generate a ScrubberConfig object from a ScrubberConfig JSON object."""
def SetOption(key, func=None):
"""Set an option in the config from JSON, using the enclosing scope.
Args:
key: unicode; the key in the JSON config and corresponding config
attribute name.
func: An optional transformation to apply to the JSON value before storing
in the config.
"""
if key in config_json:
value = config_json[key]
if func is not None:
value = func(value)
setattr(config, str(key), value)
config_utils.CheckJsonKeys('scrubber config', config_json,
_SCRUBBER_CONFIG_KEYS)
config = ScrubberConfig(codebase, input_files, extension_to_scrubber_map,
default_scrubbers, modify, output_tar, temp_dir)
# General options.
SetOption(u'ignore_files_re', func=re.compile)
SetOption(u'do_not_scrub_files_re', func=re.compile)
SetOption(u'sensitive_words')
config.sensitive_words = config_json.get(u'sensitive_words', [])
SetOption(u'extension_map',
func=lambda m: [(re.compile(r), e) for r, e in m])
SetOption(u'sensitive_res')
sensitive_string_file = config_json.get(u'sensitive_string_file')
if sensitive_string_file:
sensitive_string_json = config_utils.ReadConfigFile(
sensitive_string_file)
config_utils.CheckJsonKeys('sensitive string config',
sensitive_string_json,
[u'sensitive_words', u'sensitive_res'])
config.sensitive_words.extend(
sensitive_string_json.get(u'sensitive_words', []))
config.sensitive_res.extend(
sensitive_string_json.get(u'sensitive_res', []))
whitelist_entries = []
for entry in config_json.get(u'whitelist', []):
config_utils.CheckJsonKeys('whitelist entry', entry,
[u'filter', u'trigger', u'filename'])
whitelist_entries.append(
(entry.get(u'filter',
''), entry.get(u'trigger',
''), entry.get(u'filename', '')))
config.whitelist = whitelist.Whitelist(whitelist_entries)
SetOption(u'scrub_sensitive_comments')
SetOption(u'rearranging_config')
SetOption(u'string_replacements')
SetOption(u'regex_replacements')
SetOption(u'scrub_non_documentation_comments')
SetOption(u'scrub_all_comments')
# User options.
# TODO(dborowitz): Make the scrubbers pass unicode to the UsernameFilter.
# TODO(dborowitz): Make these names consistent so we can use SetOption.
strs = lambda us: [str(u) for u in us]
if u'usernames_to_publish' in config_json:
config.publishable_usernames = strs(
config_json[u'usernames_to_publish'])
if u'usernames_to_scrub' in config_json:
config.scrubbable_usernames = strs(config_json[u'usernames_to_scrub'])
SetOption(u'usernames_file')
SetOption(u'scrub_unknown_users')
SetOption(u'scrub_authors')
SetOption(u'scrub_proto_comments')
# C/C++-specific options.
SetOption(u'c_includes_config_file')
# Java-specific options.
action_map = {
'IGNORE': base.ACTION_IGNORE,
'DELETE': base.ACTION_DELETE,
'ERROR': base.ACTION_ERROR,
}
SetOption(u'empty_java_file_action', func=lambda a: action_map[a])
SetOption(u'maximum_blank_lines')
SetOption(u'scrub_java_testsize_annotations')
config.java_renames = []
for rename in config_json.get(u'java_renames', []):
config_utils.CheckJsonKeys('java rename', rename,
[u'internal_package', u'public_package'])
config.java_renames.append(
java_scrubber.JavaRenameScrubber(rename[u'internal_package'],
rename[u'public_package']))
# Javascript-specific options.
# TODO(user): Remove js_directory_rename after all config files have been
# migrated to use js_directory_renames.
js_directory_rename = config_json.get(u'js_directory_rename')
if js_directory_rename is not None:
config_utils.CheckJsonKeys(
'JS directory rename', js_directory_rename,
[u'internal_directory', u'public_directory'])
config.js_directory_renames.append(
line_scrubber.JsDirectoryRename(
js_directory_rename[u'internal_directory'],
js_directory_rename[u'public_directory']))
js_directory_renames = config_json.get(u'js_directory_renames', [])
for js_directory_rename in js_directory_renames:
config_utils.CheckJsonKeys(
'JS directory rename', js_directory_rename,
[u'internal_directory', u'public_directory'])
config.js_directory_renames.append(
line_scrubber.JsDirectoryRename(
js_directory_rename[u'internal_directory'],
js_directory_rename[u'public_directory']))
# Python-specific options.
config.python_module_renames = []
for rename in config_json.get(u'python_module_renames', []):
config_utils.CheckJsonKeys(
'python module rename', rename,
[u'internal_module', u'public_module', u'as_name'])
config.python_module_renames.append(
python_scrubber.PythonModuleRename(rename[u'internal_module'],
rename[u'public_module'],
as_name=rename.get(u'as_name')))
# TODO(dborowitz): Find out why these are singleton protobufs; possibly
# flatten them.
config.python_module_removes = []
for remove in config_json.get(u'python_module_removes', []):
config_utils.CheckJsonKeys('python module removal', remove,
[u'import_module'])
config.python_module_removes.append(
python_scrubber.PythonModuleRemove(remove[u'import_module']))
python_shebang_replace = config_json.get(u'python_shebang_replace')
if python_shebang_replace is not None:
config_utils.CheckJsonKeys('python shebang replacement',
python_shebang_replace, [u'shebang_line'])
config.python_shebang_replace = python_scrubber.PythonShebangReplace(
python_shebang_replace[u'shebang_line'])
# GWT-specific options.
SetOption(u'scrub_gwt_inherits')
config.ResetScrubbers(extension_to_scrubber_map, default_scrubbers)
return config