def setUp(self):
self.fs = FileSystem()
self.setup_generic_test_dir()
self._this_dir = os.path.dirname(os.path.abspath(__file__))
self._missing_file = os.path.join(self._this_dir, 'missing_file.py')
self._this_file = os.path.join(self._this_dir, 'filesystem_unittest.py')
def main():
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s %(levelname)s %(name)s] %(message)s',
datefmt='%H:%M:%S')
parser = argparse.ArgumentParser(description='Blink source mover')
sub_parsers = parser.add_subparsers()
update_parser = sub_parsers.add_parser('update')
update_parser.set_defaults(command='update')
update_parser.add_argument(
'--run', dest='run', action='store_true', help='Update file contents')
move_parser = sub_parsers.add_parser('move')
move_parser.set_defaults(command='move')
move_parser.add_argument(
'--git',
dest='run_git',
action='store_true',
help='Run |git mv| command instead of |mv|.')
fixbranch_parser = sub_parsers.add_parser('fixbranch')
fixbranch_parser.set_defaults(command='fixbranch', run=True, run_git=True)
options = parser.parse_args()
mover = MoveBlinkSource(FileSystem(), options, get_chromium_src_dir())
if options.command == 'update':
mover.update()
elif options.command == 'move':
mover.move()
elif options.command == 'fixbranch':
mover.fix_branch()
def test_chdir__notexists(self):
fs = FileSystem()
newdir = '/dirdoesnotexist'
if sys.platform == 'win32':
newdir = 'c:\\dirdoesnotexist'
with self.assertRaises(OSError):
fs.chdir(newdir)
def setUp(self):
LoggingTestCase.setUp(self)
# FIXME: This should be a MockFileSystem once TextFileReader is moved entirely on top of FileSystem.
self.filesystem = FileSystem()
self._temp_dir = str(self.filesystem.mkdtemp())
self._processor = TextFileReaderTest.MockProcessor()
self._file_reader = TextFileReader(self.filesystem, self._processor)
def main():
fs = FileSystem()
file_pairs = plan_blink_move(fs, sys.argv[1:])
print 'Show renaming plan. It contains files not in the repository.'
print '<Source path relative to third_party/WebKit> => <Destination path relative to third_party/blink>'
for pair in file_pairs:
print '%s\t=>\t%s' % pair
def setUp(self):
self.executive = Executive()
self.filesystem = FileSystem()
self.original_cwd = self.filesystem.getcwd()
# Set up fresh git repository with one commit.
self.untracking_checkout_path = self._mkdtemp(
suffix='-git_unittest_untracking')
self._run(['git', 'init', self.untracking_checkout_path])
self._chdir(self.untracking_checkout_path)
self._set_user_config()
self._write_text_file('foo_file', 'foo')
self._run(['git', 'add', 'foo_file'])
self._run(['git', 'commit', '-am', 'dummy commit'])
self.untracking_git = Git(cwd=self.untracking_checkout_path,
filesystem=self.filesystem,
executive=self.executive)
# Then set up a second git repo that tracks the first one.
self.tracking_git_checkout_path = self._mkdtemp(
suffix='-git_unittest_tracking')
self._run([
'git', 'clone', '--quiet', self.untracking_checkout_path,
self.tracking_git_checkout_path
])
self._chdir(self.tracking_git_checkout_path)
self._set_user_config()
self.tracking_git = Git(cwd=self.tracking_git_checkout_path,
filesystem=self.filesystem,
executive=self.executive)
def test_walk(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(list(fs.walk(d)), [(d, [], [])])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(list(fs.walk(d)), [(d, [], ['foo'])])
os.remove(new_file)
def __init__(self, filesystem=None):
Merger.__init__(self)
self.filesystem = filesystem or FileSystem()
# Default to just checking the file contents matches.
self.add_helper(lambda *args: True, MergeFilesMatchingContents(self.filesystem))
# Copy the file it it's the only one.
self.add_helper(lambda _, to_merge: len(to_merge) == 1, MergeFilesOne(self.filesystem))
def test_chdir(self):
fs = FileSystem()
cwd = fs.getcwd()
newdir = '/'
if sys.platform == 'win32':
newdir = 'c:\\'
fs.chdir(newdir)
self.assertEqual(fs.getcwd(), newdir)
fs.chdir(cwd)
def __init__(self):
self.executable = sys.executable
self.executive = Executive()
self.filesystem = FileSystem()
self.user = User()
self.platform = PlatformInfo(sys, platform, self.filesystem,
self.executive)
self.stdin = sys.stdin
self.stdout = sys.stdout
self.stderr = sys.stderr
self.environ = os.environ
def __init__(self, expectations, port):
"""
Args:
expectations: a blinkpy.web_tests.models.test_expectations.TestExpectations object
port: a blinkpy.web_tests.port.Port object
"""
self.expectations = expectations
self.port = port
# TODO(lpz): Use self.fs everywhere in this class and add tests
self.fs = FileSystem()
self.wpt_manifest = self.port.wpt_manifest("external/wpt")
self.metadata_output_dir = ""
self.checked_in_metadata_dir = ""
def test_remove_file_with_retry(self):
RealFileSystemTest._remove_failures = 2
def remove_with_exception(filename):
RealFileSystemTest._remove_failures -= 1
if RealFileSystemTest._remove_failures >= 0:
try:
raise WindowsError
except NameError:
raise FileSystem._WindowsError
fs = FileSystem()
self.assertTrue(fs.remove('filename', remove_with_exception))
self.assertEqual(-1, RealFileSystemTest._remove_failures)
def mark_missing_shards(summary_json,
input_directories,
merged_output_json,
fs=None):
"""Merge the contents of one or more results JSONs into a single JSON.
Args:
summary_json: swarming summary containing shard info.
input_directories: A list of dir paths to JSON files that should be merged.
merged_output_json: A path to a JSON file to which the merged results should be
written.
fs: filesystem object - MockFileSystem or FileSystem.
"""
# summary.json is produced by swarming client.
if fs != None:
filesystem = fs
else:
filesystem = FileSystem()
try:
with filesystem.open_binary_file_for_reading(summary_json) as f:
summary = json.load(f)
except (IOError, ValueError) as e:
raise MergeFailure('summary_json is missing or can not be read',
summary_json, None)
missing_shards = []
_log.debug("Missing shard processing: %s", input_directories)
for index, result in enumerate(summary['shards']):
output_path = None
if result:
output_path = find_shard_output_path(index, result.get('task_id'),
input_directories)
if not output_path:
missing_shards.append(index)
if missing_shards:
# TODO(crbug.com/1111954) - process summary_json along with others
# so the merged output json can be written once to disk.
with filesystem.open_binary_file_for_reading(merged_output_json) as f:
try:
json_contents_merged = json.load(f)
except ValueError:
raise MergeFailure(
'Failed to parse JSON from merged output.json',
merged_output_json, None)
json_contents_merged['missing_shards'] = missing_shards
with filesystem.open_binary_file_for_writing(merged_output_json) as f:
MergeFilesJSONP.dump_jsonp(f, '', json_contents_merged, '')
def test_real_code(self):
# This test makes sure the real (unmocked) code actually works.
info = PlatformInfo(sys, platform, FileSystem(), Executive())
self.assertNotEquals(info.os_name, '')
self.assertNotEquals(info.os_version, '')
self.assertNotEquals(info.display_name(), '')
self.assertTrue(info.is_mac() or info.is_win() or info.is_linux()
or info.is_freebsd())
self.assertIsNotNone(info.terminal_width())
if info.is_linux():
self.assertIsNotNone(info.linux_distribution())
if info.is_mac():
self.assertTrue(info.total_bytes_memory() > 0)
else:
self.assertIsNone(info.total_bytes_memory())
def __init__(self, expectations, port):
"""
Args:
expectations: a blinkpy.web_tests.models.test_expectations.TestExpectations object
port: a blinkpy.web_tests.port.Port object
"""
self.expectations = expectations
self.port = port
# TODO(lpz): Use self.fs everywhere in this class and add tests
self.fs = FileSystem()
self.wpt_manifest = self.port.wpt_manifest("external/wpt")
self.metadata_output_dir = ""
self.checked_in_metadata_dir = ""
self.process_baselines = True
self.handle_annotations = True
self.checked_in_metadata_copied = set()
self.use_subtest_results = False
def test_read_and_write_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform == 'win32':
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
with self.assertRaises(OSError):
fs.maybe_make_directory(sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_maybe_make_directory__success(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as base_path:
sub_path = os.path.join(base_path, 'newdir')
self.assertFalse(os.path.exists(sub_path))
self.assertFalse(fs.isdir(sub_path))
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Make sure we can re-create it.
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Clean up.
os.rmdir(sub_path)
self.assertFalse(os.path.exists(base_path))
self.assertFalse(fs.isdir(base_path))
def run_pylint(self, path):
finder = PathFinder(FileSystem())
executive = Executive()
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join([
get_blink_tools_dir(),
finder.path_from_blink_source('build', 'scripts'),
get_blinkpy_thirdparty_dir(),
finder.path_from_blink_source('bindings', 'scripts'),
finder.path_from_chromium_base('build', 'android'),
finder.path_from_chromium_base('third_party', 'catapult', 'devil'),
finder.path_from_chromium_base('third_party', 'pymock'),
])
return executive.run_command([
sys.executable,
finder.path_from_depot_tools_base('pylint.py'),
'--output-format=parseable',
'--rcfile=' + finder.path_from_blink_tools('blinkpy', 'pylintrc'),
path,
],
env=env,
error_handler=executive.ignore_error)
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def setUp(self):
self.executive = Executive()
self.filesystem = FileSystem()
self.original_cwd = self.filesystem.getcwd()
# Set up fresh git repository with one commit.
self.untracking_checkout_path = self._mkdtemp(
suffix='-git_unittest_untracking')
self._run(['git', 'init', self.untracking_checkout_path])
self._chdir(self.untracking_checkout_path)
# Explicitly create the default branch instead of relying on
# init.defaultBranch. We don't use the new --initial-branch flag with
# `git init` to keep the tests compatible with older versions of git.
self._run(['git', 'checkout', '-b', 'master'])
self._set_user_config()
self._write_text_file('foo_file', 'foo')
self._run(['git', 'add', 'foo_file'])
self._run(['git', 'commit', '-am', 'dummy commit'])
self.untracking_git = Git(
cwd=self.untracking_checkout_path,
filesystem=self.filesystem,
executive=self.executive)
# Then set up a second git repo that tracks the first one.
self.tracking_git_checkout_path = self._mkdtemp(
suffix='-git_unittest_tracking')
self._run([
'git', 'clone', '--quiet', self.untracking_checkout_path,
self.tracking_git_checkout_path
])
self._chdir(self.tracking_git_checkout_path)
self._set_user_config()
self.tracking_git = Git(
cwd=self.tracking_git_checkout_path,
filesystem=self.filesystem,
executive=self.executive)
def __init__(self,
cwd=None,
executive=None,
filesystem=None,
platform=None):
self._executive = executive or Executive()
self._filesystem = filesystem or FileSystem()
self._executable_name = self.find_executable_name(
self._executive, platform)
self.cwd = cwd or self._filesystem.abspath(self._filesystem.getcwd())
if not self.in_working_directory(self.cwd):
module_directory = self._filesystem.abspath(
self._filesystem.dirname(
self._filesystem.path_to_module(self.__module__)))
_log.info(
'The current directory (%s) is not in a git repo, trying directory %s.',
cwd, module_directory)
if self.in_working_directory(module_directory):
self.cwd = module_directory
_log.error('Failed to find Git repo for %s or %s', cwd,
module_directory)
self.checkout_root = self.find_checkout_root(self.cwd)
def __init__(self):
self.web = Web()
self.builders = BuilderList.load_default_builder_list(FileSystem())
def test_isdir__true(self):
fs = FileSystem()
self.assertTrue(fs.isdir(self._this_dir))
def test_isdir__false(self):
fs = FileSystem()
self.assertFalse(fs.isdir(self._this_file))
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.path_finder = PathFinder(FileSystem())
def test_sep(self):
fs = FileSystem()
self.assertEqual(fs.sep, os.sep)
self.assertEqual(fs.join('foo', 'bar'),
os.path.join('foo', 'bar'))
def test_getcwd(self):
fs = FileSystem()
self.assertTrue(fs.exists(fs.getcwd()))
def main(argv):
parser = argparse.ArgumentParser()
parser.description = """\
Merges sharded web test results into a single output directory.
"""
parser.epilog = """\
If a post merge script is given, it will be run on the resulting merged output
directory. The script will be given the arguments plus
'--results_dir <output_directory>'.
"""
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output information about merging progress.')
parser.add_argument(
'--results-json-override-value',
nargs=2,
metavar=('KEY', 'VALUE'),
default=[],
action='append',
help='Override the value of a value in the result style JSON file '
'(--result-jsons-override-value layout_test_dirs /tmp/output).')
parser.add_argument(
'--results-json-allow-unknown-if-matching',
action='store_true',
default=False,
help='Allow unknown values in the result.json file as long as the '
'value match on all shards.')
parser.add_argument('--output-directory',
help='Directory to create the merged results in.')
parser.add_argument(
'--allow-existing-output-directory',
action='store_true',
default=False,
help='Allow merging results into a directory which already exists.')
parser.add_argument(
'--remove-existing-layout-test-results',
action='store_true',
default=False,
help='Remove existing layout test results from the output directory.')
parser.add_argument('--input-directories',
nargs='+',
help='Directories to merge the results from.')
# Swarming Isolated Merge Script API
# script.py \
# --build-properties /s/build.json \
# --output-json /tmp/output.json \
# --task-output-dir /path/to/task/output/dir \
# shard0/output.json \
# shard1/output.json
parser.add_argument(
'-o',
'--output-json',
help='(Swarming Isolated Merge Script API) Output JSON file to create.'
)
parser.add_argument(
'--build-properties',
help=
'(Swarming Isolated Merge Script API) Build property JSON file provided by recipes.'
)
parser.add_argument(
'--task-output-dir',
help=
'(Swarming Isolated Merge Script API) Directory containing all swarming task results.'
)
parser.add_argument(
'--results-json-override-with-build-property',
nargs=2,
metavar=('RESULT_JSON_KEY', 'BUILD_PROPERTY_KEY'),
default=[],
action='append',
help='Override the value of a value in the result style JSON file '
'(--result-jsons-override-value layout_test_dirs /tmp/output).')
parser.add_argument(
'--summary-json',
help=
'(Swarming Isolated Merge Script API) Summary of shard state running on swarming.'
'(Output of the swarming.py collect --task-summary-json=XXX command.)')
# Script to run after merging the directories together. Normally used with archive_layout_test_results.py
# scripts/slave/chromium/archive_layout_test_results.py \
# --results-dir /b/rr/tmpIcChUS/w/layout-test-results \
# --build-dir /b/rr/tmpIcChUS/w/src/out \
# --build-number 3665 \
# --builder-name 'WebKit Linux - RandomOrder' \
# --gs-bucket gs://chromium-layout-test-archives \
# --staging-dir /b/c/chrome_staging \
# --slave-utils-gsutil-py-path /b/rr/tmpIcChUS/rw/scripts/slave/.recipe_deps/depot_tools/gsutil.py
# in dir /b/rr/tmpIcChUS/w
parser.add_argument(
'--post-merge-script',
nargs='*',
help='Script to call after the results have been merged.')
# The position arguments depend on if we are using the isolated merge
# script API mode or not.
parser.add_argument('positional',
nargs='*',
help='output.json from shards.')
args = parser.parse_args(argv)
if args.verbose:
logging_level = logging.DEBUG
else:
logging_level = logging.INFO
configure_logging(logging_level=logging_level)
# Map the isolate arguments back to our output / input arguments.
if args.output_json:
logging.info('Running with isolated arguments')
assert args.positional
# TODO(tansell): Once removed everywhere, these lines can be removed.
# For now we just check nobody is supply arguments we didn't expect.
if args.results_json_override_with_build_property:
for result_key, build_prop_key in args.results_json_override_with_build_property:
assert (result_key, build_prop_key
) in RESULTS_JSON_VALUE_OVERRIDE_WITH_BUILD_PROPERTY, (
"%s not in %s" %
(result_key,
RESULTS_JSON_VALUE_OVERRIDE_WITH_BUILD_PROPERTY))
if not args.output_directory:
args.output_directory = os.getcwd()
args.allow_existing_output_directory = True
args.remove_existing_layout_test_results = True
assert not args.input_directories
args.input_directories = [os.path.dirname(f) for f in args.positional]
args.positional = []
# Allow skipping the --input-directories bit, for example,
# merge_web_test_results.py -o outputdir shard0 shard1 shard2
if args.positional and not args.input_directories:
args.input_directories = args.positional
if not args.output_directory:
args.output_directory = tempfile.mkdtemp(
suffix='_merged_web_test_results')
args.allow_existing_output_directory = True
assert args.output_directory
assert args.input_directories
results_json_value_overrides = {}
if args.build_properties:
build_properties = json.loads(args.build_properties)
for result_key, build_prop_key in RESULTS_JSON_VALUE_OVERRIDE_WITH_BUILD_PROPERTY:
if build_prop_key not in build_properties:
logging.warn('Required build property key "%s" was not found!',
build_prop_key)
continue
results_json_value_overrides[result_key] = build_properties[
build_prop_key]
logging.debug('results_json_value_overrides: %r',
results_json_value_overrides)
merger = WebTestDirMerger(
results_json_value_overrides=results_json_value_overrides,
results_json_allow_unknown_if_matching=args.
results_json_allow_unknown_if_matching)
ensure_empty_dir(FileSystem(),
args.output_directory,
allow_existing=args.allow_existing_output_directory,
remove_existing=args.remove_existing_layout_test_results)
merger.merge(args.output_directory, args.input_directories)
merged_output_json = os.path.join(args.output_directory, 'output.json')
if os.path.exists(merged_output_json) and args.output_json:
# process summary_json to mark missing shards.
mark_missing_shards(args.summary_json, args.input_directories,
merged_output_json)
logging.debug('Copying output.json from %s to %s', merged_output_json,
args.output_json)
shutil.copyfile(merged_output_json, args.output_json)
if args.post_merge_script:
logging.debug('Changing directory to %s', args.output_directory)
os.chdir(args.output_directory)
post_script = list(args.post_merge_script)
post_script.append('--result-dir', args.output_directory)
logging.info('Running post merge script %r', post_script)
os.execlp(post_script)
def test_read_text_file__missing(self):
fs = FileSystem()
with self.assertRaises(IOError):
fs.read_text_file(self._missing_file)