def test_convert_for_webkit_with_utf8(self):
files = {
'/file':
'foo',
'/mock-checkout/third_party/WebKit/Source/core/css/CSSProperties.in':
'',
}
host = MockSystemHost(filesystem=MockFileSystem(files=files))
convert_for_webkit('', '/file', '', host)
def copy_file(self, file_to_copy, dest_dir):
"""Converts and copies a file, if it should be copied.
Args:
file_to_copy: A dict in a file copy list constructed by
find_importable_tests, which represents one file to copy, including
the keys:
"src": Absolute path to the source location of the file.
"destination": File name of the destination file.
And possibly also the keys "reference_support_info" or "is_jstest".
dest_dir: Path to the directory where the file should be copied.
Returns:
The path to the new file, relative to the Blink root (//third_party/WebKit).
"""
source_path = self.filesystem.normpath(file_to_copy['src'])
dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])
if self.filesystem.isdir(source_path):
_log.error('%s refers to a directory', source_path)
return None
if not self.filesystem.exists(source_path):
_log.error('%s not found. Possible error in the test.',
source_path)
return None
reference_support_info = file_to_copy.get(
'reference_support_info') or None
if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
if not self.import_in_place:
self.filesystem.maybe_make_directory(
self.filesystem.dirname(dest_path))
relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.debug(' copying %s', relpath)
if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
converted_file = convert_for_webkit(
dest_dir,
filename=source_path,
reference_support_info=reference_support_info,
host=self.host)
for prefixed_property in converted_file[0]:
self._prefixed_properties.setdefault(prefixed_property, 0)
self._prefixed_properties[prefixed_property] += 1
self.filesystem.write_text_file(dest_path, converted_file[1])
else:
if not self.import_in_place:
self.filesystem.copyfile(source_path, dest_path)
if self.filesystem.read_binary_file(source_path)[:2] == '#!':
self.filesystem.make_executable(dest_path)
return dest_path.replace(self._webkit_root, '')
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = os.path.relpath(orig_path, self.top_of_repo)
new_path = os.path.join(self.destination_directory, subpath)
if not(os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy['src'])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not(os.path.exists(orig_filepath)):
_log.warning('%s not found. Possible error in the test.', orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy['dest'])
if not(os.path.exists(os.path.dirname(new_filepath))):
if not self.import_in_place and not self.options.dry_run:
os.makedirs(os.path.dirname(new_filepath))
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info(' skipping import of existing file ' + new_filepath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' importing %s', os.path.relpath(new_filepath, self.layout_tests_dir))
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
converted_file = convert_for_webkit(new_path, filename=orig_filepath)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, 'wb')
outfile.write(converted_file[1])
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ''))
if not self.import_in_place and not self.options.dry_run:
self.remove_deleted_files(new_path, copied_files)
self.write_import_log(new_path, copied_files, prefixed_properties)
_log.info('')
_log.info('Import complete')
_log.info('')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
_log.info('')
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
total_prefixed_property_values = {}
failed_conversion_files = []
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
prefixed_property_values = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = os.path.relpath(orig_path, self.source_directory)
new_path = os.path.join(self.destination_directory, subpath)
if not(os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy['src'])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not(os.path.exists(orig_filepath)):
_log.warning('%s not found. Possible error in the test.', orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy['dest'])
if 'reference_support_info' in file_to_copy.keys() and file_to_copy['reference_support_info'] != {}:
reference_support_info = file_to_copy['reference_support_info']
else:
reference_support_info = None
if not(os.path.exists(os.path.dirname(new_filepath))):
os.makedirs(os.path.dirname(new_filepath))
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info('Skipping import of existing file ' + new_filepath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info('Importing: %s', orig_filepath)
_log.info(' As: %s', new_filepath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
try:
converted_file = convert_for_webkit(new_path, filename=orig_filepath, reference_support_info=reference_support_info, convert_test_harness_links=self.options.convert_test_harness_links)
except:
_log.warn('Failed converting %s', orig_filepath)
failed_conversion_files.append(orig_filepath)
converted_file = None
if not converted_file:
shutil.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
for prefixed_value in converted_file[1]:
total_prefixed_property_values.setdefault(prefixed_value, 0)
total_prefixed_property_values[prefixed_value] += 1
prefixed_property_values.extend(set(converted_file[1]) - set(prefixed_property_values))
outfile = open(new_filepath, 'wb')
outfile.write(converted_file[2])
outfile.close()
elif orig_filepath.endswith('__init__.py') and not self.filesystem.getsize(orig_filepath):
# Some bots dislike empty __init__.py.
self.filesystem.write_text_file(new_filepath, '# This file is required for Python to search this directory for modules.')
else:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ''))
self.remove_deleted_files(new_path, copied_files)
self.write_import_log(new_path, copied_files, prefixed_properties, prefixed_property_values)
_log.info('Import complete')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
if len(failed_conversion_files):
_log.warn('Failed converting %d files (files copied without being converted)', len(failed_conversion_files))
_log.info('')
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
_log.info('')
_log.info('Property values needing prefixes (by count):')
for prefixed_value in sorted(total_prefixed_property_values, key=lambda p: total_prefixed_property_values[p]):
_log.info(' %s: %s', prefixed_value, total_prefixed_property_values[prefixed_value])
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy["total_tests"]
total_imported_reftests += dir_to_copy["reftests"]
total_imported_jstests += dir_to_copy["jstests"]
prefixed_properties = []
if not dir_to_copy["copy_list"]:
continue
orig_path = dir_to_copy["dirname"]
subpath = os.path.relpath(orig_path, self.top_of_repo)
new_path = os.path.join(self.destination_directory, subpath)
if not (os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy["copy_list"]:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy["src"])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error("%s refers to a directory" % orig_filepath)
continue
if not (os.path.exists(orig_filepath)):
_log.warning("%s not found. Possible error in the test.", orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy["dest"])
if "reference_support_info" in file_to_copy.keys() and file_to_copy["reference_support_info"] != {}:
reference_support_info = file_to_copy["reference_support_info"]
else:
reference_support_info = None
if not (os.path.exists(os.path.dirname(new_filepath))):
if not self.import_in_place and not self.options.dry_run:
os.makedirs(os.path.dirname(new_filepath))
relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info(" skipping %s" % relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(" %s" % relpath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if "html" in str(mimetype[0]) or "xml" in str(mimetype[0]) or "css" in str(mimetype[0]):
converted_file = convert_for_webkit(
new_path, filename=orig_filepath, reference_support_info=reference_support_info
)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, "wb")
outfile.write(converted_file[1])
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ""))
_log.info("")
_log.info("Import complete")
_log.info("")
_log.info("IMPORTED %d TOTAL TESTS", total_imported_tests)
_log.info("Imported %d reftests", total_imported_reftests)
_log.info("Imported %d JS tests", total_imported_jstests)
_log.info(
"Imported %d pixel/manual tests", total_imported_tests - total_imported_jstests - total_imported_reftests
)
_log.info("")
if total_prefixed_properties:
_log.info("Properties needing prefixes (by count):")
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(" %s: %s", prefixed_property, total_prefixed_properties[prefixed_property])
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
total_prefixed_property_values = {}
failed_conversion_files = []
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
prefixed_property_values = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = self.filesystem.relpath(orig_path, self.source_directory)
new_path = self.filesystem.join(self.destination_directory,
subpath)
if not (self.filesystem.exists(new_path)):
self.filesystem.maybe_make_directory(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = self.filesystem.normpath(file_to_copy['src'])
if self.filesystem.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not (self.filesystem.exists(orig_filepath)):
_log.warning('%s not found. Possible error in the test.',
orig_filepath)
continue
new_filepath = self.filesystem.join(new_path,
file_to_copy['dest'])
if 'reference_support_info' in file_to_copy.keys(
) and file_to_copy['reference_support_info'] != {}:
reference_support_info = file_to_copy[
'reference_support_info']
else:
reference_support_info = None
if not (self.filesystem.exists(
self.filesystem.dirname(new_filepath))):
self.filesystem.maybe_make_directory(
self.filesystem.dirname(new_filepath))
if not self.options.overwrite and self.filesystem.exists(
new_filepath):
_log.info('Skipping import of existing file ' +
new_filepath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info('Importing: %s', orig_filepath)
_log.info(' As: %s', new_filepath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(
mimetype[0]) or 'css' in str(mimetype[0]):
try:
converted_file = convert_for_webkit(
new_path,
filename=orig_filepath,
reference_support_info=reference_support_info,
host=self.host,
convert_test_harness_links=self.
should_convert_test_harness_links(subpath))
except:
_log.warn('Failed converting %s', orig_filepath)
failed_conversion_files.append(orig_filepath)
converted_file = None
if not converted_file:
self.filesystem.copyfile(
orig_filepath,
new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(
prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(
set(converted_file[0]) - set(prefixed_properties))
for prefixed_value in converted_file[1]:
total_prefixed_property_values.setdefault(
prefixed_value, 0)
total_prefixed_property_values[prefixed_value] += 1
prefixed_property_values.extend(
set(converted_file[1]) -
set(prefixed_property_values))
self.filesystem.write_binary_file(
new_filepath, converted_file[2])
elif orig_filepath.endswith(
'__init__.py'
) and not self.filesystem.getsize(orig_filepath):
# Some bots dislike empty __init__.py.
self.write_init_py(new_filepath)
else:
self.filesystem.copyfile(orig_filepath, new_filepath)
self.write_html_files_for_templated_js_tests(
orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(
self._webkit_root, ''))
self.remove_deleted_files(new_path, copied_files)
self.write_import_log(new_path, copied_files, prefixed_properties,
prefixed_property_values)
_log.info('Import complete')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info(
'Imported %d pixel/manual tests', total_imported_tests -
total_imported_jstests - total_imported_reftests)
if len(failed_conversion_files):
_log.warn(
'Failed converting %d files (files copied without being converted)',
len(failed_conversion_files))
_log.info('')
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(
total_prefixed_properties,
key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property,
total_prefixed_properties[prefixed_property])
_log.info('')
_log.info('Property values needing prefixes (by count):')
for prefixed_value in sorted(
total_prefixed_property_values,
key=lambda p: total_prefixed_property_values[p]):
_log.info(' %s: %s', prefixed_value,
total_prefixed_property_values[prefixed_value])
if self._potential_test_resource_files and self._test_resource_files:
# FIXME: We should check that actual tests are not in the test_resource_files list
should_update_json_file = False
files = self._test_resource_files["files"]
for full_path in self._potential_test_resource_files:
resource_file_path = self.filesystem.relpath(
full_path, self.source_directory)
if not self._already_identified_as_resource_file(
resource_file_path):
files.append(resource_file_path)
should_update_json_file = True
if should_update_json_file:
files.sort()
self.filesystem.write_text_file(
self._test_resource_files_json_path,
json.dumps(self._test_resource_files,
sort_keys=True,
indent=4).replace(' \n', '\n'))
if self._tests_options:
self.update_tests_options()
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = os.path.relpath(orig_path, self.top_of_repo)
new_path = os.path.join(self.destination_directory, subpath)
if not (os.path.exists(new_path)):
os.makedirs(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = os.path.normpath(file_to_copy['src'])
if os.path.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not (os.path.exists(orig_filepath)):
_log.warning('%s not found. Possible error in the test.',
orig_filepath)
continue
new_filepath = os.path.join(new_path, file_to_copy['dest'])
if 'reference_support_info' in file_to_copy.keys(
) and file_to_copy['reference_support_info'] != {}:
reference_support_info = file_to_copy[
'reference_support_info']
else:
reference_support_info = None
if not (os.path.exists(os.path.dirname(new_filepath))):
if not self.import_in_place and not self.options.dry_run:
os.makedirs(os.path.dirname(new_filepath))
relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
if not self.options.overwrite and os.path.exists(new_filepath):
_log.info(' skipping %s' % relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' %s' % relpath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(
mimetype[0]) or 'css' in str(mimetype[0]):
converted_file = convert_for_webkit(
new_path,
filename=orig_filepath,
reference_support_info=reference_support_info)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(
orig_filepath,
new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(
prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(
set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, 'wb')
outfile.write(converted_file[1])
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
shutil.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(
self._webkit_root, ''))
_log.info('')
_log.info('Import complete')
_log.info('')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info(
'Imported %d pixel/manual tests', total_imported_tests -
total_imported_jstests - total_imported_reftests)
_log.info('')
if total_prefixed_properties:
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(
total_prefixed_properties,
key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property,
total_prefixed_properties[prefixed_property])
def test_convert_for_webkit_with_utf8(self):
files = {'/file': 'foo',
'/mock-checkout/third_party/WebKit/Source/core/css/CSSProperties.in': '', }
host = MockSystemHost(filesystem=MockFileSystem(files=files))
convert_for_webkit('', '/file', '', host)
def test_convert_for_webkit_with_utf8(self):
files = {'/file': 'foo', }
host = MockSystemHost(filesystem=MockFileSystem(files=files))
convert_for_webkit('', '/file', '', host)
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = self.filesystem.relpath(orig_path, self.top_of_repo)
new_path = self.filesystem.join(self.destination_directory, subpath)
if not self.filesystem.exists(new_path):
self.filesystem.maybe_make_directory(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = self.filesystem.normpath(file_to_copy['src'])
if self.filesystem.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory' % orig_filepath)
continue
if not self.filesystem.exists(orig_filepath):
_log.warning('%s not found. Possible error in the test.', orig_filepath)
continue
if self.path_too_long(orig_filepath):
_log.warning('%s skipped (longer than %d chars), to avoid hitting Windows max path length on builders (http://crbug.com/609871).',
orig_filepath, MAX_PATH_LENGTH)
continue
new_filepath = self.filesystem.join(new_path, file_to_copy['dest'])
if 'reference_support_info' in file_to_copy.keys() and file_to_copy['reference_support_info'] != {}:
reference_support_info = file_to_copy['reference_support_info']
else:
reference_support_info = None
if not self.filesystem.exists(self.filesystem.dirname(new_filepath)):
if not self.import_in_place and not self.options.dry_run:
self.filesystem.maybe_make_directory(self.filesystem.dirname(new_filepath))
relpath = self.filesystem.relpath(new_filepath, self.layout_tests_dir)
if not self.options.overwrite and self.filesystem.exists(new_filepath):
_log.info(' skipping %s' % relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' %s' % relpath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
converted_file = convert_for_webkit(new_path, filename=orig_filepath,
reference_support_info=reference_support_info)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
outfile = open(new_filepath, 'wb')
outfile.write(converted_file[1].encode('utf-8'))
outfile.close()
else:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ''))
_log.info('')
_log.info('Import complete')
_log.info('')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
_log.info('')
if total_prefixed_properties:
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
def import_tests(self):
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
total_prefixed_property_values = {}
failed_conversion_files = []
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy["total_tests"]
total_imported_reftests += dir_to_copy["reftests"]
total_imported_jstests += dir_to_copy["jstests"]
prefixed_properties = []
prefixed_property_values = []
if not dir_to_copy["copy_list"]:
continue
orig_path = dir_to_copy["dirname"]
subpath = self.filesystem.relpath(orig_path, self.source_directory)
new_path = self.filesystem.join(self.destination_directory, subpath)
if not (self.filesystem.exists(new_path)):
self.filesystem.maybe_make_directory(new_path)
copied_files = []
for file_to_copy in dir_to_copy["copy_list"]:
# FIXME: Split this block into a separate function.
orig_filepath = self.filesystem.normpath(file_to_copy["src"])
if self.filesystem.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error("%s refers to a directory" % orig_filepath)
continue
if not (self.filesystem.exists(orig_filepath)):
_log.warning("%s not found. Possible error in the test.", orig_filepath)
continue
new_filepath = self.filesystem.join(new_path, file_to_copy["dest"])
if "reference_support_info" in file_to_copy.keys() and file_to_copy["reference_support_info"] != {}:
reference_support_info = file_to_copy["reference_support_info"]
else:
reference_support_info = None
if not (self.filesystem.exists(self.filesystem.dirname(new_filepath))):
self.filesystem.maybe_make_directory(self.filesystem.dirname(new_filepath))
if not self.options.overwrite and self.filesystem.exists(new_filepath):
_log.info("Skipping import of existing file " + new_filepath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info("Importing: %s", orig_filepath)
_log.info(" As: %s", new_filepath)
# Only html, xml, or css should be converted
# FIXME: Eventually, so should js when support is added for this type of conversion
mimetype = mimetypes.guess_type(orig_filepath)
if "html" in str(mimetype[0]) or "xml" in str(mimetype[0]) or "css" in str(mimetype[0]):
try:
converted_file = convert_for_webkit(
new_path,
filename=orig_filepath,
reference_support_info=reference_support_info,
host=self.host,
convert_test_harness_links=self.should_convert_test_harness_links(subpath),
)
except:
_log.warn("Failed converting %s", orig_filepath)
failed_conversion_files.append(orig_filepath)
converted_file = None
if not converted_file:
self.filesystem.copyfile(orig_filepath, new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
for prefixed_value in converted_file[1]:
total_prefixed_property_values.setdefault(prefixed_value, 0)
total_prefixed_property_values[prefixed_value] += 1
prefixed_property_values.extend(set(converted_file[1]) - set(prefixed_property_values))
self.filesystem.write_binary_file(new_filepath, converted_file[2])
elif orig_filepath.endswith("__init__.py") and not self.filesystem.getsize(orig_filepath):
# Some bots dislike empty __init__.py.
self.filesystem.write_text_file(
new_filepath, "# This file is required for Python to search this directory for modules."
)
else:
self.filesystem.copyfile(orig_filepath, new_filepath)
copied_files.append(new_filepath.replace(self._webkit_root, ""))
self.remove_deleted_files(new_path, copied_files)
self.write_import_log(new_path, copied_files, prefixed_properties, prefixed_property_values)
_log.info("Import complete")
_log.info("IMPORTED %d TOTAL TESTS", total_imported_tests)
_log.info("Imported %d reftests", total_imported_reftests)
_log.info("Imported %d JS tests", total_imported_jstests)
_log.info(
"Imported %d pixel/manual tests", total_imported_tests - total_imported_jstests - total_imported_reftests
)
if len(failed_conversion_files):
_log.warn("Failed converting %d files (files copied without being converted)", len(failed_conversion_files))
_log.info("")
_log.info("Properties needing prefixes (by count):")
for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
_log.info(" %s: %s", prefixed_property, total_prefixed_properties[prefixed_property])
_log.info("")
_log.info("Property values needing prefixes (by count):")
for prefixed_value in sorted(total_prefixed_property_values, key=lambda p: total_prefixed_property_values[p]):
_log.info(" %s: %s", prefixed_value, total_prefixed_property_values[prefixed_value])
if self._potential_test_resource_files:
_log.info(
"The following files may be resource files and should be marked as skipped in the TestExpectations:"
)
for filename in sorted([test["src"] for test in self._potential_test_resource_files]):
_log.info(filename.replace(self.source_directory, self.tests_w3c_relative_path) + " [ Skip ]")
def copy_file(self, file_to_copy, dest_dir):
"""Converts and copies a file, if it should be copied.
Args:
file_to_copy: A dict in a file copy list constructed by
find_importable_tests, which represents one file to copy, including
the keys:
"src": Absolute path to the source location of the file.
"destination": File name of the destination file.
And possibly also the keys "reference_support_info" or "is_jstest".
dest_dir: Path to the directory where the file should be copied.
Returns:
The path to the new file, relative to the Blink root (//third_party/WebKit).
"""
source_path = self.filesystem.normpath(file_to_copy['src'])
dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])
if self.filesystem.isdir(source_path):
_log.error('%s refers to a directory', source_path)
return None
if not self.filesystem.exists(source_path):
_log.error('%s not found. Possible error in the test.', source_path)
return None
if file_to_copy.get('reference_support_info'):
reference_support_info = file_to_copy['reference_support_info']
else:
reference_support_info = None
if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
if not self.import_in_place and not self.options.dry_run:
self.filesystem.maybe_make_directory(self.filesystem.dirname(dest_path))
relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
if not self.options.overwrite and self.filesystem.exists(dest_path):
_log.info(' skipping %s', relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' %s', relpath)
if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
converted_file = convert_for_webkit(
dest_dir, filename=source_path,
reference_support_info=reference_support_info,
host=self.host)
for prefixed_property in converted_file[0]:
self._prefixed_properties.setdefault(prefixed_property, 0)
self._prefixed_properties[prefixed_property] += 1
if not self.options.dry_run:
self.filesystem.write_text_file(dest_path, converted_file[1])
else:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(source_path, dest_path)
if self.filesystem.read_binary_file(source_path)[:2] == '#!':
self.filesystem.make_executable(dest_path)
return dest_path.replace(self._webkit_root, '')
def import_tests(self):
"""Reads |self.import_list|, and converts and copies files to their destination."""
total_imported_tests = 0
total_imported_reftests = 0
total_imported_jstests = 0
total_prefixed_properties = {}
for dir_to_copy in self.import_list:
total_imported_tests += dir_to_copy['total_tests']
total_imported_reftests += dir_to_copy['reftests']
total_imported_jstests += dir_to_copy['jstests']
prefixed_properties = []
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
subpath = self.filesystem.relpath(orig_path, self.source_repo_path)
new_path = self.filesystem.join(self.destination_directory,
subpath)
if not self.filesystem.exists(new_path):
self.filesystem.maybe_make_directory(new_path)
copied_files = []
for file_to_copy in dir_to_copy['copy_list']:
# FIXME: Split this block into a separate function.
orig_filepath = self.filesystem.normpath(file_to_copy['src'])
if self.filesystem.isdir(orig_filepath):
# FIXME: Figure out what is triggering this and what to do about it.
_log.error('%s refers to a directory', orig_filepath)
continue
if not self.filesystem.exists(orig_filepath):
_log.error('%s not found. Possible error in the test.',
orig_filepath)
continue
new_filepath = self.filesystem.join(new_path,
file_to_copy['dest'])
if 'reference_support_info' in file_to_copy.keys(
) and file_to_copy['reference_support_info'] != {}:
reference_support_info = file_to_copy[
'reference_support_info']
else:
reference_support_info = None
if not self.filesystem.exists(
self.filesystem.dirname(new_filepath)):
if not self.import_in_place and not self.options.dry_run:
self.filesystem.maybe_make_directory(
self.filesystem.dirname(new_filepath))
relpath = self.filesystem.relpath(new_filepath,
self.layout_tests_dir)
if not self.options.overwrite and self.filesystem.exists(
new_filepath):
_log.info(' skipping %s', relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' %s', relpath)
# Only HTML, XML, or CSS should be converted.
# FIXME: Eventually, so should JS when support is added for this type of conversion.
mimetype = mimetypes.guess_type(orig_filepath)
if 'is_jstest' not in file_to_copy and (
'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])
or 'css' in str(mimetype[0])):
converted_file = convert_for_webkit(
new_path,
filename=orig_filepath,
reference_support_info=reference_support_info,
host=self.host)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(
orig_filepath,
new_filepath) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
total_prefixed_properties.setdefault(
prefixed_property, 0)
total_prefixed_properties[prefixed_property] += 1
prefixed_properties.extend(
set(converted_file[0]) - set(prefixed_properties))
if not self.options.dry_run:
self.filesystem.write_text_file(
new_filepath, converted_file[1])
else:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(orig_filepath, new_filepath)
if self.filesystem.read_binary_file(
orig_filepath)[:2] == '#!':
self.filesystem.make_executable(new_filepath)
copied_files.append(new_filepath.replace(
self._webkit_root, ''))
_log.info('')
_log.info('Import complete')
_log.info('')
_log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
_log.info('Imported %d reftests', total_imported_reftests)
_log.info('Imported %d JS tests', total_imported_jstests)
_log.info(
'Imported %d pixel/manual tests', total_imported_tests -
total_imported_jstests - total_imported_reftests)
_log.info('')
if total_prefixed_properties:
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(
total_prefixed_properties,
key=lambda p: total_prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property,
total_prefixed_properties[prefixed_property])
def copy_file(self, file_to_copy, dest_dir):
"""Converts and copies a file, if it should be copied.
Args:
file_to_copy: A dict in a file copy list constructed by
find_importable_tests, which represents one file to copy, including
the keys:
"src": Absolute path to the source location of the file.
"destination": File name of the destination file.
And possibly also the keys "reference_support_info" or "is_jstest".
dest_dir: Path to the directory where the file should be copied.
Returns:
The path to the new file, relative to the Blink root (//third_party/WebKit).
"""
source_path = self.filesystem.normpath(file_to_copy['src'])
dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])
if self.filesystem.isdir(source_path):
_log.error('%s refers to a directory', source_path)
return None
if not self.filesystem.exists(source_path):
_log.error('%s not found. Possible error in the test.',
source_path)
return None
if file_to_copy.get('reference_support_info'):
reference_support_info = file_to_copy['reference_support_info']
else:
reference_support_info = None
if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
if not self.import_in_place and not self.options.dry_run:
self.filesystem.maybe_make_directory(
self.filesystem.dirname(dest_path))
relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
if not self.options.overwrite and self.filesystem.exists(dest_path):
_log.info(' skipping %s', relpath)
else:
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.info(' %s', relpath)
# Only HTML, XML, or CSS should be converted.
# FIXME: Eventually, so should JS when support is added for this type of conversion.
mimetype = mimetypes.guess_type(source_path)
if 'is_jstest' not in file_to_copy and ('html' in str(mimetype[0])
or 'xml' in str(mimetype[0])
or 'css' in str(mimetype[0])):
converted_file = convert_for_webkit(
dest_dir,
filename=source_path,
reference_support_info=reference_support_info,
host=self.host)
if not converted_file:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(
source_path, dest_path) # The file was unmodified.
else:
for prefixed_property in converted_file[0]:
self._prefixed_properties.setdefault(prefixed_property, 0)
self._prefixed_properties[prefixed_property] += 1
if not self.options.dry_run:
self.filesystem.write_text_file(dest_path,
converted_file[1])
else:
if not self.import_in_place and not self.options.dry_run:
self.filesystem.copyfile(source_path, dest_path)
if self.filesystem.read_binary_file(source_path)[:2] == '#!':
self.filesystem.make_executable(dest_path)
return dest_path.replace(self._webkit_root, '')