Ejemplo n.º 1
0
 def test_convert_for_webkit_with_utf8(self):
     files = {
         '/file':
         'foo',
         '/mock-checkout/third_party/WebKit/Source/core/css/CSSProperties.in':
         '',
     }
     host = MockSystemHost(filesystem=MockFileSystem(files=files))
     convert_for_webkit('', '/file', '', host)
Ejemplo n.º 2
0
    def copy_file(self, file_to_copy, dest_dir):
        """Converts and copies a file, if it should be copied.

        Args:
            file_to_copy: A dict in a file copy list constructed by
                find_importable_tests, which represents one file to copy, including
                the keys:
                    "src": Absolute path to the source location of the file.
                    "destination": File name of the destination file.
                And possibly also the keys "reference_support_info" or "is_jstest".
            dest_dir: Path to the directory where the file should be copied.

        Returns:
            The path to the new file, relative to the Blink root (//third_party/WebKit).
        """
        source_path = self.filesystem.normpath(file_to_copy['src'])
        dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])

        if self.filesystem.isdir(source_path):
            _log.error('%s refers to a directory', source_path)
            return None

        if not self.filesystem.exists(source_path):
            _log.error('%s not found. Possible error in the test.',
                       source_path)
            return None

        reference_support_info = file_to_copy.get(
            'reference_support_info') or None

        if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
            if not self.import_in_place:
                self.filesystem.maybe_make_directory(
                    self.filesystem.dirname(dest_path))

        relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
        # FIXME: Maybe doing a file diff is in order here for existing files?
        # In other words, there's no sense in overwriting identical files, but
        # there's no harm in copying the identical thing.
        _log.debug('  copying %s', relpath)

        if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
            converted_file = convert_for_webkit(
                dest_dir,
                filename=source_path,
                reference_support_info=reference_support_info,
                host=self.host)
            for prefixed_property in converted_file[0]:
                self._prefixed_properties.setdefault(prefixed_property, 0)
                self._prefixed_properties[prefixed_property] += 1

            self.filesystem.write_text_file(dest_path, converted_file[1])
        else:
            if not self.import_in_place:
                self.filesystem.copyfile(source_path, dest_path)
                if self.filesystem.read_binary_file(source_path)[:2] == '#!':
                    self.filesystem.make_executable(dest_path)

        return dest_path.replace(self._webkit_root, '')
Ejemplo n.º 3
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = os.path.relpath(orig_path, self.top_of_repo)
            new_path = os.path.join(self.destination_directory, subpath)

            if not(os.path.exists(new_path)):
                os.makedirs(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = os.path.normpath(file_to_copy['src'])

                if os.path.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory' % orig_filepath)
                    continue

                if not(os.path.exists(orig_filepath)):
                    _log.warning('%s not found. Possible error in the test.', orig_filepath)
                    continue

                new_filepath = os.path.join(new_path, file_to_copy['dest'])

                if not(os.path.exists(os.path.dirname(new_filepath))):
                    if not self.import_in_place and not self.options.dry_run:
                        os.makedirs(os.path.dirname(new_filepath))

                if not self.options.overwrite and os.path.exists(new_filepath):
                    _log.info('  skipping import of existing file ' + new_filepath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('  importing %s', os.path.relpath(new_filepath, self.layout_tests_dir))

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])  or 'css' in str(mimetype[0]):
                    converted_file = convert_for_webkit(new_path, filename=orig_filepath)

                    if not converted_file:
                        if not self.import_in_place and not self.options.dry_run:
                            shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
                        if not self.options.dry_run:
                            outfile = open(new_filepath, 'wb')
                            outfile.write(converted_file[1])
                            outfile.close()
                else:
                    if not self.import_in_place and not self.options.dry_run:
                        shutil.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(self._webkit_root, ''))

            if not self.import_in_place and not self.options.dry_run:
                self.remove_deleted_files(new_path, copied_files)
                self.write_import_log(new_path, copied_files, prefixed_properties)

        _log.info('')
        _log.info('Import complete')
        _log.info('')
        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
        _log.info('')
        _log.info('Properties needing prefixes (by count):')
        for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
            _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
Ejemplo n.º 4
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}
        total_prefixed_property_values = {}

        failed_conversion_files = []

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []
            prefixed_property_values = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = os.path.relpath(orig_path, self.source_directory)
            new_path = os.path.join(self.destination_directory, subpath)

            if not(os.path.exists(new_path)):
                os.makedirs(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = os.path.normpath(file_to_copy['src'])

                if os.path.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory' % orig_filepath)
                    continue

                if not(os.path.exists(orig_filepath)):
                    _log.warning('%s not found. Possible error in the test.', orig_filepath)
                    continue

                new_filepath = os.path.join(new_path, file_to_copy['dest'])
                if 'reference_support_info' in file_to_copy.keys() and file_to_copy['reference_support_info'] != {}:
                    reference_support_info = file_to_copy['reference_support_info']
                else:
                    reference_support_info = None

                if not(os.path.exists(os.path.dirname(new_filepath))):
                    os.makedirs(os.path.dirname(new_filepath))

                if not self.options.overwrite and os.path.exists(new_filepath):
                    _log.info('Skipping import of existing file ' + new_filepath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('Importing: %s', orig_filepath)
                    _log.info('       As: %s', new_filepath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])  or 'css' in str(mimetype[0]):
                    try:
                        converted_file = convert_for_webkit(new_path, filename=orig_filepath, reference_support_info=reference_support_info, convert_test_harness_links=self.options.convert_test_harness_links)
                    except:
                        _log.warn('Failed converting %s', orig_filepath)
                        failed_conversion_files.append(orig_filepath)
                        converted_file = None

                    if not converted_file:
                        shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))

                        for prefixed_value in converted_file[1]:
                            total_prefixed_property_values.setdefault(prefixed_value, 0)
                            total_prefixed_property_values[prefixed_value] += 1

                        prefixed_property_values.extend(set(converted_file[1]) - set(prefixed_property_values))

                        outfile = open(new_filepath, 'wb')
                        outfile.write(converted_file[2])
                        outfile.close()
                elif orig_filepath.endswith('__init__.py') and not self.filesystem.getsize(orig_filepath):
                    # Some bots dislike empty __init__.py.
                    self.filesystem.write_text_file(new_filepath, '# This file is required for Python to search this directory for modules.')
                else:
                    shutil.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(self._webkit_root, ''))

            self.remove_deleted_files(new_path, copied_files)
            self.write_import_log(new_path, copied_files, prefixed_properties, prefixed_property_values)

        _log.info('Import complete')

        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
        if len(failed_conversion_files):
            _log.warn('Failed converting %d files (files copied without being converted)', len(failed_conversion_files))
        _log.info('')
        _log.info('Properties needing prefixes (by count):')

        for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
            _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
        _log.info('')
        _log.info('Property values needing prefixes (by count):')

        for prefixed_value in sorted(total_prefixed_property_values, key=lambda p: total_prefixed_property_values[p]):
            _log.info('  %s: %s', prefixed_value, total_prefixed_property_values[prefixed_value])
Ejemplo n.º 5
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy["total_tests"]
            total_imported_reftests += dir_to_copy["reftests"]
            total_imported_jstests += dir_to_copy["jstests"]

            prefixed_properties = []

            if not dir_to_copy["copy_list"]:
                continue

            orig_path = dir_to_copy["dirname"]

            subpath = os.path.relpath(orig_path, self.top_of_repo)
            new_path = os.path.join(self.destination_directory, subpath)

            if not (os.path.exists(new_path)):
                os.makedirs(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy["copy_list"]:
                # FIXME: Split this block into a separate function.
                orig_filepath = os.path.normpath(file_to_copy["src"])

                if os.path.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error("%s refers to a directory" % orig_filepath)
                    continue

                if not (os.path.exists(orig_filepath)):
                    _log.warning("%s not found. Possible error in the test.", orig_filepath)
                    continue

                new_filepath = os.path.join(new_path, file_to_copy["dest"])
                if "reference_support_info" in file_to_copy.keys() and file_to_copy["reference_support_info"] != {}:
                    reference_support_info = file_to_copy["reference_support_info"]
                else:
                    reference_support_info = None

                if not (os.path.exists(os.path.dirname(new_filepath))):
                    if not self.import_in_place and not self.options.dry_run:
                        os.makedirs(os.path.dirname(new_filepath))

                relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
                if not self.options.overwrite and os.path.exists(new_filepath):
                    _log.info("  skipping %s" % relpath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info("  %s" % relpath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if "html" in str(mimetype[0]) or "xml" in str(mimetype[0]) or "css" in str(mimetype[0]):
                    converted_file = convert_for_webkit(
                        new_path, filename=orig_filepath, reference_support_info=reference_support_info
                    )

                    if not converted_file:
                        if not self.import_in_place and not self.options.dry_run:
                            shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
                        if not self.options.dry_run:
                            outfile = open(new_filepath, "wb")
                            outfile.write(converted_file[1])
                            outfile.close()
                else:
                    if not self.import_in_place and not self.options.dry_run:
                        shutil.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(self._webkit_root, ""))

        _log.info("")
        _log.info("Import complete")
        _log.info("")
        _log.info("IMPORTED %d TOTAL TESTS", total_imported_tests)
        _log.info("Imported %d reftests", total_imported_reftests)
        _log.info("Imported %d JS tests", total_imported_jstests)
        _log.info(
            "Imported %d pixel/manual tests", total_imported_tests - total_imported_jstests - total_imported_reftests
        )
        _log.info("")

        if total_prefixed_properties:
            _log.info("Properties needing prefixes (by count):")
            for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
                _log.info("  %s: %s", prefixed_property, total_prefixed_properties[prefixed_property])
Ejemplo n.º 6
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}
        total_prefixed_property_values = {}

        failed_conversion_files = []

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []
            prefixed_property_values = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = self.filesystem.relpath(orig_path, self.source_directory)
            new_path = self.filesystem.join(self.destination_directory,
                                            subpath)

            if not (self.filesystem.exists(new_path)):
                self.filesystem.maybe_make_directory(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = self.filesystem.normpath(file_to_copy['src'])

                if self.filesystem.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory' % orig_filepath)
                    continue

                if not (self.filesystem.exists(orig_filepath)):
                    _log.warning('%s not found. Possible error in the test.',
                                 orig_filepath)
                    continue

                new_filepath = self.filesystem.join(new_path,
                                                    file_to_copy['dest'])
                if 'reference_support_info' in file_to_copy.keys(
                ) and file_to_copy['reference_support_info'] != {}:
                    reference_support_info = file_to_copy[
                        'reference_support_info']
                else:
                    reference_support_info = None

                if not (self.filesystem.exists(
                        self.filesystem.dirname(new_filepath))):
                    self.filesystem.maybe_make_directory(
                        self.filesystem.dirname(new_filepath))

                if not self.options.overwrite and self.filesystem.exists(
                        new_filepath):
                    _log.info('Skipping import of existing file ' +
                              new_filepath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('Importing: %s', orig_filepath)
                    _log.info('       As: %s', new_filepath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'html' in str(mimetype[0]) or 'xml' in str(
                        mimetype[0]) or 'css' in str(mimetype[0]):
                    try:
                        converted_file = convert_for_webkit(
                            new_path,
                            filename=orig_filepath,
                            reference_support_info=reference_support_info,
                            host=self.host,
                            convert_test_harness_links=self.
                            should_convert_test_harness_links(subpath))
                    except:
                        _log.warn('Failed converting %s', orig_filepath)
                        failed_conversion_files.append(orig_filepath)
                        converted_file = None

                    if not converted_file:
                        self.filesystem.copyfile(
                            orig_filepath,
                            new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(
                                prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(
                            set(converted_file[0]) - set(prefixed_properties))

                        for prefixed_value in converted_file[1]:
                            total_prefixed_property_values.setdefault(
                                prefixed_value, 0)
                            total_prefixed_property_values[prefixed_value] += 1

                        prefixed_property_values.extend(
                            set(converted_file[1]) -
                            set(prefixed_property_values))

                        self.filesystem.write_binary_file(
                            new_filepath, converted_file[2])
                elif orig_filepath.endswith(
                        '__init__.py'
                ) and not self.filesystem.getsize(orig_filepath):
                    # Some bots dislike empty __init__.py.
                    self.write_init_py(new_filepath)
                else:
                    self.filesystem.copyfile(orig_filepath, new_filepath)

                self.write_html_files_for_templated_js_tests(
                    orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(
                    self._webkit_root, ''))

            self.remove_deleted_files(new_path, copied_files)
            self.write_import_log(new_path, copied_files, prefixed_properties,
                                  prefixed_property_values)

        _log.info('Import complete')

        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info(
            'Imported %d pixel/manual tests', total_imported_tests -
            total_imported_jstests - total_imported_reftests)
        if len(failed_conversion_files):
            _log.warn(
                'Failed converting %d files (files copied without being converted)',
                len(failed_conversion_files))
        _log.info('')
        _log.info('Properties needing prefixes (by count):')

        for prefixed_property in sorted(
                total_prefixed_properties,
                key=lambda p: total_prefixed_properties[p]):
            _log.info('  %s: %s', prefixed_property,
                      total_prefixed_properties[prefixed_property])
        _log.info('')
        _log.info('Property values needing prefixes (by count):')

        for prefixed_value in sorted(
                total_prefixed_property_values,
                key=lambda p: total_prefixed_property_values[p]):
            _log.info('  %s: %s', prefixed_value,
                      total_prefixed_property_values[prefixed_value])

        if self._potential_test_resource_files and self._test_resource_files:
            # FIXME: We should check that actual tests are not in the test_resource_files list
            should_update_json_file = False
            files = self._test_resource_files["files"]
            for full_path in self._potential_test_resource_files:
                resource_file_path = self.filesystem.relpath(
                    full_path, self.source_directory)
                if not self._already_identified_as_resource_file(
                        resource_file_path):
                    files.append(resource_file_path)
                    should_update_json_file = True
            if should_update_json_file:
                files.sort()
                self.filesystem.write_text_file(
                    self._test_resource_files_json_path,
                    json.dumps(self._test_resource_files,
                               sort_keys=True,
                               indent=4).replace(' \n', '\n'))

        if self._tests_options:
            self.update_tests_options()
Ejemplo n.º 7
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = os.path.relpath(orig_path, self.top_of_repo)
            new_path = os.path.join(self.destination_directory, subpath)

            if not (os.path.exists(new_path)):
                os.makedirs(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = os.path.normpath(file_to_copy['src'])

                if os.path.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory' % orig_filepath)
                    continue

                if not (os.path.exists(orig_filepath)):
                    _log.warning('%s not found. Possible error in the test.',
                                 orig_filepath)
                    continue

                new_filepath = os.path.join(new_path, file_to_copy['dest'])
                if 'reference_support_info' in file_to_copy.keys(
                ) and file_to_copy['reference_support_info'] != {}:
                    reference_support_info = file_to_copy[
                        'reference_support_info']
                else:
                    reference_support_info = None

                if not (os.path.exists(os.path.dirname(new_filepath))):
                    if not self.import_in_place and not self.options.dry_run:
                        os.makedirs(os.path.dirname(new_filepath))

                relpath = os.path.relpath(new_filepath, self.layout_tests_dir)
                if not self.options.overwrite and os.path.exists(new_filepath):
                    _log.info('  skipping %s' % relpath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('  %s' % relpath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'html' in str(mimetype[0]) or 'xml' in str(
                        mimetype[0]) or 'css' in str(mimetype[0]):
                    converted_file = convert_for_webkit(
                        new_path,
                        filename=orig_filepath,
                        reference_support_info=reference_support_info)

                    if not converted_file:
                        if not self.import_in_place and not self.options.dry_run:
                            shutil.copyfile(
                                orig_filepath,
                                new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(
                                prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(
                            set(converted_file[0]) - set(prefixed_properties))
                        if not self.options.dry_run:
                            outfile = open(new_filepath, 'wb')
                            outfile.write(converted_file[1])
                            outfile.close()
                else:
                    if not self.import_in_place and not self.options.dry_run:
                        shutil.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(
                    self._webkit_root, ''))

        _log.info('')
        _log.info('Import complete')
        _log.info('')
        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info(
            'Imported %d pixel/manual tests', total_imported_tests -
            total_imported_jstests - total_imported_reftests)
        _log.info('')

        if total_prefixed_properties:
            _log.info('Properties needing prefixes (by count):')
            for prefixed_property in sorted(
                    total_prefixed_properties,
                    key=lambda p: total_prefixed_properties[p]):
                _log.info('  %s: %s', prefixed_property,
                          total_prefixed_properties[prefixed_property])
Ejemplo n.º 8
0
 def test_convert_for_webkit_with_utf8(self):
     files = {'/file': 'foo',
              '/mock-checkout/third_party/WebKit/Source/core/css/CSSProperties.in': '', }
     host = MockSystemHost(filesystem=MockFileSystem(files=files))
     convert_for_webkit('', '/file', '', host)
Ejemplo n.º 9
0
 def test_convert_for_webkit_with_utf8(self):
     files = {'/file': 'foo', }
     host = MockSystemHost(filesystem=MockFileSystem(files=files))
     convert_for_webkit('', '/file', '', host)
Ejemplo n.º 10
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = self.filesystem.relpath(orig_path, self.top_of_repo)
            new_path = self.filesystem.join(self.destination_directory, subpath)

            if not self.filesystem.exists(new_path):
                self.filesystem.maybe_make_directory(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = self.filesystem.normpath(file_to_copy['src'])

                if self.filesystem.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory' % orig_filepath)
                    continue

                if not self.filesystem.exists(orig_filepath):
                    _log.warning('%s not found. Possible error in the test.', orig_filepath)
                    continue

                if self.path_too_long(orig_filepath):
                    _log.warning('%s skipped (longer than %d chars), to avoid hitting Windows max path length on builders (http://crbug.com/609871).',
                                 orig_filepath, MAX_PATH_LENGTH)
                    continue

                new_filepath = self.filesystem.join(new_path, file_to_copy['dest'])
                if 'reference_support_info' in file_to_copy.keys() and file_to_copy['reference_support_info'] != {}:
                    reference_support_info = file_to_copy['reference_support_info']
                else:
                    reference_support_info = None

                if not self.filesystem.exists(self.filesystem.dirname(new_filepath)):
                    if not self.import_in_place and not self.options.dry_run:
                        self.filesystem.maybe_make_directory(self.filesystem.dirname(new_filepath))

                relpath = self.filesystem.relpath(new_filepath, self.layout_tests_dir)
                if not self.options.overwrite and self.filesystem.exists(new_filepath):
                    _log.info('  skipping %s' % relpath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('  %s' % relpath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
                    converted_file = convert_for_webkit(new_path, filename=orig_filepath,
                                                        reference_support_info=reference_support_info)

                    if not converted_file:
                        if not self.import_in_place and not self.options.dry_run:
                            self.filesystem.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
                        if not self.options.dry_run:
                            outfile = open(new_filepath, 'wb')
                            outfile.write(converted_file[1].encode('utf-8'))
                            outfile.close()
                else:
                    if not self.import_in_place and not self.options.dry_run:
                        self.filesystem.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(self._webkit_root, ''))

        _log.info('')
        _log.info('Import complete')
        _log.info('')
        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
        _log.info('')

        if total_prefixed_properties:
            _log.info('Properties needing prefixes (by count):')
            for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
                _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
Ejemplo n.º 11
0
    def import_tests(self):
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}
        total_prefixed_property_values = {}

        failed_conversion_files = []

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy["total_tests"]
            total_imported_reftests += dir_to_copy["reftests"]
            total_imported_jstests += dir_to_copy["jstests"]

            prefixed_properties = []
            prefixed_property_values = []

            if not dir_to_copy["copy_list"]:
                continue

            orig_path = dir_to_copy["dirname"]

            subpath = self.filesystem.relpath(orig_path, self.source_directory)
            new_path = self.filesystem.join(self.destination_directory, subpath)

            if not (self.filesystem.exists(new_path)):
                self.filesystem.maybe_make_directory(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy["copy_list"]:
                # FIXME: Split this block into a separate function.
                orig_filepath = self.filesystem.normpath(file_to_copy["src"])

                if self.filesystem.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error("%s refers to a directory" % orig_filepath)
                    continue

                if not (self.filesystem.exists(orig_filepath)):
                    _log.warning("%s not found. Possible error in the test.", orig_filepath)
                    continue

                new_filepath = self.filesystem.join(new_path, file_to_copy["dest"])
                if "reference_support_info" in file_to_copy.keys() and file_to_copy["reference_support_info"] != {}:
                    reference_support_info = file_to_copy["reference_support_info"]
                else:
                    reference_support_info = None

                if not (self.filesystem.exists(self.filesystem.dirname(new_filepath))):
                    self.filesystem.maybe_make_directory(self.filesystem.dirname(new_filepath))

                if not self.options.overwrite and self.filesystem.exists(new_filepath):
                    _log.info("Skipping import of existing file " + new_filepath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info("Importing: %s", orig_filepath)
                    _log.info("       As: %s", new_filepath)

                # Only html, xml, or css should be converted
                # FIXME: Eventually, so should js when support is added for this type of conversion
                mimetype = mimetypes.guess_type(orig_filepath)
                if "html" in str(mimetype[0]) or "xml" in str(mimetype[0]) or "css" in str(mimetype[0]):
                    try:
                        converted_file = convert_for_webkit(
                            new_path,
                            filename=orig_filepath,
                            reference_support_info=reference_support_info,
                            host=self.host,
                            convert_test_harness_links=self.should_convert_test_harness_links(subpath),
                        )
                    except:
                        _log.warn("Failed converting %s", orig_filepath)
                        failed_conversion_files.append(orig_filepath)
                        converted_file = None

                    if not converted_file:
                        self.filesystem.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))

                        for prefixed_value in converted_file[1]:
                            total_prefixed_property_values.setdefault(prefixed_value, 0)
                            total_prefixed_property_values[prefixed_value] += 1

                        prefixed_property_values.extend(set(converted_file[1]) - set(prefixed_property_values))

                        self.filesystem.write_binary_file(new_filepath, converted_file[2])
                elif orig_filepath.endswith("__init__.py") and not self.filesystem.getsize(orig_filepath):
                    # Some bots dislike empty __init__.py.
                    self.filesystem.write_text_file(
                        new_filepath, "# This file is required for Python to search this directory for modules."
                    )
                else:
                    self.filesystem.copyfile(orig_filepath, new_filepath)

                copied_files.append(new_filepath.replace(self._webkit_root, ""))

            self.remove_deleted_files(new_path, copied_files)
            self.write_import_log(new_path, copied_files, prefixed_properties, prefixed_property_values)

        _log.info("Import complete")

        _log.info("IMPORTED %d TOTAL TESTS", total_imported_tests)
        _log.info("Imported %d reftests", total_imported_reftests)
        _log.info("Imported %d JS tests", total_imported_jstests)
        _log.info(
            "Imported %d pixel/manual tests", total_imported_tests - total_imported_jstests - total_imported_reftests
        )
        if len(failed_conversion_files):
            _log.warn("Failed converting %d files (files copied without being converted)", len(failed_conversion_files))
        _log.info("")
        _log.info("Properties needing prefixes (by count):")

        for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
            _log.info("  %s: %s", prefixed_property, total_prefixed_properties[prefixed_property])
        _log.info("")
        _log.info("Property values needing prefixes (by count):")

        for prefixed_value in sorted(total_prefixed_property_values, key=lambda p: total_prefixed_property_values[p]):
            _log.info("  %s: %s", prefixed_value, total_prefixed_property_values[prefixed_value])

        if self._potential_test_resource_files:
            _log.info(
                "The following files may be resource files and should be marked as skipped in the TestExpectations:"
            )
            for filename in sorted([test["src"] for test in self._potential_test_resource_files]):
                _log.info(filename.replace(self.source_directory, self.tests_w3c_relative_path) + " [ Skip ]")
Ejemplo n.º 12
0
    def copy_file(self, file_to_copy, dest_dir):
        """Converts and copies a file, if it should be copied.

        Args:
            file_to_copy: A dict in a file copy list constructed by
                find_importable_tests, which represents one file to copy, including
                the keys:
                    "src": Absolute path to the source location of the file.
                    "destination": File name of the destination file.
                And possibly also the keys "reference_support_info" or "is_jstest".
            dest_dir: Path to the directory where the file should be copied.

        Returns:
            The path to the new file, relative to the Blink root (//third_party/WebKit).
        """
        source_path = self.filesystem.normpath(file_to_copy['src'])
        dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])

        if self.filesystem.isdir(source_path):
            _log.error('%s refers to a directory', source_path)
            return None

        if not self.filesystem.exists(source_path):
            _log.error('%s not found. Possible error in the test.', source_path)
            return None

        if file_to_copy.get('reference_support_info'):
            reference_support_info = file_to_copy['reference_support_info']
        else:
            reference_support_info = None

        if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
            if not self.import_in_place and not self.options.dry_run:
                self.filesystem.maybe_make_directory(self.filesystem.dirname(dest_path))

        relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
        if not self.options.overwrite and self.filesystem.exists(dest_path):
            _log.info('  skipping %s', relpath)
        else:
            # FIXME: Maybe doing a file diff is in order here for existing files?
            # In other words, there's no sense in overwriting identical files, but
            # there's no harm in copying the identical thing.
            _log.info('  %s', relpath)

        if self.should_try_to_convert(file_to_copy, source_path, dest_dir):
            converted_file = convert_for_webkit(
                dest_dir, filename=source_path,
                reference_support_info=reference_support_info,
                host=self.host)
            for prefixed_property in converted_file[0]:
                self._prefixed_properties.setdefault(prefixed_property, 0)
                self._prefixed_properties[prefixed_property] += 1

            if not self.options.dry_run:
                self.filesystem.write_text_file(dest_path, converted_file[1])
        else:
            if not self.import_in_place and not self.options.dry_run:
                self.filesystem.copyfile(source_path, dest_path)
                if self.filesystem.read_binary_file(source_path)[:2] == '#!':
                    self.filesystem.make_executable(dest_path)

        return dest_path.replace(self._webkit_root, '')
Ejemplo n.º 13
0
    def import_tests(self):
        """Reads |self.import_list|, and converts and copies files to their destination."""
        total_imported_tests = 0
        total_imported_reftests = 0
        total_imported_jstests = 0
        total_prefixed_properties = {}

        for dir_to_copy in self.import_list:
            total_imported_tests += dir_to_copy['total_tests']
            total_imported_reftests += dir_to_copy['reftests']
            total_imported_jstests += dir_to_copy['jstests']

            prefixed_properties = []

            if not dir_to_copy['copy_list']:
                continue

            orig_path = dir_to_copy['dirname']

            subpath = self.filesystem.relpath(orig_path, self.source_repo_path)
            new_path = self.filesystem.join(self.destination_directory,
                                            subpath)

            if not self.filesystem.exists(new_path):
                self.filesystem.maybe_make_directory(new_path)

            copied_files = []

            for file_to_copy in dir_to_copy['copy_list']:
                # FIXME: Split this block into a separate function.
                orig_filepath = self.filesystem.normpath(file_to_copy['src'])

                if self.filesystem.isdir(orig_filepath):
                    # FIXME: Figure out what is triggering this and what to do about it.
                    _log.error('%s refers to a directory', orig_filepath)
                    continue

                if not self.filesystem.exists(orig_filepath):
                    _log.error('%s not found. Possible error in the test.',
                               orig_filepath)
                    continue

                new_filepath = self.filesystem.join(new_path,
                                                    file_to_copy['dest'])
                if 'reference_support_info' in file_to_copy.keys(
                ) and file_to_copy['reference_support_info'] != {}:
                    reference_support_info = file_to_copy[
                        'reference_support_info']
                else:
                    reference_support_info = None

                if not self.filesystem.exists(
                        self.filesystem.dirname(new_filepath)):
                    if not self.import_in_place and not self.options.dry_run:
                        self.filesystem.maybe_make_directory(
                            self.filesystem.dirname(new_filepath))

                relpath = self.filesystem.relpath(new_filepath,
                                                  self.layout_tests_dir)
                if not self.options.overwrite and self.filesystem.exists(
                        new_filepath):
                    _log.info('  skipping %s', relpath)
                else:
                    # FIXME: Maybe doing a file diff is in order here for existing files?
                    # In other words, there's no sense in overwriting identical files, but
                    # there's no harm in copying the identical thing.
                    _log.info('  %s', relpath)

                # Only HTML, XML, or CSS should be converted.
                # FIXME: Eventually, so should JS when support is added for this type of conversion.
                mimetype = mimetypes.guess_type(orig_filepath)
                if 'is_jstest' not in file_to_copy and (
                        'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])
                        or 'css' in str(mimetype[0])):
                    converted_file = convert_for_webkit(
                        new_path,
                        filename=orig_filepath,
                        reference_support_info=reference_support_info,
                        host=self.host)

                    if not converted_file:
                        if not self.import_in_place and not self.options.dry_run:
                            self.filesystem.copyfile(
                                orig_filepath,
                                new_filepath)  # The file was unmodified.
                    else:
                        for prefixed_property in converted_file[0]:
                            total_prefixed_properties.setdefault(
                                prefixed_property, 0)
                            total_prefixed_properties[prefixed_property] += 1

                        prefixed_properties.extend(
                            set(converted_file[0]) - set(prefixed_properties))
                        if not self.options.dry_run:
                            self.filesystem.write_text_file(
                                new_filepath, converted_file[1])
                else:
                    if not self.import_in_place and not self.options.dry_run:
                        self.filesystem.copyfile(orig_filepath, new_filepath)
                        if self.filesystem.read_binary_file(
                                orig_filepath)[:2] == '#!':
                            self.filesystem.make_executable(new_filepath)

                copied_files.append(new_filepath.replace(
                    self._webkit_root, ''))

        _log.info('')
        _log.info('Import complete')
        _log.info('')
        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
        _log.info('Imported %d reftests', total_imported_reftests)
        _log.info('Imported %d JS tests', total_imported_jstests)
        _log.info(
            'Imported %d pixel/manual tests', total_imported_tests -
            total_imported_jstests - total_imported_reftests)
        _log.info('')

        if total_prefixed_properties:
            _log.info('Properties needing prefixes (by count):')
            for prefixed_property in sorted(
                    total_prefixed_properties,
                    key=lambda p: total_prefixed_properties[p]):
                _log.info('  %s: %s', prefixed_property,
                          total_prefixed_properties[prefixed_property])
Ejemplo n.º 14
0
    def copy_file(self, file_to_copy, dest_dir):
        """Converts and copies a file, if it should be copied.

        Args:
            file_to_copy: A dict in a file copy list constructed by
                find_importable_tests, which represents one file to copy, including
                the keys:
                    "src": Absolute path to the source location of the file.
                    "destination": File name of the destination file.
                And possibly also the keys "reference_support_info" or "is_jstest".
            dest_dir: Path to the directory where the file should be copied.

        Returns:
            The path to the new file, relative to the Blink root (//third_party/WebKit).
        """
        source_path = self.filesystem.normpath(file_to_copy['src'])
        dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])

        if self.filesystem.isdir(source_path):
            _log.error('%s refers to a directory', source_path)
            return None

        if not self.filesystem.exists(source_path):
            _log.error('%s not found. Possible error in the test.',
                       source_path)
            return None

        if file_to_copy.get('reference_support_info'):
            reference_support_info = file_to_copy['reference_support_info']
        else:
            reference_support_info = None

        if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
            if not self.import_in_place and not self.options.dry_run:
                self.filesystem.maybe_make_directory(
                    self.filesystem.dirname(dest_path))

        relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
        if not self.options.overwrite and self.filesystem.exists(dest_path):
            _log.info('  skipping %s', relpath)
        else:
            # FIXME: Maybe doing a file diff is in order here for existing files?
            # In other words, there's no sense in overwriting identical files, but
            # there's no harm in copying the identical thing.
            _log.info('  %s', relpath)

        # Only HTML, XML, or CSS should be converted.
        # FIXME: Eventually, so should JS when support is added for this type of conversion.
        mimetype = mimetypes.guess_type(source_path)
        if 'is_jstest' not in file_to_copy and ('html' in str(mimetype[0])
                                                or 'xml' in str(mimetype[0])
                                                or 'css' in str(mimetype[0])):
            converted_file = convert_for_webkit(
                dest_dir,
                filename=source_path,
                reference_support_info=reference_support_info,
                host=self.host)

            if not converted_file:
                if not self.import_in_place and not self.options.dry_run:
                    self.filesystem.copyfile(
                        source_path, dest_path)  # The file was unmodified.
            else:
                for prefixed_property in converted_file[0]:
                    self._prefixed_properties.setdefault(prefixed_property, 0)
                    self._prefixed_properties[prefixed_property] += 1

                if not self.options.dry_run:
                    self.filesystem.write_text_file(dest_path,
                                                    converted_file[1])
        else:
            if not self.import_in_place and not self.options.dry_run:
                self.filesystem.copyfile(source_path, dest_path)
                if self.filesystem.read_binary_file(source_path)[:2] == '#!':
                    self.filesystem.make_executable(dest_path)

        return dest_path.replace(self._webkit_root, '')