Beispiel #1
0
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.LEGAL_DOCS_PATH,
                       settings.LEGAL_DOCS_REPO,
                       branch_name=settings.LEGAL_DOCS_BRANCH,
                       name='Legal Docs')
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No content card updates')
            return

        self.output('Loading legal docs into database')
        count, errors = LegalDoc.objects.refresh()

        self.output(f'{count} legal docs successfully loaded')
        self.output(f'Encountered {errors} errors while loading docs')

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')

        if not errors and settings.LEGAL_DOCS_DMS_URL:
            requests.get(settings.LEGAL_DOCS_DMS_URL)
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.EXTERNAL_FILES_PATH,
                       settings.EXTERNAL_FILES_REPO,
                       branch_name=settings.EXTERNAL_FILES_BRANCH,
                       name="Community Data")
        self.output("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.output("No community data updates")
            return

        self.output("Loading community data into database")

        for fid, finfo in settings.EXTERNAL_FILES.items():
            klass = import_string(finfo["type"])
            try:
                klass(fid).update()
            except ValueError as e:
                raise CommandError(str(e))

        self.output("Community data successfully loaded")

        repo.set_db_latest()

        self.output("Saved latest git repo state to database")
        self.output("Done!")
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO,
                       branch_name=settings.EXTERNAL_FILES_BRANCH,
                       name='Community Data')
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No community data updates')
            return

        self.output('Loading community data into database')

        for fid, finfo in settings.EXTERNAL_FILES.items():
            klass = import_string(finfo['type'])
            try:
                klass(fid).update()
            except ValueError as e:
                raise CommandError(str(e))

        self.output('Community data successfully loaded')

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
Beispiel #4
0
    def handle_noargs(self, **options):
        quiet = options['quiet']
        no_git = options['no_git']
        clear_db = options['clear_db']
        force = no_git or clear_db
        repo = GitRepo(ADVISORIES_PATH,
                       ADVISORIES_REPO,
                       branch_name=ADVISORIES_BRANCH,
                       name='Security Advisories')

        def printout(msg, ending=None):
            if not quiet:
                self.stdout.write(msg, ending=ending)

        if clear_db:
            printout('Clearing all security advisories.')
            SecurityAdvisory.objects.all().delete()
            Product.objects.all().delete()
            MitreCVE.objects.all().delete()

        if not no_git:
            printout('Updating repository.')
            repo.update()

        if not (force or repo.has_changes()):
            printout('Nothing to update.')
            return

        errors = []
        updates = 0
        all_files = get_all_file_names()
        for mf in all_files:
            try:
                update_db_from_file(mf)
            except Exception as e:
                errors.append('ERROR parsing %s: %s' % (mf, e))
                if not quiet:
                    sys.stdout.write('E')
                    sys.stdout.flush()
                continue
            if not quiet:
                sys.stdout.write('.')
                sys.stdout.flush()
            updates += 1
        printout('\nUpdated {0} files.'.format(updates))

        if not clear_db:
            deleted_files = get_files_to_delete_from_db(all_files)
            delete_files(deleted_files)
            printout('Deleted {0} files.'.format(len(deleted_files)))
            num_products = delete_orphaned_products()
            if num_products:
                printout('Deleted {0} orphaned products.'.format(num_products))

        if errors:
            raise CommandError(
                'Encountered {0} errors:\n\n'.format(len(errors)) +
                '\n==========\n'.join(errors))

        repo.set_db_latest()
 def handle(self, *args, **options):
     repo = GitRepo(settings.RELEASE_NOTES_PATH,
                    settings.RELEASE_NOTES_REPO,
                    branch_name=settings.RELEASE_NOTES_BRANCH)
     repo.update()
     if not options['quiet']:
         print('Release Notes Successfully Updated')
Beispiel #6
0
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.LEGAL_DOCS_PATH,
                       settings.LEGAL_DOCS_REPO,
                       branch_name=settings.LEGAL_DOCS_BRANCH,
                       name="Legal Docs")
        self.output("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.output("No legal docs updates")
            self.snitch()
            return

        self.output("Loading legal docs into database")
        count, errors = LegalDoc.objects.refresh()
        self.output(f"{count} legal docs successfully loaded")
        if errors:
            self.output(f"Encountered {errors} errors while loading docs")
        else:
            # only set latest if there are no errors so that it will try the errors again next time
            # also so that it will fail again and thus not ping the snitch so that we'll be notified
            repo.set_db_latest()
            self.output("Saved latest git repo state to database")
            self.snitch()

        self.output("Done!")
Beispiel #7
0
    def update_lang_files(self, clean=False):
        repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
        if clean:
            repo.reclone()
        else:
            repo.update()

        self.stdout.write('Updated .lang files')
Beispiel #8
0
    def update_fluent_files(self, clean=False):
        repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
        if clean:
            rmtree(repo.path_str, ignore_errors=True)
            self.stdout.write('Removed old .ftl repo')

        repo.update()
        self.stdout.write('Updated .ftl files')
Beispiel #9
0
    def update_lang_files(self, clean=False):
        repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
        if clean:
            rmtree(repo.path_str, ignore_errors=True)
            self.stdout.write('Removed old .lang repo')

        repo.update()
        self.stdout.write('Updated .lang files')
Beispiel #10
0
    def handle_safe(self, quiet, no_git, clear_db, **options):
        force = no_git or clear_db
        repo = GitRepo(
            ADVISORIES_PATH,
            ADVISORIES_REPO,
            branch_name=ADVISORIES_BRANCH,
            name="Security Advisories",
        )

        def printout(msg, ending=None):
            if not quiet:
                self.stdout.write(msg, ending=ending)

        if clear_db:
            printout("Clearing all security advisories.")
            SecurityAdvisory.objects.all().delete()
            Product.objects.all().delete()
            MitreCVE.objects.all().delete()

        if not no_git:
            printout("Updating repository.")
            repo.update()

        if not (force or repo.has_changes()):
            printout("Nothing to update.")
            return

        errors = []
        updates = 0
        all_files = get_all_file_names()
        for mf in all_files:
            try:
                update_db_from_file(mf)
            except Exception as e:
                errors.append(f"ERROR parsing {mf}: {e}")
                if not quiet:
                    sys.stdout.write("E")
                    sys.stdout.flush()
                continue
            if not quiet:
                sys.stdout.write(".")
                sys.stdout.flush()
            updates += 1
        printout(f"\nUpdated {updates} files.")

        if not clear_db:
            deleted_files = get_files_to_delete_from_db(all_files)
            delete_files(deleted_files)
            printout(f"Deleted {len(deleted_files)} files.")
            num_products = delete_orphaned_products()
            if num_products:
                printout(f"Deleted {num_products} orphaned products.")

        if errors:
            raise CommandError(f"Encountered {len(errors)} errors:\n\n" +
                               "\n==========\n".join(errors))

        repo.set_db_latest()
    def handle_noargs(self, **options):
        quiet = options['quiet']
        no_git = options['no_git']
        clear_db = options['clear_db']
        force = no_git or clear_db
        repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH,
                       name='Security Advisories')

        def printout(msg, ending=None):
            if not quiet:
                self.stdout.write(msg, ending=ending)

        if clear_db:
            printout('Clearing all security advisories.')
            SecurityAdvisory.objects.all().delete()
            Product.objects.all().delete()
            MitreCVE.objects.all().delete()

        if not no_git:
            printout('Updating repository.')
            repo.update()

        if not (force or repo.has_changes()):
            printout('Nothing to update.')
            return

        errors = []
        updates = 0
        all_files = get_all_file_names()
        for mf in all_files:
            try:
                update_db_from_file(mf)
            except Exception as e:
                errors.append('ERROR parsing %s: %s' % (mf, e))
                if not quiet:
                    sys.stdout.write('E')
                    sys.stdout.flush()
                continue
            if not quiet:
                sys.stdout.write('.')
                sys.stdout.flush()
            updates += 1
        printout('\nUpdated {0} files.'.format(updates))

        if not clear_db:
            deleted_files = get_files_to_delete_from_db(all_files)
            delete_files(deleted_files)
            printout('Deleted {0} files.'.format(len(deleted_files)))
            num_products = delete_orphaned_products()
            if num_products:
                printout('Deleted {0} orphaned products.'.format(num_products))

        if errors:
            raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
                               '\n==========\n'.join(errors))

        repo.set_db_latest()
Beispiel #12
0
    def update_fluent_files(self, clean=False):
        repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
        if clean:
            repo.reclone()
        else:
            repo.update()

        repo.update()
        self.stdout.write("Updated .ftl files")
Beispiel #13
0
    def update_fluent_files(self, clean=False):
        for site, params in FLUENT_L10N_UPDATE_PARAMS.items():
            repo = GitRepo(**params)
            if clean:
                repo.reclone()
            else:
                repo.update()

            repo.update()
            self.stdout.write(f"Updated .ftl files for {site}")
Beispiel #14
0
    def handle(self, *args, **options):
        if options["quiet"]:
            self.stdout._out = StringIO()

        repo = GitRepo(settings.SITEMAPS_PATH,
                       settings.SITEMAPS_REPO,
                       name="Sitemaps")
        self.stdout.write("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.stdout.write("No sitemap updates")
            return

        SitemapURL.objects.refresh()
        repo.set_db_latest()
        self.stdout.write("Updated sitemaps files")
Beispiel #15
0
class FTLRepoCommand(BaseCommand):
    meao_repo = None
    l10n_repo = None

    def add_arguments(self, parser):
        parser.add_argument("-q",
                            "--quiet",
                            action="store_true",
                            dest="quiet",
                            default=False,
                            help="If no error occurs, swallow all output.")

    def handle(self, *args, **options):
        if options["quiet"]:
            self.stdout._out = StringIO()

        self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH,
                                 settings.FLUENT_L10N_TEAM_REPO_URL,
                                 settings.FLUENT_L10N_TEAM_REPO_BRANCH)
        self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH,
                                 settings.FLUENT_REPO_URL,
                                 settings.FLUENT_REPO_BRANCH)

    def update_l10n_team_files(self):
        try:
            # this will fail on first run
            self.l10n_repo.clean()
        except FileNotFoundError:
            pass
        self.l10n_repo.update()
        self.stdout.write(
            f"Updated l10n team .ftl files for {settings.FLUENT_L10N_TEAM_REPO_URL}"
        )

    def update_fluent_files(self):
        try:
            self.meao_repo.clean()
        except FileNotFoundError:
            pass
        self.meao_repo.update()
        self.stdout.write(f"Updated .ftl files for {settings.FLUENT_REPO_URL}")

    def config_git(self):
        """Set user config so that committing will work"""
        for repo in (self.meao_repo, self.l10n_repo):
            repo.git("config", "user.email", GIT_COMMIT_EMAIL)
            repo.git("config", "user.name", GIT_COMMIT_NAME)
Beispiel #16
0
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name="Content Cards")
        self.output("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.output("No content card updates")
            return

        self.output("Loading content cards into database")
        count = ContentCard.objects.refresh()

        self.output(f"{count} content cards successfully loaded")

        repo.set_db_latest()

        self.output("Saved latest git repo state to database")
        self.output("Done!")
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
                       branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No content card updates')
            return

        self.output('Loading content cards into database')
        count = ContentCard.objects.refresh()

        self.output('%s content cards successfully loaded' % count)

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
                       branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No content card updates')
            return

        self.output('Loading content cards into database')
        count = ContentCard.objects.refresh()

        self.output('%s content cards successfully loaded' % count)

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO,
                       branch_name=settings.RELEASE_NOTES_BRANCH)
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No release note updates')
            return

        self.output('Loading releases into database')
        count = ProductRelease.objects.refresh()

        self.output('%s release notes successfully loaded' % count)

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
Beispiel #20
0
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.RELEASE_NOTES_PATH,
                       settings.RELEASE_NOTES_REPO,
                       branch_name=settings.RELEASE_NOTES_BRANCH)
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No release note updates')
            return

        self.output('Loading releases into database')
        count = ProductRelease.objects.refresh()

        self.output('%s release notes successfully loaded' % count)

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
Beispiel #21
0
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
                       branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config')
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No config updates')
            return

        self.output('Loading configs into database')
        count = refresh_db_values()

        if count:
            self.output('%s configs successfully loaded' % count)
        else:
            self.output('No configs found. Please try again later.')

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
    def handle(self, *args, **options):
        self.quiet = options['quiet']
        repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
                       branch_name=settings.WWW_CONFIG_BRANCH)
        self.output('Updating git repo')
        repo.update()
        if not (options['force'] or repo.has_changes()):
            self.output('No config updates')
            return

        self.output('Loading configs into database')
        count = refresh_db_values()

        if count:
            self.output('%s configs successfully loaded' % count)
        else:
            self.output('No configs found. Please try again later.')

        repo.set_db_latest()

        self.output('Saved latest git repo state to database')
        self.output('Done!')
Beispiel #23
0
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.WEBVISION_DOCS_PATH,
                       settings.WEBVISION_DOCS_REPO,
                       branch_name=settings.WEBVISION_DOCS_BRANCH,
                       name="Webvision Docs")
        self.output("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.output("No webvision docs updates")
            return

        self.output("Loading webvision docs into database")
        count, errors = WebvisionDoc.objects.refresh()
        self.output(f"{count} webvision docs successfully loaded")
        if errors:
            self.output(f"Encountered {errors} errors while loading docs")
        else:
            # Only `set_db_latest` if there are no errors so that it will try without errors again next time.
            repo.set_db_latest()
            self.output("Saved latest git repo state to database")

        self.output("Done!")
Beispiel #24
0
    def handle(self, *args, **options):
        if options["quiet"]:
            self.stdout._out = StringIO()

        data_path = settings.SITEMAPS_PATH.joinpath("data")
        repo = GitRepo(settings.SITEMAPS_PATH, settings.SITEMAPS_REPO)
        repo.update()

        for src_path in data_path.rglob("*.*"):
            rel_path = src_path.relative_to(data_path)
            if rel_path.parts[0] == "sitemaps":
                rel_path = rel_path.relative_to("sitemaps")
            target_path = ROOT_FILES.joinpath(rel_path)
            if target_path.exists():
                if target_path.is_symlink():
                    continue
                else:
                    target_path.unlink()

            target_path.parent.mkdir(exist_ok=True)
            target_path.symlink_to(src_path)

        self.stdout.write("Updated sitemaps files")
Beispiel #25
0
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.WWW_CONFIG_PATH,
                       settings.WWW_CONFIG_REPO,
                       branch_name=settings.WWW_CONFIG_BRANCH,
                       name="WWW Config")
        self.output("Updating git repo")
        repo.update()
        if not (options["force"] or repo.has_changes()):
            self.output("No config updates")
            return

        self.output("Loading configs into database")
        count = refresh_db_values()

        if count:
            self.output(f"{count} configs successfully loaded")
        else:
            self.output("No configs found. Please try again later.")

        repo.set_db_latest()

        self.output("Saved latest git repo state to database")
        self.output("Done!")
Beispiel #26
0
    def handle(self, *args, **options):
        self.quiet = options["quiet"]
        repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH, name="Release Notes")
        self.output("Updating git repo")
        self.output(repo.update())
        if not (options["force"] or repo.has_changes()):
            self.output("No release note updates")
            return

        self.output("Loading releases into database")
        count = ProductRelease.objects.refresh()

        self.output(f"{count} release notes successfully loaded")

        repo.set_db_latest()

        self.output("Saved latest git repo state to database")
        self.output("Done!")
Beispiel #27
0
class Command(BaseCommand):
    help = 'Processes .ftl files from l10n team for use in bedrock'
    meao_repo = None
    l10n_repo = None
    parser = None

    def add_arguments(self, parser):
        parser.add_argument('-q',
                            '--quiet',
                            action='store_true',
                            dest='quiet',
                            default=False,
                            help='If no error occurs, swallow all output.')
        parser.add_argument(
            '--push',
            action='store_true',
            dest='push',
            default=False,
            help='Push the changes to the MEAO Fluent files repo.')

    def handle(self, *args, **options):
        if options['quiet']:
            self.stdout._out = StringIO()

        self.parser = NoisyFluentParser()
        self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH,
                                 settings.FLUENT_L10N_TEAM_REPO)
        self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH,
                                 settings.FLUENT_REPO)
        self.update_fluent_files()
        self.update_l10n_team_files()
        no_errors = self.copy_ftl_files()
        self.set_activation()
        self.copy_configs()
        if options['push']:
            changes = self.commit_changes()
            if changes:
                self.push_changes()

        if not no_errors:
            raise CommandError(
                'Some errors were discovered in some .ftl files and they were not updated.'
                'See above for details.')

    def update_l10n_team_files(self):
        try:
            # this will fail on first run
            self.l10n_repo.clean()
        except FileNotFoundError:
            pass
        self.l10n_repo.update()
        self.stdout.write('Updated l10n team .ftl files')

    def update_fluent_files(self):
        try:
            self.meao_repo.clean()
        except FileNotFoundError:
            pass
        self.meao_repo.update()
        self.stdout.write('Updated .ftl files')

    def config_fluent_repo(self):
        """Set user config so that committing will work"""
        self.meao_repo.git('config', 'user.email', GIT_COMMIT_EMAIL)
        self.meao_repo.git('config', 'user.name', GIT_COMMIT_NAME)

    def commit_changes(self):
        self.config_fluent_repo()
        self.meao_repo.git('add', '.')
        try:
            self.meao_repo.git('commit', '-m', 'Update files from l10n repo')
        except CalledProcessError:
            self.stdout.write('No changes to commit')
            return False

        self.stdout.write('Committed changes to local repo')
        return True

    def push_changes(self):
        try:
            self.meao_repo.git('push', self.git_push_url, 'HEAD:master')
        except CalledProcessError:
            raise CommandError(
                f'There was a problem pushing to {self.meao_repo.remote_url}')

        commit = self.meao_repo.git('rev-parse', '--short', 'HEAD')
        self.stdout.write(f'Pushed {commit} to {self.meao_repo.remote_url}')

    @property
    def git_push_url(self):
        if not settings.FLUENT_REPO_AUTH:
            raise CommandError('Git push authentication not configured')

        return self.meao_repo.remote_url_auth(settings.FLUENT_REPO_AUTH)

    def _copy_file(self, filepath):
        relative_filepath = filepath.relative_to(self.l10n_repo.path)
        to_filepath = self.meao_repo.path.joinpath(relative_filepath)
        to_filepath.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(str(filepath), str(to_filepath))
        self.stdout.write('.', ending='')
        self.stdout.flush()

    def copy_configs(self):
        count = 0
        for filepath in self.l10n_repo.path.rglob('*.toml'):
            self._copy_file(filepath)
            count += 1

        self.stdout.write(f'\nCopied {count} .toml files')

    def copy_ftl_files(self):
        count = 0
        errors = []
        for filepath in self.l10n_repo.path.rglob('*.ftl'):
            if not self.lint_ftl_file(filepath):
                errors.append(filepath.relative_to(self.l10n_repo.path))
                continue

            self._copy_file(filepath)
            count += 1

        self.stdout.write(f'\nCopied {count} .ftl files')
        if errors:
            self.stdout.write(
                'The following files had parse errors and were not copied:')
            for fpath in errors:
                self.stdout.write(f'- {fpath}')
            return False

        return True

    def lint_ftl_file(self, filepath):
        with filepath.open() as ftl:
            try:
                self.parser.parse(ftl.read())
            except ParseError:
                return False

            return True

    def set_activation(self):
        updated_ftl = set()
        modified, _ = self.meao_repo.modified_files()
        for fname in modified:
            if not fname.endswith('.ftl'):
                continue

            locale, ftl_name = fname.split('/', 1)
            updated_ftl.add(ftl_name)

        for ftl_name in updated_ftl:
            self.calculate_activation(ftl_name)

    def calculate_activation(self, ftl_file):
        translations = self.meao_repo.path.glob(f'*/{ftl_file}')
        metadata = get_metadata(ftl_file)
        active_locales = metadata.get('active_locales', [])
        inactive_locales = metadata.get('inactive_locales', [])
        percent_required = metadata.get(
            'percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED)
        all_locales = {
            str(x.relative_to(self.meao_repo.path)).split('/', 1)[0]
            for x in translations
        }
        locales_to_check = all_locales.difference(['en'], active_locales,
                                                  inactive_locales)
        new_activations = []
        for locale in locales_to_check:
            l10n = fluent_l10n([locale, 'en'], [ftl_file])
            if not l10n.has_required_messages:
                continue

            percent_trans = l10n.percent_translated
            if percent_trans < percent_required:
                continue

            new_activations.append(locale)

        if new_activations:
            active_locales.extend(new_activations)
            metadata['active_locales'] = sorted(active_locales)
            write_metadata(ftl_file, metadata)
            self.stdout.write(
                f'Activated {len(new_activations)} new locales for {ftl_file}')
    def handle_noargs(self, **options):
        quiet = options['quiet']
        force = options['force']
        no_git = options['no_git']
        clear_db = options['clear_db']
        if no_git or clear_db:
            force = True
        cloned = False
        repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH)
        modified_files = deleted_files = []

        def printout(msg, ending=None):
            if not quiet:
                self.stdout.write(msg, ending=ending)

        if clear_db:
            printout('Clearing all security advisories.')
            SecurityAdvisory.objects.all().delete()
            Product.objects.all().delete()

        if not no_git:
            printout('Updating repository.')
            modified_files, deleted_files = repo.update()
            if modified_files is None:
                cloned = True
            else:
                modified_files = filter_advisory_filenames(modified_files)
                deleted_files = filter_advisory_filenames(deleted_files)

        if force or cloned:
            printout('Reading all files.')
            modified_files = get_all_mfsa_files()
            if clear_db:
                deleted_files = []
            else:
                deleted_files = get_files_to_delete_from_db(modified_files)

        errors = []
        updates = 0
        if modified_files:
            for mf in modified_files:
                mf = os.path.join(ADVISORIES_PATH, mf)
                try:
                    update_db_from_file(mf)
                except Exception as e:
                    errors.append('ERROR parsing %s: %s' % (mf, e))
                    if not quiet:
                        sys.stdout.write('E')
                        sys.stdout.flush()
                    continue
                if not quiet:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                updates += 1
            printout('\nUpdated {0} files.'.format(updates))

        if deleted_files:
            deleted_files = filter_updated_from_deleted(modified_files, deleted_files)
            delete_files(deleted_files)
            printout('Deleted {0} files.'.format(len(deleted_files)))
            num_products = delete_orphaned_products()
            if num_products:
                printout('Deleted {0} orphaned products.'.format(num_products))

        if not modified_files and not deleted_files:
            printout('Nothing to update.')

        if errors:
            raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
                               '\n==========\n'.join(errors))
Beispiel #29
0
 def handle(self, *args, **options):
     repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
     repo.update()
Beispiel #30
0
 def update_fluent_files(self):
     repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
     repo.update()
     self.stdout.write('Updated .ftl files')
class Command(BaseCommand):
    def __init__(self, stdout=None, stderr=None, no_color=False):
        self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
        self.db_storage = PDDatabaseStorage()
        self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
                            settings.PROD_DETAILS_JSON_REPO_URI,
                            name='Product Details')
        super(Command, self).__init__(stdout, stderr, no_color)

    def add_arguments(self, parser):
        parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
                            help='If no error occurs, swallow all output.'),
        parser.add_argument('--database', default='default',
                            help=('Specifies the database to use, if using a db. '
                                  'Defaults to "default".')),

    def handle(self, *args, **options):
        # don't really care about deleted files. almost never happens in p-d.
        if not self.update_file_data():
            if not options['quiet']:
                print('Product Details data was already up to date')
            return

        try:
            self.validate_data()
        except Exception:
            raise CommandError('Product Details data is invalid')

        if not options['quiet']:
            print('Product Details data is valid')

        if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
            # no need to continue if not using DB backend
            return

        self.load_changes(options, self.file_storage.all_json_files())
        self.repo.set_db_latest()

        if not options['quiet']:
            print('Product Details data update is complete')

    def load_changes(self, options, modified_files):
        with transaction.atomic(using=options['database']):
            for filename in modified_files:
                self.db_storage.update(filename,
                                       self.file_storage.content(filename),
                                       self.file_storage.last_modified(filename))
                if not options['quiet']:
                    print('Updated ' + filename)

            self.db_storage.update('/', '', self.file_storage.last_modified('/'))
            self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))

    def update_file_data(self):
        self.repo.update()
        return self.repo.has_changes()

    def count_builds(self, version_key, min_builds=20):
        version = self.file_storage.data('firefox_versions.json')[version_key]
        if not version:
            if version_key == 'FIREFOX_ESR_NEXT':
                return
        builds = len([locale for locale, build in
                      self.file_storage.data('firefox_primary_builds.json').items()
                      if version in build])
        if builds < min_builds:
            raise ValueError('Too few builds for {}'.format(version_key))

    def validate_data(self):
        self.file_storage.clear_cache()
        for key in FIREFOX_VERSION_KEYS:
            self.count_builds(key)
Beispiel #32
0
 def update_l10n_team_files(self):
     repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
     repo.update()
     self.stdout.write('Updated l10n team .ftl files')
Beispiel #33
0
class Command(BaseCommand):
    help = 'Processes .ftl files from l10n team for use in bedrock'
    meao_repo = None
    l10n_repo = None
    parser = None

    def add_arguments(self, parser):
        parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
                            help='If no error occurs, swallow all output.'),

    def handle(self, *args, **options):
        if options['quiet']:
            self.stdout._out = StringIO()

        self.parser = NoisyFluentParser()
        self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
        self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
        self.update_fluent_files()
        self.update_l10n_team_files()
        no_errors = self.copy_ftl_files()
        self.set_activation()
        if no_errors:
            self.stdout.write('There were no errors found in the .ftl files.')
        else:
            raise CommandError('Some errors were discovered in some .ftl files and they were not updated.'
                               'See above for details.')

    def update_l10n_team_files(self):
        try:
            # this will fail on first run
            self.l10n_repo.clean()
        except FileNotFoundError:
            pass
        self.l10n_repo.update()
        self.stdout.write('Updated l10n team .ftl files')

    def update_fluent_files(self):
        try:
            self.meao_repo.clean()
        except FileNotFoundError:
            pass
        self.meao_repo.update()
        self.stdout.write('Updated .ftl files')

    def copy_ftl_files(self):
        count = 0
        errors = []
        for filepath in self.l10n_repo.path.rglob('*.ftl'):
            relative_filepath = filepath.relative_to(self.l10n_repo.path)
            if not self.lint_ftl_file(filepath):
                errors.append(relative_filepath)
                continue

            to_filepath = self.meao_repo.path.joinpath(relative_filepath)
            to_filepath.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy2(str(filepath), str(to_filepath))
            count += 1
            self.stdout.write('.', ending='')
            self.stdout.flush()

        self.stdout.write(f'\nCopied {count} .ftl files')
        if errors:
            self.stdout.write('The following files had parse errors and were not copied:')
            for fpath in errors:
                self.stdout.write(f'- {fpath}')
            return False

        return True

    def lint_ftl_file(self, filepath):
        with filepath.open() as ftl:
            try:
                self.parser.parse(ftl.read())
            except ParseError:
                return False

            return True

    def set_activation(self):
        updated_ftl = set()
        modified, _ = self.meao_repo.modified_files()
        for fname in modified:
            locale, ftl_name = fname.split('/', 1)
            updated_ftl.add(ftl_name)

        for ftl_name in updated_ftl:
            self.calculate_activation(ftl_name)

    def calculate_activation(self, ftl_file):
        translations = self.meao_repo.path.glob(f'*/{ftl_file}')
        metadata = get_metadata(ftl_file)
        active_locales = metadata.get('active_locales', [])
        inactive_locales = metadata.get('inactive_locales', [])
        percent_required = metadata.get('percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED)
        all_locales = {str(x.relative_to(self.meao_repo.path)).split('/', 1)[0] for x in translations}
        locales_to_check = all_locales.difference(['en'], active_locales, inactive_locales)
        new_activations = []
        for locale in locales_to_check:
            l10n = fluent_l10n([locale, 'en'], [ftl_file])
            if not l10n.has_required_messages:
                continue

            percent_trans = l10n.percent_translated
            if percent_trans < percent_required:
                continue

            new_activations.append(locale)

        if new_activations:
            active_locales.extend(new_activations)
            metadata['active_locales'] = sorted(active_locales)
            write_metadata(ftl_file, metadata)
            self.stdout.write(f'Activated {len(new_activations)} new locales for {ftl_file}')
Beispiel #34
0
 def update_lang_files(self):
     repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
     repo.update()
     self.stdout.write('Updated .lang files')
class Command(BaseCommand):
    def __init__(self, stdout=None, stderr=None, no_color=False):
        self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
        self.db_storage = PDDatabaseStorage()
        self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
                            settings.PROD_DETAILS_JSON_REPO_URI)
        super(Command, self).__init__(stdout, stderr, no_color)

    def add_arguments(self, parser):
        parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
                            help='If no error occurs, swallow all output.'),
        parser.add_argument('--database', default='default',
                            help=('Specifies the database to use, if using a db. '
                                  'Defaults to "default".')),

    def handle(self, *args, **options):
        # don't really care about deleted files. almost never happens in p-d.
        if not self.update_file_data():
            if not options['quiet']:
                print('Product Details data was already up to date')
            return

        try:
            self.validate_data()
        except Exception:
            raise CommandError('Product Details data is invalid')

        if not options['quiet']:
            print('Product Details data is valid')

        if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
            # no need to continue if not using DB backend
            return

        self.load_changes(options, self.file_storage.all_json_files())
        self.repo.set_db_latest()

        if not options['quiet']:
            print('Product Details data update is complete')

    def load_changes(self, options, modified_files):
        with transaction.atomic(using=options['database']):
            for filename in modified_files:
                self.db_storage.update(filename,
                                       self.file_storage.content(filename),
                                       self.file_storage.last_modified(filename))
                if not options['quiet']:
                    print('Updated ' + filename)

            self.db_storage.update('/', '', self.file_storage.last_modified('/'))
            self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))

    def update_file_data(self):
        self.repo.update()
        return self.repo.has_changes()

    def count_builds(self, version_key, min_builds=20):
        version = self.file_storage.data('firefox_versions.json')[version_key]
        if not version:
            if version_key == 'FIREFOX_ESR_NEXT':
                return
        builds = len([locale for locale, build in
                      self.file_storage.data('firefox_primary_builds.json').items()
                      if version in build])
        if builds < min_builds:
            raise ValueError('Too few builds for {}'.format(version_key))

    def validate_data(self):
        self.file_storage.clear_cache()
        for key in FIREFOX_VERSION_KEYS:
            self.count_builds(key)
Beispiel #36
0
    def handle_noargs(self, **options):
        quiet = options['quiet']
        force = options['force']
        no_git = options['no_git']
        clear_db = options['clear_db']
        if no_git or clear_db:
            force = True
        cloned = False
        repo = GitRepo(ADVISORIES_PATH,
                       ADVISORIES_REPO,
                       branch_name=ADVISORIES_BRANCH)
        modified_files = deleted_files = []

        def printout(msg, ending=None):
            if not quiet:
                self.stdout.write(msg, ending=ending)

        if clear_db:
            printout('Clearing all security advisories.')
            SecurityAdvisory.objects.all().delete()
            Product.objects.all().delete()

        if not no_git:
            printout('Updating repository.')
            modified_files, deleted_files = repo.update()
            if modified_files is None:
                cloned = True
            else:
                modified_files = filter_advisory_filenames(modified_files)
                deleted_files = filter_advisory_filenames(deleted_files)

        if force or cloned:
            printout('Reading all files.')
            modified_files = get_all_mfsa_files()
            if clear_db:
                deleted_files = []
            else:
                deleted_files = get_files_to_delete_from_db(modified_files)

        errors = []
        updates = 0
        if modified_files:
            for mf in modified_files:
                mf = os.path.join(ADVISORIES_PATH, mf)
                try:
                    update_db_from_file(mf)
                except Exception as e:
                    errors.append('ERROR parsing %s: %s' % (mf, e))
                    if not quiet:
                        sys.stdout.write('E')
                        sys.stdout.flush()
                    continue
                if not quiet:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                updates += 1
            printout('\nUpdated {0} files.'.format(updates))

        if deleted_files:
            deleted_files = filter_updated_from_deleted(
                modified_files, deleted_files)
            delete_files(deleted_files)
            printout('Deleted {0} files.'.format(len(deleted_files)))
            num_products = delete_orphaned_products()
            if num_products:
                printout('Deleted {0} orphaned products.'.format(num_products))

        if not modified_files and not deleted_files:
            printout('Nothing to update.')

        if errors:
            raise CommandError(
                'Encountered {0} errors:\n\n'.format(len(errors)) +
                '\n==========\n'.join(errors))