def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.LEGAL_DOCS_PATH, settings.LEGAL_DOCS_REPO, branch_name=settings.LEGAL_DOCS_BRANCH, name='Legal Docs') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No content card updates') return self.output('Loading legal docs into database') count, errors = LegalDoc.objects.refresh() self.output(f'{count} legal docs successfully loaded') self.output(f'Encountered {errors} errors while loading docs') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!') if not errors and settings.LEGAL_DOCS_DMS_URL: requests.get(settings.LEGAL_DOCS_DMS_URL)
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO, branch_name=settings.EXTERNAL_FILES_BRANCH, name="Community Data") self.output("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.output("No community data updates") return self.output("Loading community data into database") for fid, finfo in settings.EXTERNAL_FILES.items(): klass = import_string(finfo["type"]) try: klass(fid).update() except ValueError as e: raise CommandError(str(e)) self.output("Community data successfully loaded") repo.set_db_latest() self.output("Saved latest git repo state to database") self.output("Done!")
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO, branch_name=settings.EXTERNAL_FILES_BRANCH, name='Community Data') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No community data updates') return self.output('Loading community data into database') for fid, finfo in settings.EXTERNAL_FILES.items(): klass = import_string(finfo['type']) try: klass(fid).update() except ValueError as e: raise CommandError(str(e)) self.output('Community data successfully loaded') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle_noargs(self, **options): quiet = options['quiet'] no_git = options['no_git'] clear_db = options['clear_db'] force = no_git or clear_db repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH, name='Security Advisories') def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() MitreCVE.objects.all().delete() if not no_git: printout('Updating repository.') repo.update() if not (force or repo.has_changes()): printout('Nothing to update.') return errors = [] updates = 0 all_files = get_all_file_names() for mf in all_files: try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if not clear_db: deleted_files = get_files_to_delete_from_db(all_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if errors: raise CommandError( 'Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors)) repo.set_db_latest()
def handle(self, *args, **options): repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH) repo.update() if not options['quiet']: print('Release Notes Successfully Updated')
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.LEGAL_DOCS_PATH, settings.LEGAL_DOCS_REPO, branch_name=settings.LEGAL_DOCS_BRANCH, name="Legal Docs") self.output("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.output("No legal docs updates") self.snitch() return self.output("Loading legal docs into database") count, errors = LegalDoc.objects.refresh() self.output(f"{count} legal docs successfully loaded") if errors: self.output(f"Encountered {errors} errors while loading docs") else: # only set latest if there are no errors so that it will try the errors again next time # also so that it will fail again and thus not ping the snitch so that we'll be notified repo.set_db_latest() self.output("Saved latest git repo state to database") self.snitch() self.output("Done!")
def update_lang_files(self, clean=False): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) if clean: repo.reclone() else: repo.update() self.stdout.write('Updated .lang files')
def update_fluent_files(self, clean=False): repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL) if clean: rmtree(repo.path_str, ignore_errors=True) self.stdout.write('Removed old .ftl repo') repo.update() self.stdout.write('Updated .ftl files')
def update_lang_files(self, clean=False): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) if clean: rmtree(repo.path_str, ignore_errors=True) self.stdout.write('Removed old .lang repo') repo.update() self.stdout.write('Updated .lang files')
def handle_safe(self, quiet, no_git, clear_db, **options): force = no_git or clear_db repo = GitRepo( ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH, name="Security Advisories", ) def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout("Clearing all security advisories.") SecurityAdvisory.objects.all().delete() Product.objects.all().delete() MitreCVE.objects.all().delete() if not no_git: printout("Updating repository.") repo.update() if not (force or repo.has_changes()): printout("Nothing to update.") return errors = [] updates = 0 all_files = get_all_file_names() for mf in all_files: try: update_db_from_file(mf) except Exception as e: errors.append(f"ERROR parsing {mf}: {e}") if not quiet: sys.stdout.write("E") sys.stdout.flush() continue if not quiet: sys.stdout.write(".") sys.stdout.flush() updates += 1 printout(f"\nUpdated {updates} files.") if not clear_db: deleted_files = get_files_to_delete_from_db(all_files) delete_files(deleted_files) printout(f"Deleted {len(deleted_files)} files.") num_products = delete_orphaned_products() if num_products: printout(f"Deleted {num_products} orphaned products.") if errors: raise CommandError(f"Encountered {len(errors)} errors:\n\n" + "\n==========\n".join(errors)) repo.set_db_latest()
def handle_noargs(self, **options): quiet = options['quiet'] no_git = options['no_git'] clear_db = options['clear_db'] force = no_git or clear_db repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH, name='Security Advisories') def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() MitreCVE.objects.all().delete() if not no_git: printout('Updating repository.') repo.update() if not (force or repo.has_changes()): printout('Nothing to update.') return errors = [] updates = 0 all_files = get_all_file_names() for mf in all_files: try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if not clear_db: deleted_files = get_files_to_delete_from_db(all_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if errors: raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors)) repo.set_db_latest()
def update_fluent_files(self, clean=False): repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL) if clean: repo.reclone() else: repo.update() repo.update() self.stdout.write("Updated .ftl files")
def update_fluent_files(self, clean=False): for site, params in FLUENT_L10N_UPDATE_PARAMS.items(): repo = GitRepo(**params) if clean: repo.reclone() else: repo.update() repo.update() self.stdout.write(f"Updated .ftl files for {site}")
def handle(self, *args, **options): if options["quiet"]: self.stdout._out = StringIO() repo = GitRepo(settings.SITEMAPS_PATH, settings.SITEMAPS_REPO, name="Sitemaps") self.stdout.write("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.stdout.write("No sitemap updates") return SitemapURL.objects.refresh() repo.set_db_latest() self.stdout.write("Updated sitemaps files")
class FTLRepoCommand(BaseCommand): meao_repo = None l10n_repo = None def add_arguments(self, parser): parser.add_argument("-q", "--quiet", action="store_true", dest="quiet", default=False, help="If no error occurs, swallow all output.") def handle(self, *args, **options): if options["quiet"]: self.stdout._out = StringIO() self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO_URL, settings.FLUENT_L10N_TEAM_REPO_BRANCH) self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL, settings.FLUENT_REPO_BRANCH) def update_l10n_team_files(self): try: # this will fail on first run self.l10n_repo.clean() except FileNotFoundError: pass self.l10n_repo.update() self.stdout.write( f"Updated l10n team .ftl files for {settings.FLUENT_L10N_TEAM_REPO_URL}" ) def update_fluent_files(self): try: self.meao_repo.clean() except FileNotFoundError: pass self.meao_repo.update() self.stdout.write(f"Updated .ftl files for {settings.FLUENT_REPO_URL}") def config_git(self): """Set user config so that committing will work""" for repo in (self.meao_repo, self.l10n_repo): repo.git("config", "user.email", GIT_COMMIT_EMAIL) repo.git("config", "user.name", GIT_COMMIT_NAME)
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name="Content Cards") self.output("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.output("No content card updates") return self.output("Loading content cards into database") count = ContentCard.objects.refresh() self.output(f"{count} content cards successfully loaded") repo.set_db_latest() self.output("Saved latest git repo state to database") self.output("Done!")
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No content card updates') return self.output('Loading content cards into database') count = ContentCard.objects.refresh() self.output('%s content cards successfully loaded' % count) repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH) self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No release note updates') return self.output('Loading releases into database') count = ProductRelease.objects.refresh() self.output('%s release notes successfully loaded' % count) repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO, branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No config updates') return self.output('Loading configs into database') count = refresh_db_values() if count: self.output('%s configs successfully loaded' % count) else: self.output('No configs found. Please try again later.') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO, branch_name=settings.WWW_CONFIG_BRANCH) self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No config updates') return self.output('Loading configs into database') count = refresh_db_values() if count: self.output('%s configs successfully loaded' % count) else: self.output('No configs found. Please try again later.') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.WEBVISION_DOCS_PATH, settings.WEBVISION_DOCS_REPO, branch_name=settings.WEBVISION_DOCS_BRANCH, name="Webvision Docs") self.output("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.output("No webvision docs updates") return self.output("Loading webvision docs into database") count, errors = WebvisionDoc.objects.refresh() self.output(f"{count} webvision docs successfully loaded") if errors: self.output(f"Encountered {errors} errors while loading docs") else: # Only `set_db_latest` if there are no errors so that it will try without errors again next time. repo.set_db_latest() self.output("Saved latest git repo state to database") self.output("Done!")
def handle(self, *args, **options): if options["quiet"]: self.stdout._out = StringIO() data_path = settings.SITEMAPS_PATH.joinpath("data") repo = GitRepo(settings.SITEMAPS_PATH, settings.SITEMAPS_REPO) repo.update() for src_path in data_path.rglob("*.*"): rel_path = src_path.relative_to(data_path) if rel_path.parts[0] == "sitemaps": rel_path = rel_path.relative_to("sitemaps") target_path = ROOT_FILES.joinpath(rel_path) if target_path.exists(): if target_path.is_symlink(): continue else: target_path.unlink() target_path.parent.mkdir(exist_ok=True) target_path.symlink_to(src_path) self.stdout.write("Updated sitemaps files")
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO, branch_name=settings.WWW_CONFIG_BRANCH, name="WWW Config") self.output("Updating git repo") repo.update() if not (options["force"] or repo.has_changes()): self.output("No config updates") return self.output("Loading configs into database") count = refresh_db_values() if count: self.output(f"{count} configs successfully loaded") else: self.output("No configs found. Please try again later.") repo.set_db_latest() self.output("Saved latest git repo state to database") self.output("Done!")
def handle(self, *args, **options): self.quiet = options["quiet"] repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH, name="Release Notes") self.output("Updating git repo") self.output(repo.update()) if not (options["force"] or repo.has_changes()): self.output("No release note updates") return self.output("Loading releases into database") count = ProductRelease.objects.refresh() self.output(f"{count} release notes successfully loaded") repo.set_db_latest() self.output("Saved latest git repo state to database") self.output("Done!")
class Command(BaseCommand): help = 'Processes .ftl files from l10n team for use in bedrock' meao_repo = None l10n_repo = None parser = None def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.') parser.add_argument( '--push', action='store_true', dest='push', default=False, help='Push the changes to the MEAO Fluent files repo.') def handle(self, *args, **options): if options['quiet']: self.stdout._out = StringIO() self.parser = NoisyFluentParser() self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO) self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO) self.update_fluent_files() self.update_l10n_team_files() no_errors = self.copy_ftl_files() self.set_activation() self.copy_configs() if options['push']: changes = self.commit_changes() if changes: self.push_changes() if not no_errors: raise CommandError( 'Some errors were discovered in some .ftl files and they were not updated.' 'See above for details.') def update_l10n_team_files(self): try: # this will fail on first run self.l10n_repo.clean() except FileNotFoundError: pass self.l10n_repo.update() self.stdout.write('Updated l10n team .ftl files') def update_fluent_files(self): try: self.meao_repo.clean() except FileNotFoundError: pass self.meao_repo.update() self.stdout.write('Updated .ftl files') def config_fluent_repo(self): """Set user config so that committing will work""" self.meao_repo.git('config', 'user.email', GIT_COMMIT_EMAIL) self.meao_repo.git('config', 'user.name', GIT_COMMIT_NAME) def commit_changes(self): self.config_fluent_repo() self.meao_repo.git('add', '.') try: self.meao_repo.git('commit', '-m', 'Update files from l10n repo') except CalledProcessError: self.stdout.write('No changes to commit') return False self.stdout.write('Committed changes to local repo') return True def push_changes(self): try: self.meao_repo.git('push', self.git_push_url, 'HEAD:master') except CalledProcessError: raise CommandError( f'There was a problem pushing to {self.meao_repo.remote_url}') commit = self.meao_repo.git('rev-parse', '--short', 'HEAD') self.stdout.write(f'Pushed {commit} to {self.meao_repo.remote_url}') @property def git_push_url(self): if not settings.FLUENT_REPO_AUTH: raise CommandError('Git push authentication not configured') return self.meao_repo.remote_url_auth(settings.FLUENT_REPO_AUTH) def _copy_file(self, filepath): relative_filepath = filepath.relative_to(self.l10n_repo.path) to_filepath = self.meao_repo.path.joinpath(relative_filepath) to_filepath.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(filepath), str(to_filepath)) self.stdout.write('.', ending='') self.stdout.flush() def copy_configs(self): count = 0 for filepath in self.l10n_repo.path.rglob('*.toml'): self._copy_file(filepath) count += 1 self.stdout.write(f'\nCopied {count} .toml files') def copy_ftl_files(self): count = 0 errors = [] for filepath in self.l10n_repo.path.rglob('*.ftl'): if not self.lint_ftl_file(filepath): errors.append(filepath.relative_to(self.l10n_repo.path)) continue self._copy_file(filepath) count += 1 self.stdout.write(f'\nCopied {count} .ftl files') if errors: self.stdout.write( 'The following files had parse errors and were not copied:') for fpath in errors: self.stdout.write(f'- {fpath}') return False return True def lint_ftl_file(self, filepath): with filepath.open() as ftl: try: self.parser.parse(ftl.read()) except ParseError: return False return True def set_activation(self): updated_ftl = set() modified, _ = self.meao_repo.modified_files() for fname in modified: if not fname.endswith('.ftl'): continue locale, ftl_name = fname.split('/', 1) updated_ftl.add(ftl_name) for ftl_name in updated_ftl: self.calculate_activation(ftl_name) def calculate_activation(self, ftl_file): translations = self.meao_repo.path.glob(f'*/{ftl_file}') metadata = get_metadata(ftl_file) active_locales = metadata.get('active_locales', []) inactive_locales = metadata.get('inactive_locales', []) percent_required = metadata.get( 'percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED) all_locales = { str(x.relative_to(self.meao_repo.path)).split('/', 1)[0] for x in translations } locales_to_check = all_locales.difference(['en'], active_locales, inactive_locales) new_activations = [] for locale in locales_to_check: l10n = fluent_l10n([locale, 'en'], [ftl_file]) if not l10n.has_required_messages: continue percent_trans = l10n.percent_translated if percent_trans < percent_required: continue new_activations.append(locale) if new_activations: active_locales.extend(new_activations) metadata['active_locales'] = sorted(active_locales) write_metadata(ftl_file, metadata) self.stdout.write( f'Activated {len(new_activations)} new locales for {ftl_file}')
def handle_noargs(self, **options): quiet = options['quiet'] force = options['force'] no_git = options['no_git'] clear_db = options['clear_db'] if no_git or clear_db: force = True cloned = False repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH) modified_files = deleted_files = [] def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() if not no_git: printout('Updating repository.') modified_files, deleted_files = repo.update() if modified_files is None: cloned = True else: modified_files = filter_advisory_filenames(modified_files) deleted_files = filter_advisory_filenames(deleted_files) if force or cloned: printout('Reading all files.') modified_files = get_all_mfsa_files() if clear_db: deleted_files = [] else: deleted_files = get_files_to_delete_from_db(modified_files) errors = [] updates = 0 if modified_files: for mf in modified_files: mf = os.path.join(ADVISORIES_PATH, mf) try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if deleted_files: deleted_files = filter_updated_from_deleted(modified_files, deleted_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if not modified_files and not deleted_files: printout('Nothing to update.') if errors: raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors))
def handle(self, *args, **options): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) repo.update()
def update_fluent_files(self): repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO) repo.update() self.stdout.write('Updated .ftl files')
class Command(BaseCommand): def __init__(self, stdout=None, stderr=None, no_color=False): self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR) self.db_storage = PDDatabaseStorage() self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH, settings.PROD_DETAILS_JSON_REPO_URI, name='Product Details') super(Command, self).__init__(stdout, stderr, no_color) def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.'), parser.add_argument('--database', default='default', help=('Specifies the database to use, if using a db. ' 'Defaults to "default".')), def handle(self, *args, **options): # don't really care about deleted files. almost never happens in p-d. if not self.update_file_data(): if not options['quiet']: print('Product Details data was already up to date') return try: self.validate_data() except Exception: raise CommandError('Product Details data is invalid') if not options['quiet']: print('Product Details data is valid') if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'): # no need to continue if not using DB backend return self.load_changes(options, self.file_storage.all_json_files()) self.repo.set_db_latest() if not options['quiet']: print('Product Details data update is complete') def load_changes(self, options, modified_files): with transaction.atomic(using=options['database']): for filename in modified_files: self.db_storage.update(filename, self.file_storage.content(filename), self.file_storage.last_modified(filename)) if not options['quiet']: print('Updated ' + filename) self.db_storage.update('/', '', self.file_storage.last_modified('/')) self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/')) def update_file_data(self): self.repo.update() return self.repo.has_changes() def count_builds(self, version_key, min_builds=20): version = self.file_storage.data('firefox_versions.json')[version_key] if not version: if version_key == 'FIREFOX_ESR_NEXT': return builds = len([locale for locale, build in self.file_storage.data('firefox_primary_builds.json').items() if version in build]) if builds < min_builds: raise ValueError('Too few builds for {}'.format(version_key)) def validate_data(self): self.file_storage.clear_cache() for key in FIREFOX_VERSION_KEYS: self.count_builds(key)
def update_l10n_team_files(self): repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO) repo.update() self.stdout.write('Updated l10n team .ftl files')
class Command(BaseCommand): help = 'Processes .ftl files from l10n team for use in bedrock' meao_repo = None l10n_repo = None parser = None def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.'), def handle(self, *args, **options): if options['quiet']: self.stdout._out = StringIO() self.parser = NoisyFluentParser() self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO) self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO) self.update_fluent_files() self.update_l10n_team_files() no_errors = self.copy_ftl_files() self.set_activation() if no_errors: self.stdout.write('There were no errors found in the .ftl files.') else: raise CommandError('Some errors were discovered in some .ftl files and they were not updated.' 'See above for details.') def update_l10n_team_files(self): try: # this will fail on first run self.l10n_repo.clean() except FileNotFoundError: pass self.l10n_repo.update() self.stdout.write('Updated l10n team .ftl files') def update_fluent_files(self): try: self.meao_repo.clean() except FileNotFoundError: pass self.meao_repo.update() self.stdout.write('Updated .ftl files') def copy_ftl_files(self): count = 0 errors = [] for filepath in self.l10n_repo.path.rglob('*.ftl'): relative_filepath = filepath.relative_to(self.l10n_repo.path) if not self.lint_ftl_file(filepath): errors.append(relative_filepath) continue to_filepath = self.meao_repo.path.joinpath(relative_filepath) to_filepath.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(filepath), str(to_filepath)) count += 1 self.stdout.write('.', ending='') self.stdout.flush() self.stdout.write(f'\nCopied {count} .ftl files') if errors: self.stdout.write('The following files had parse errors and were not copied:') for fpath in errors: self.stdout.write(f'- {fpath}') return False return True def lint_ftl_file(self, filepath): with filepath.open() as ftl: try: self.parser.parse(ftl.read()) except ParseError: return False return True def set_activation(self): updated_ftl = set() modified, _ = self.meao_repo.modified_files() for fname in modified: locale, ftl_name = fname.split('/', 1) updated_ftl.add(ftl_name) for ftl_name in updated_ftl: self.calculate_activation(ftl_name) def calculate_activation(self, ftl_file): translations = self.meao_repo.path.glob(f'*/{ftl_file}') metadata = get_metadata(ftl_file) active_locales = metadata.get('active_locales', []) inactive_locales = metadata.get('inactive_locales', []) percent_required = metadata.get('percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED) all_locales = {str(x.relative_to(self.meao_repo.path)).split('/', 1)[0] for x in translations} locales_to_check = all_locales.difference(['en'], active_locales, inactive_locales) new_activations = [] for locale in locales_to_check: l10n = fluent_l10n([locale, 'en'], [ftl_file]) if not l10n.has_required_messages: continue percent_trans = l10n.percent_translated if percent_trans < percent_required: continue new_activations.append(locale) if new_activations: active_locales.extend(new_activations) metadata['active_locales'] = sorted(active_locales) write_metadata(ftl_file, metadata) self.stdout.write(f'Activated {len(new_activations)} new locales for {ftl_file}')
def update_lang_files(self): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) repo.update() self.stdout.write('Updated .lang files')
class Command(BaseCommand): def __init__(self, stdout=None, stderr=None, no_color=False): self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR) self.db_storage = PDDatabaseStorage() self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH, settings.PROD_DETAILS_JSON_REPO_URI) super(Command, self).__init__(stdout, stderr, no_color) def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.'), parser.add_argument('--database', default='default', help=('Specifies the database to use, if using a db. ' 'Defaults to "default".')), def handle(self, *args, **options): # don't really care about deleted files. almost never happens in p-d. if not self.update_file_data(): if not options['quiet']: print('Product Details data was already up to date') return try: self.validate_data() except Exception: raise CommandError('Product Details data is invalid') if not options['quiet']: print('Product Details data is valid') if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'): # no need to continue if not using DB backend return self.load_changes(options, self.file_storage.all_json_files()) self.repo.set_db_latest() if not options['quiet']: print('Product Details data update is complete') def load_changes(self, options, modified_files): with transaction.atomic(using=options['database']): for filename in modified_files: self.db_storage.update(filename, self.file_storage.content(filename), self.file_storage.last_modified(filename)) if not options['quiet']: print('Updated ' + filename) self.db_storage.update('/', '', self.file_storage.last_modified('/')) self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/')) def update_file_data(self): self.repo.update() return self.repo.has_changes() def count_builds(self, version_key, min_builds=20): version = self.file_storage.data('firefox_versions.json')[version_key] if not version: if version_key == 'FIREFOX_ESR_NEXT': return builds = len([locale for locale, build in self.file_storage.data('firefox_primary_builds.json').items() if version in build]) if builds < min_builds: raise ValueError('Too few builds for {}'.format(version_key)) def validate_data(self): self.file_storage.clear_cache() for key in FIREFOX_VERSION_KEYS: self.count_builds(key)
def handle_noargs(self, **options): quiet = options['quiet'] force = options['force'] no_git = options['no_git'] clear_db = options['clear_db'] if no_git or clear_db: force = True cloned = False repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH) modified_files = deleted_files = [] def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() if not no_git: printout('Updating repository.') modified_files, deleted_files = repo.update() if modified_files is None: cloned = True else: modified_files = filter_advisory_filenames(modified_files) deleted_files = filter_advisory_filenames(deleted_files) if force or cloned: printout('Reading all files.') modified_files = get_all_mfsa_files() if clear_db: deleted_files = [] else: deleted_files = get_files_to_delete_from_db(modified_files) errors = [] updates = 0 if modified_files: for mf in modified_files: mf = os.path.join(ADVISORIES_PATH, mf) try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if deleted_files: deleted_files = filter_updated_from_deleted( modified_files, deleted_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if not modified_files and not deleted_files: printout('Nothing to update.') if errors: raise CommandError( 'Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors))