def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name='Legal Docs')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading legal docs into database')
count, errors = LegalDoc.objects.refresh()
self.output(f'{count} legal docs successfully loaded')
self.output(f'Encountered {errors} errors while loading docs')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
if not errors and settings.LEGAL_DOCS_DMS_URL:
requests.get(settings.LEGAL_DOCS_DMS_URL)
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.EXTERNAL_FILES_PATH,
settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name="Community Data")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No community data updates")
return
self.output("Loading community data into database")
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo["type"])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output("Community data successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name='Community Data')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No community data updates')
return
self.output('Loading community data into database')
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo['type'])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output('Community data successfully loaded')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError(
'Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def handle(self, *args, **options):
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
repo.update()
if not options['quiet']:
print('Release Notes Successfully Updated')
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name="Legal Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No legal docs updates")
self.snitch()
return
self.output("Loading legal docs into database")
count, errors = LegalDoc.objects.refresh()
self.output(f"{count} legal docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# only set latest if there are no errors so that it will try the errors again next time
# also so that it will fail again and thus not ping the snitch so that we'll be notified
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.snitch()
self.output("Done!")
def update_lang_files(self, clean=False):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
if clean:
repo.reclone()
else:
repo.update()
self.stdout.write('Updated .lang files')
def update_fluent_files(self, clean=False):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
if clean:
rmtree(repo.path_str, ignore_errors=True)
self.stdout.write('Removed old .ftl repo')
repo.update()
self.stdout.write('Updated .ftl files')
def update_lang_files(self, clean=False):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
if clean:
rmtree(repo.path_str, ignore_errors=True)
self.stdout.write('Removed old .lang repo')
repo.update()
self.stdout.write('Updated .lang files')
def handle_safe(self, quiet, no_git, clear_db, **options):
force = no_git or clear_db
repo = GitRepo(
ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name="Security Advisories",
)
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout("Clearing all security advisories.")
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout("Updating repository.")
repo.update()
if not (force or repo.has_changes()):
printout("Nothing to update.")
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append(f"ERROR parsing {mf}: {e}")
if not quiet:
sys.stdout.write("E")
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write(".")
sys.stdout.flush()
updates += 1
printout(f"\nUpdated {updates} files.")
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout(f"Deleted {len(deleted_files)} files.")
num_products = delete_orphaned_products()
if num_products:
printout(f"Deleted {num_products} orphaned products.")
if errors:
raise CommandError(f"Encountered {len(errors)} errors:\n\n" +
"\n==========\n".join(errors))
repo.set_db_latest()
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def update_fluent_files(self, clean=False):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
if clean:
repo.reclone()
else:
repo.update()
repo.update()
self.stdout.write("Updated .ftl files")
def update_fluent_files(self, clean=False):
for site, params in FLUENT_L10N_UPDATE_PARAMS.items():
repo = GitRepo(**params)
if clean:
repo.reclone()
else:
repo.update()
repo.update()
self.stdout.write(f"Updated .ftl files for {site}")
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
repo = GitRepo(settings.SITEMAPS_PATH,
settings.SITEMAPS_REPO,
name="Sitemaps")
self.stdout.write("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.stdout.write("No sitemap updates")
return
SitemapURL.objects.refresh()
repo.set_db_latest()
self.stdout.write("Updated sitemaps files")
class FTLRepoCommand(BaseCommand):
meao_repo = None
l10n_repo = None
def add_arguments(self, parser):
parser.add_argument("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="If no error occurs, swallow all output.")
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH,
settings.FLUENT_L10N_TEAM_REPO_URL,
settings.FLUENT_L10N_TEAM_REPO_BRANCH)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH,
settings.FLUENT_REPO_URL,
settings.FLUENT_REPO_BRANCH)
def update_l10n_team_files(self):
try:
# this will fail on first run
self.l10n_repo.clean()
except FileNotFoundError:
pass
self.l10n_repo.update()
self.stdout.write(
f"Updated l10n team .ftl files for {settings.FLUENT_L10N_TEAM_REPO_URL}"
)
def update_fluent_files(self):
try:
self.meao_repo.clean()
except FileNotFoundError:
pass
self.meao_repo.update()
self.stdout.write(f"Updated .ftl files for {settings.FLUENT_REPO_URL}")
def config_git(self):
"""Set user config so that committing will work"""
for repo in (self.meao_repo, self.l10n_repo):
repo.git("config", "user.email", GIT_COMMIT_EMAIL)
repo.git("config", "user.name", GIT_COMMIT_NAME)
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name="Content Cards")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No content card updates")
return
self.output("Loading content cards into database")
count = ContentCard.objects.refresh()
self.output(f"{count} content cards successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WEBVISION_DOCS_PATH,
settings.WEBVISION_DOCS_REPO,
branch_name=settings.WEBVISION_DOCS_BRANCH,
name="Webvision Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No webvision docs updates")
return
self.output("Loading webvision docs into database")
count, errors = WebvisionDoc.objects.refresh()
self.output(f"{count} webvision docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# Only `set_db_latest` if there are no errors so that it will try without errors again next time.
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
data_path = settings.SITEMAPS_PATH.joinpath("data")
repo = GitRepo(settings.SITEMAPS_PATH, settings.SITEMAPS_REPO)
repo.update()
for src_path in data_path.rglob("*.*"):
rel_path = src_path.relative_to(data_path)
if rel_path.parts[0] == "sitemaps":
rel_path = rel_path.relative_to("sitemaps")
target_path = ROOT_FILES.joinpath(rel_path)
if target_path.exists():
if target_path.is_symlink():
continue
else:
target_path.unlink()
target_path.parent.mkdir(exist_ok=True)
target_path.symlink_to(src_path)
self.stdout.write("Updated sitemaps files")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WWW_CONFIG_PATH,
settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH,
name="WWW Config")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No config updates")
return
self.output("Loading configs into database")
count = refresh_db_values()
if count:
self.output(f"{count} configs successfully loaded")
else:
self.output("No configs found. Please try again later.")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH, name="Release Notes")
self.output("Updating git repo")
self.output(repo.update())
if not (options["force"] or repo.has_changes()):
self.output("No release note updates")
return
self.output("Loading releases into database")
count = ProductRelease.objects.refresh()
self.output(f"{count} release notes successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
class Command(BaseCommand):
help = 'Processes .ftl files from l10n team for use in bedrock'
meao_repo = None
l10n_repo = None
parser = None
def add_arguments(self, parser):
parser.add_argument('-q',
'--quiet',
action='store_true',
dest='quiet',
default=False,
help='If no error occurs, swallow all output.')
parser.add_argument(
'--push',
action='store_true',
dest='push',
default=False,
help='Push the changes to the MEAO Fluent files repo.')
def handle(self, *args, **options):
if options['quiet']:
self.stdout._out = StringIO()
self.parser = NoisyFluentParser()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH,
settings.FLUENT_L10N_TEAM_REPO)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH,
settings.FLUENT_REPO)
self.update_fluent_files()
self.update_l10n_team_files()
no_errors = self.copy_ftl_files()
self.set_activation()
self.copy_configs()
if options['push']:
changes = self.commit_changes()
if changes:
self.push_changes()
if not no_errors:
raise CommandError(
'Some errors were discovered in some .ftl files and they were not updated.'
'See above for details.')
def update_l10n_team_files(self):
try:
# this will fail on first run
self.l10n_repo.clean()
except FileNotFoundError:
pass
self.l10n_repo.update()
self.stdout.write('Updated l10n team .ftl files')
def update_fluent_files(self):
try:
self.meao_repo.clean()
except FileNotFoundError:
pass
self.meao_repo.update()
self.stdout.write('Updated .ftl files')
def config_fluent_repo(self):
"""Set user config so that committing will work"""
self.meao_repo.git('config', 'user.email', GIT_COMMIT_EMAIL)
self.meao_repo.git('config', 'user.name', GIT_COMMIT_NAME)
def commit_changes(self):
self.config_fluent_repo()
self.meao_repo.git('add', '.')
try:
self.meao_repo.git('commit', '-m', 'Update files from l10n repo')
except CalledProcessError:
self.stdout.write('No changes to commit')
return False
self.stdout.write('Committed changes to local repo')
return True
def push_changes(self):
try:
self.meao_repo.git('push', self.git_push_url, 'HEAD:master')
except CalledProcessError:
raise CommandError(
f'There was a problem pushing to {self.meao_repo.remote_url}')
commit = self.meao_repo.git('rev-parse', '--short', 'HEAD')
self.stdout.write(f'Pushed {commit} to {self.meao_repo.remote_url}')
@property
def git_push_url(self):
if not settings.FLUENT_REPO_AUTH:
raise CommandError('Git push authentication not configured')
return self.meao_repo.remote_url_auth(settings.FLUENT_REPO_AUTH)
def _copy_file(self, filepath):
relative_filepath = filepath.relative_to(self.l10n_repo.path)
to_filepath = self.meao_repo.path.joinpath(relative_filepath)
to_filepath.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(filepath), str(to_filepath))
self.stdout.write('.', ending='')
self.stdout.flush()
def copy_configs(self):
count = 0
for filepath in self.l10n_repo.path.rglob('*.toml'):
self._copy_file(filepath)
count += 1
self.stdout.write(f'\nCopied {count} .toml files')
def copy_ftl_files(self):
count = 0
errors = []
for filepath in self.l10n_repo.path.rglob('*.ftl'):
if not self.lint_ftl_file(filepath):
errors.append(filepath.relative_to(self.l10n_repo.path))
continue
self._copy_file(filepath)
count += 1
self.stdout.write(f'\nCopied {count} .ftl files')
if errors:
self.stdout.write(
'The following files had parse errors and were not copied:')
for fpath in errors:
self.stdout.write(f'- {fpath}')
return False
return True
def lint_ftl_file(self, filepath):
with filepath.open() as ftl:
try:
self.parser.parse(ftl.read())
except ParseError:
return False
return True
def set_activation(self):
updated_ftl = set()
modified, _ = self.meao_repo.modified_files()
for fname in modified:
if not fname.endswith('.ftl'):
continue
locale, ftl_name = fname.split('/', 1)
updated_ftl.add(ftl_name)
for ftl_name in updated_ftl:
self.calculate_activation(ftl_name)
def calculate_activation(self, ftl_file):
translations = self.meao_repo.path.glob(f'*/{ftl_file}')
metadata = get_metadata(ftl_file)
active_locales = metadata.get('active_locales', [])
inactive_locales = metadata.get('inactive_locales', [])
percent_required = metadata.get(
'percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED)
all_locales = {
str(x.relative_to(self.meao_repo.path)).split('/', 1)[0]
for x in translations
}
locales_to_check = all_locales.difference(['en'], active_locales,
inactive_locales)
new_activations = []
for locale in locales_to_check:
l10n = fluent_l10n([locale, 'en'], [ftl_file])
if not l10n.has_required_messages:
continue
percent_trans = l10n.percent_translated
if percent_trans < percent_required:
continue
new_activations.append(locale)
if new_activations:
active_locales.extend(new_activations)
metadata['active_locales'] = sorted(active_locales)
write_metadata(ftl_file, metadata)
self.stdout.write(
f'Activated {len(new_activations)} new locales for {ftl_file}')
def handle_noargs(self, **options):
quiet = options['quiet']
force = options['force']
no_git = options['no_git']
clear_db = options['clear_db']
if no_git or clear_db:
force = True
cloned = False
repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH)
modified_files = deleted_files = []
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
if not no_git:
printout('Updating repository.')
modified_files, deleted_files = repo.update()
if modified_files is None:
cloned = True
else:
modified_files = filter_advisory_filenames(modified_files)
deleted_files = filter_advisory_filenames(deleted_files)
if force or cloned:
printout('Reading all files.')
modified_files = get_all_mfsa_files()
if clear_db:
deleted_files = []
else:
deleted_files = get_files_to_delete_from_db(modified_files)
errors = []
updates = 0
if modified_files:
for mf in modified_files:
mf = os.path.join(ADVISORIES_PATH, mf)
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if deleted_files:
deleted_files = filter_updated_from_deleted(modified_files, deleted_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if not modified_files and not deleted_files:
printout('Nothing to update.')
if errors:
raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
def handle(self, *args, **options):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
def update_fluent_files(self):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
repo.update()
self.stdout.write('Updated .ftl files')
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
def update_l10n_team_files(self):
repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
repo.update()
self.stdout.write('Updated l10n team .ftl files')
class Command(BaseCommand):
help = 'Processes .ftl files from l10n team for use in bedrock'
meao_repo = None
l10n_repo = None
parser = None
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
def handle(self, *args, **options):
if options['quiet']:
self.stdout._out = StringIO()
self.parser = NoisyFluentParser()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
self.update_fluent_files()
self.update_l10n_team_files()
no_errors = self.copy_ftl_files()
self.set_activation()
if no_errors:
self.stdout.write('There were no errors found in the .ftl files.')
else:
raise CommandError('Some errors were discovered in some .ftl files and they were not updated.'
'See above for details.')
def update_l10n_team_files(self):
try:
# this will fail on first run
self.l10n_repo.clean()
except FileNotFoundError:
pass
self.l10n_repo.update()
self.stdout.write('Updated l10n team .ftl files')
def update_fluent_files(self):
try:
self.meao_repo.clean()
except FileNotFoundError:
pass
self.meao_repo.update()
self.stdout.write('Updated .ftl files')
def copy_ftl_files(self):
count = 0
errors = []
for filepath in self.l10n_repo.path.rglob('*.ftl'):
relative_filepath = filepath.relative_to(self.l10n_repo.path)
if not self.lint_ftl_file(filepath):
errors.append(relative_filepath)
continue
to_filepath = self.meao_repo.path.joinpath(relative_filepath)
to_filepath.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(filepath), str(to_filepath))
count += 1
self.stdout.write('.', ending='')
self.stdout.flush()
self.stdout.write(f'\nCopied {count} .ftl files')
if errors:
self.stdout.write('The following files had parse errors and were not copied:')
for fpath in errors:
self.stdout.write(f'- {fpath}')
return False
return True
def lint_ftl_file(self, filepath):
with filepath.open() as ftl:
try:
self.parser.parse(ftl.read())
except ParseError:
return False
return True
def set_activation(self):
updated_ftl = set()
modified, _ = self.meao_repo.modified_files()
for fname in modified:
locale, ftl_name = fname.split('/', 1)
updated_ftl.add(ftl_name)
for ftl_name in updated_ftl:
self.calculate_activation(ftl_name)
def calculate_activation(self, ftl_file):
translations = self.meao_repo.path.glob(f'*/{ftl_file}')
metadata = get_metadata(ftl_file)
active_locales = metadata.get('active_locales', [])
inactive_locales = metadata.get('inactive_locales', [])
percent_required = metadata.get('percent_required', settings.FLUENT_DEFAULT_PERCENT_REQUIRED)
all_locales = {str(x.relative_to(self.meao_repo.path)).split('/', 1)[0] for x in translations}
locales_to_check = all_locales.difference(['en'], active_locales, inactive_locales)
new_activations = []
for locale in locales_to_check:
l10n = fluent_l10n([locale, 'en'], [ftl_file])
if not l10n.has_required_messages:
continue
percent_trans = l10n.percent_translated
if percent_trans < percent_required:
continue
new_activations.append(locale)
if new_activations:
active_locales.extend(new_activations)
metadata['active_locales'] = sorted(active_locales)
write_metadata(ftl_file, metadata)
self.stdout.write(f'Activated {len(new_activations)} new locales for {ftl_file}')
def update_lang_files(self):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
self.stdout.write('Updated .lang files')
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI)
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
def handle_noargs(self, **options):
quiet = options['quiet']
force = options['force']
no_git = options['no_git']
clear_db = options['clear_db']
if no_git or clear_db:
force = True
cloned = False
repo = GitRepo(ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH)
modified_files = deleted_files = []
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
if not no_git:
printout('Updating repository.')
modified_files, deleted_files = repo.update()
if modified_files is None:
cloned = True
else:
modified_files = filter_advisory_filenames(modified_files)
deleted_files = filter_advisory_filenames(deleted_files)
if force or cloned:
printout('Reading all files.')
modified_files = get_all_mfsa_files()
if clear_db:
deleted_files = []
else:
deleted_files = get_files_to_delete_from_db(modified_files)
errors = []
updates = 0
if modified_files:
for mf in modified_files:
mf = os.path.join(ADVISORIES_PATH, mf)
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if deleted_files:
deleted_files = filter_updated_from_deleted(
modified_files, deleted_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if not modified_files and not deleted_files:
printout('Nothing to update.')
if errors:
raise CommandError(
'Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
