def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH,
settings.FLUENT_L10N_TEAM_REPO_URL)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH,
settings.FLUENT_REPO_URL)
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.EXTERNAL_FILES_PATH,
settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name="Community Data")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No community data updates")
return
self.output("Loading community data into database")
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo["type"])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output("Community data successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name='Legal Docs')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading legal docs into database')
count, errors = LegalDoc.objects.refresh()
self.output(f'{count} legal docs successfully loaded')
self.output(f'Encountered {errors} errors while loading docs')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
if not errors and settings.LEGAL_DOCS_DMS_URL:
requests.get(settings.LEGAL_DOCS_DMS_URL)
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name="Legal Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No legal docs updates")
self.snitch()
return
self.output("Loading legal docs into database")
count, errors = LegalDoc.objects.refresh()
self.output(f"{count} legal docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# only set latest if there are no errors so that it will try the errors again next time
# also so that it will fail again and thus not ping the snitch so that we'll be notified
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.snitch()
self.output("Done!")
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(
json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI)
super(Command, self).__init__(stdout, stderr, no_color)
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError(
'Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def handle(self, *args, **options):
if options['quiet']:
self.stdout._out = StringIO()
self.parser = NoisyFluentParser()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
self.update_fluent_files()
self.update_l10n_team_files()
no_errors = self.copy_ftl_files()
self.set_activation()
if no_errors:
self.stdout.write('There were no errors found in the .ftl files.')
else:
raise CommandError('Some errors were discovered in some .ftl files and they were not updated.'
'See above for details.')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name='Community Data')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No community data updates')
return
self.output('Loading community data into database')
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo['type'])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output('Community data successfully loaded')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
repo.update()
if not options['quiet']:
print('Release Notes Successfully Updated')
def update_fluent_files(self, clean=False):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
if clean:
rmtree(repo.path_str, ignore_errors=True)
self.stdout.write('Removed old .ftl repo')
repo.update()
self.stdout.write('Updated .ftl files')
def update_lang_files(self, clean=False):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
if clean:
rmtree(repo.path_str, ignore_errors=True)
self.stdout.write('Removed old .lang repo')
repo.update()
self.stdout.write('Updated .lang files')
def update_lang_files(self, clean=False):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
if clean:
repo.reclone()
else:
repo.update()
self.stdout.write('Updated .lang files')
def handle_safe(self, quiet, no_git, clear_db, **options):
force = no_git or clear_db
repo = GitRepo(
ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name="Security Advisories",
)
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout("Clearing all security advisories.")
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout("Updating repository.")
repo.update()
if not (force or repo.has_changes()):
printout("Nothing to update.")
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append(f"ERROR parsing {mf}: {e}")
if not quiet:
sys.stdout.write("E")
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write(".")
sys.stdout.flush()
updates += 1
printout(f"\nUpdated {updates} files.")
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout(f"Deleted {len(deleted_files)} files.")
num_products = delete_orphaned_products()
if num_products:
printout(f"Deleted {num_products} orphaned products.")
if errors:
raise CommandError(f"Encountered {len(errors)} errors:\n\n" +
"\n==========\n".join(errors))
repo.set_db_latest()
def update_fluent_files(self, clean=False):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
if clean:
repo.reclone()
else:
repo.update()
repo.update()
self.stdout.write("Updated .ftl files")
def update_fluent_files(self, clean=False):
for site, params in FLUENT_L10N_UPDATE_PARAMS.items():
repo = GitRepo(**params)
if clean:
repo.reclone()
else:
repo.update()
repo.update()
self.stdout.write(f"Updated .ftl files for {site}")
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
settings.PROD_DETAILS_JSON_REPO_BRANCH,
name='Product Details')
# fake last-modified string since the releng repo doesn't store those files
# and we rely on git commits for updates
self.last_modified = datetime.now().isoformat()
super(Command, self).__init__(stdout, stderr, no_color)
def get_data_version():
"""Add the git ref from the repo to the cache keys.
This will ensure that the cache is invalidated when the repo is updated.
"""
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
git_ref = repo.get_db_latest()
if git_ref is None:
git_ref = "default"
return git_ref
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
repo = GitRepo(settings.SITEMAPS_PATH,
settings.SITEMAPS_REPO,
name="Sitemaps")
self.stdout.write("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.stdout.write("No sitemap updates")
return
SitemapURL.objects.refresh()
repo.set_db_latest()
self.stdout.write("Updated sitemaps files")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH, name="Release Notes")
self.output("Updating git repo")
self.output(repo.update())
if not (options["force"] or repo.has_changes()):
self.output("No release note updates")
return
self.output("Loading releases into database")
count = ProductRelease.objects.refresh()
self.output(f"{count} release notes successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name="Content Cards")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No content card updates")
return
self.output("Loading content cards into database")
count = ContentCard.objects.refresh()
self.output(f"{count} content cards successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
data_path = settings.SITEMAPS_PATH.joinpath("data")
repo = GitRepo(settings.SITEMAPS_PATH, settings.SITEMAPS_REPO)
repo.update()
for src_path in data_path.rglob("*.*"):
rel_path = src_path.relative_to(data_path)
if rel_path.parts[0] == "sitemaps":
rel_path = rel_path.relative_to("sitemaps")
target_path = ROOT_FILES.joinpath(rel_path)
if target_path.exists():
if target_path.is_symlink():
continue
else:
target_path.unlink()
target_path.parent.mkdir(exist_ok=True)
target_path.symlink_to(src_path)
self.stdout.write("Updated sitemaps files")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WEBVISION_DOCS_PATH,
settings.WEBVISION_DOCS_REPO,
branch_name=settings.WEBVISION_DOCS_BRANCH,
name="Webvision Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No webvision docs updates")
return
self.output("Loading webvision docs into database")
count, errors = WebvisionDoc.objects.refresh()
self.output(f"{count} webvision docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# Only `set_db_latest` if there are no errors so that it will try without errors again next time.
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WWW_CONFIG_PATH,
settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH,
name="WWW Config")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No config updates")
return
self.output("Loading configs into database")
count = refresh_db_values()
if count:
self.output(f"{count} configs successfully loaded")
else:
self.output("No configs found. Please try again later.")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
def update_fluent_files(self):
repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO)
repo.update()
self.stdout.write('Updated .ftl files')
def update_l10n_team_files(self):
repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO)
repo.update()
self.stdout.write('Updated l10n team .ftl files')
def update_lang_files(self):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
self.stdout.write('Updated .lang files')
