def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name='Community Data')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No community data updates')
return
self.output('Loading community data into database')
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo['type'])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output('Community data successfully loaded')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name="Legal Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No legal docs updates")
self.snitch()
return
self.output("Loading legal docs into database")
count, errors = LegalDoc.objects.refresh()
self.output(f"{count} legal docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# only set latest if there are no errors so that it will try the errors again next time
# also so that it will fail again and thus not ping the snitch so that we'll be notified
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.snitch()
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.EXTERNAL_FILES_PATH,
settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name="Community Data")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No community data updates")
return
self.output("Loading community data into database")
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo["type"])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output("Community data successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError(
'Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.LEGAL_DOCS_PATH,
settings.LEGAL_DOCS_REPO,
branch_name=settings.LEGAL_DOCS_BRANCH,
name='Legal Docs')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading legal docs into database')
count, errors = LegalDoc.objects.refresh()
self.output(f'{count} legal docs successfully loaded')
self.output(f'Encountered {errors} errors while loading docs')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
if not errors and settings.LEGAL_DOCS_DMS_URL:
requests.get(settings.LEGAL_DOCS_DMS_URL)
def handle_safe(self, quiet, no_git, clear_db, **options):
force = no_git or clear_db
repo = GitRepo(
ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH,
name="Security Advisories",
)
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout("Clearing all security advisories.")
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout("Updating repository.")
repo.update()
if not (force or repo.has_changes()):
printout("Nothing to update.")
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append(f"ERROR parsing {mf}: {e}")
if not quiet:
sys.stdout.write("E")
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write(".")
sys.stdout.flush()
updates += 1
printout(f"\nUpdated {updates} files.")
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout(f"Deleted {len(deleted_files)} files.")
num_products = delete_orphaned_products()
if num_products:
printout(f"Deleted {num_products} orphaned products.")
if errors:
raise CommandError(f"Encountered {len(errors)} errors:\n\n" +
"\n==========\n".join(errors))
repo.set_db_latest()
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def handle(self, *args, **options):
if options["quiet"]:
self.stdout._out = StringIO()
repo = GitRepo(settings.SITEMAPS_PATH,
settings.SITEMAPS_REPO,
name="Sitemaps")
self.stdout.write("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.stdout.write("No sitemap updates")
return
SitemapURL.objects.refresh()
repo.set_db_latest()
self.stdout.write("Updated sitemaps files")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH, name="Release Notes")
self.output("Updating git repo")
self.output(repo.update())
if not (options["force"] or repo.has_changes()):
self.output("No release note updates")
return
self.output("Loading releases into database")
count = ProductRelease.objects.refresh()
self.output(f"{count} release notes successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name="Content Cards")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No content card updates")
return
self.output("Loading content cards into database")
count = ContentCard.objects.refresh()
self.output(f"{count} content cards successfully loaded")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WEBVISION_DOCS_PATH,
settings.WEBVISION_DOCS_REPO,
branch_name=settings.WEBVISION_DOCS_BRANCH,
name="Webvision Docs")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No webvision docs updates")
return
self.output("Loading webvision docs into database")
count, errors = WebvisionDoc.objects.refresh()
self.output(f"{count} webvision docs successfully loaded")
if errors:
self.output(f"Encountered {errors} errors while loading docs")
else:
# Only `set_db_latest` if there are no errors so that it will try without errors again next time.
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
def handle(self, *args, **options):
self.quiet = options["quiet"]
repo = GitRepo(settings.WWW_CONFIG_PATH,
settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH,
name="WWW Config")
self.output("Updating git repo")
repo.update()
if not (options["force"] or repo.has_changes()):
self.output("No config updates")
return
self.output("Loading configs into database")
count = refresh_db_values()
if count:
self.output(f"{count} configs successfully loaded")
else:
self.output("No configs found. Please try again later.")
repo.set_db_latest()
self.output("Saved latest git repo state to database")
self.output("Done!")
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(
json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
settings.PROD_DETAILS_JSON_REPO_BRANCH,
name="Product Details")
# fake last-modified string since the releng repo doesn't store those files
# and we rely on git commits for updates
self.last_modified = datetime.now().isoformat()
super().__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="If no error occurs, swallow all output."),
parser.add_argument(
"--database",
default="default",
help=("Specifies the database to use, if using a db. "
'Defaults to "default".')),
parser.add_argument(
"-f",
"--force",
action="store_true",
dest="force",
default=False,
help="Load the data even if nothing new from git."),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not (self.update_file_data() or options["force"]):
if not options["quiet"]:
print("Product Details data was already up to date")
return
try:
self.validate_data()
except Exception:
raise CommandError("Product Details data is invalid")
if not options["quiet"]:
print("Product Details data is valid")
if not settings.PROD_DETAILS_STORAGE.endswith("PDDatabaseStorage"):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options["quiet"]:
print("Product Details data update is complete")
def load_changes(self, options, modified_files):
with transaction.atomic(using=options["database"]):
for filename in modified_files:
# skip the l10n directory for now
if filename.startswith("l10n/"):
continue
self.db_storage.update(filename,
self.file_storage.content(filename),
self.last_modified)
if not options["quiet"]:
print("Updated " + filename)
self.db_storage.update("/", "", self.last_modified)
self.db_storage.update("regions/", "", self.last_modified)
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data("firefox_versions.json")[version_key]
if not version:
if version_key == "FIREFOX_ESR_NEXT":
return
builds = len([
locale for locale, build in self.file_storage.data(
"firefox_primary_builds.json").items() if version in build
])
if builds < min_builds:
raise ValueError(f"Too few builds for {version_key}")
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI)
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
