def get_data_version():
"""Add the git ref from the repo to the cache keys.
This will ensure that the cache is invalidated when the repo is updated.
"""
repo = GitRepo(settings.RELEASE_NOTES_PATH,
settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
git_ref = repo.get_db_latest()
if git_ref is None:
git_ref = 'default'
return git_ref
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO,
branch_name=settings.EXTERNAL_FILES_BRANCH,
name='Community Data')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No community data updates')
return
self.output('Loading community data into database')
for fid, finfo in settings.EXTERNAL_FILES.items():
klass = import_string(finfo['type'])
try:
klass(fid).update()
except ValueError as e:
raise CommandError(str(e))
self.output('Community data successfully loaded')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle_noargs(self, **options):
quiet = options['quiet']
no_git = options['no_git']
clear_db = options['clear_db']
force = no_git or clear_db
repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH,
name='Security Advisories')
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
MitreCVE.objects.all().delete()
if not no_git:
printout('Updating repository.')
repo.update()
if not (force or repo.has_changes()):
printout('Nothing to update.')
return
errors = []
updates = 0
all_files = get_all_file_names()
for mf in all_files:
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if not clear_db:
deleted_files = get_files_to_delete_from_db(all_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if errors:
raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
repo.set_db_latest()
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO,
branch_name=settings.RELEASE_NOTES_BRANCH)
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No release note updates')
return
self.output('Loading releases into database')
count = ProductRelease.objects.refresh()
self.output('%s release notes successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO,
branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No content card updates')
return
self.output('Loading content cards into database')
count = ContentCard.objects.refresh()
self.output('%s content cards successfully loaded' % count)
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
self.quiet = options['quiet']
repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO,
branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config')
self.output('Updating git repo')
repo.update()
if not (options['force'] or repo.has_changes()):
self.output('No config updates')
return
self.output('Loading configs into database')
count = refresh_db_values()
if count:
self.output('%s configs successfully loaded' % count)
else:
self.output('No configs found. Please try again later.')
repo.set_db_latest()
self.output('Saved latest git repo state to database')
self.output('Done!')
def handle(self, *args, **options):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
def handle_noargs(self, **options):
quiet = options['quiet']
force = options['force']
no_git = options['no_git']
clear_db = options['clear_db']
if no_git or clear_db:
force = True
cloned = False
repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH)
modified_files = deleted_files = []
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
if not no_git:
printout('Updating repository.')
modified_files, deleted_files = repo.update()
if modified_files is None:
cloned = True
else:
modified_files = filter_advisory_filenames(modified_files)
deleted_files = filter_advisory_filenames(deleted_files)
if force or cloned:
printout('Reading all files.')
modified_files = get_all_mfsa_files()
if clear_db:
deleted_files = []
else:
deleted_files = get_files_to_delete_from_db(modified_files)
errors = []
updates = 0
if modified_files:
for mf in modified_files:
mf = os.path.join(ADVISORIES_PATH, mf)
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if deleted_files:
deleted_files = filter_updated_from_deleted(modified_files, deleted_files)
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if not modified_files and not deleted_files:
printout('Nothing to update.')
if errors:
raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
def update_lang_files(self):
repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO)
repo.update()
self.stdout.write('Updated .lang files')
def handle_noargs(self, **options):
quiet = options['quiet']
force = options['force']
no_git = options['no_git']
clear_db = options['clear_db']
if no_git or clear_db:
force = True
cloned = False
repo = GitRepo(ADVISORIES_PATH,
ADVISORIES_REPO,
branch_name=ADVISORIES_BRANCH)
modified_files = deleted_files = []
def printout(msg, ending=None):
if not quiet:
self.stdout.write(msg, ending=ending)
if clear_db:
printout('Clearing all security advisories.')
SecurityAdvisory.objects.all().delete()
Product.objects.all().delete()
if not no_git:
printout('Updating repository.')
modified_files, deleted_files = repo.update()
if modified_files is None:
cloned = True
else:
modified_files = filter_advisory_filenames(modified_files)
deleted_files = filter_advisory_filenames(deleted_files)
if force or cloned:
printout('Reading all files.')
modified_files = get_all_mfsa_files()
if clear_db:
deleted_files = []
else:
deleted_files = get_files_to_delete_from_db(modified_files)
errors = []
updates = 0
if modified_files:
for mf in modified_files:
mf = os.path.join(ADVISORIES_PATH, mf)
try:
update_db_from_file(mf)
except Exception as e:
errors.append('ERROR parsing %s: %s' % (mf, e))
if not quiet:
sys.stdout.write('E')
sys.stdout.flush()
continue
if not quiet:
sys.stdout.write('.')
sys.stdout.flush()
updates += 1
printout('\nUpdated {0} files.'.format(updates))
if deleted_files:
delete_files(deleted_files)
printout('Deleted {0} files.'.format(len(deleted_files)))
num_products = delete_orphaned_products()
if num_products:
printout('Deleted {0} orphaned products.'.format(num_products))
if not modified_files and not deleted_files:
printout('Nothing to update.')
if errors:
sys.stderr.write(
'Encountered {0} errors:\n\n'.format(len(errors)) +
'\n==========\n'.join(errors))
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
settings.PROD_DETAILS_JSON_REPO_BRANCH,
name='Product Details')
# fake last-modified string since the releng repo doesn't store those files
# and we rely on git commits for updates
self.last_modified = datetime.now().isoformat()
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
parser.add_argument('-f', '--force', action='store_true', dest='force', default=False,
help='Load the data even if nothing new from git.'),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not (self.update_file_data() or options['force']):
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
# skip the l10n directory for now
if filename.startswith('l10n/'):
continue
self.db_storage.update(filename,
self.file_storage.content(filename),
self.last_modified)
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.last_modified)
self.db_storage.update('regions/', '', self.last_modified)
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
def handle(self, *args, **options):
if options['quiet']:
self.stdout._out = StringIO()
self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO_URL)
self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)
