def get_data_version(): """Add the git ref from the repo to the cache keys. This will ensure that the cache is invalidated when the repo is updated. """ repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH) git_ref = repo.get_db_latest() if git_ref is None: git_ref = 'default' return git_ref
def __init__(self, stdout=None, stderr=None, no_color=False): self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR) self.db_storage = PDDatabaseStorage() self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH, settings.PROD_DETAILS_JSON_REPO_URI, name='Product Details') super(Command, self).__init__(stdout, stderr, no_color)
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.EXTERNAL_FILES_PATH, settings.EXTERNAL_FILES_REPO, branch_name=settings.EXTERNAL_FILES_BRANCH, name='Community Data') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No community data updates') return self.output('Loading community data into database') for fid, finfo in settings.EXTERNAL_FILES.items(): klass = import_string(finfo['type']) try: klass(fid).update() except ValueError as e: raise CommandError(str(e)) self.output('Community data successfully loaded') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle_noargs(self, **options): quiet = options['quiet'] no_git = options['no_git'] clear_db = options['clear_db'] force = no_git or clear_db repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH, name='Security Advisories') def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() MitreCVE.objects.all().delete() if not no_git: printout('Updating repository.') repo.update() if not (force or repo.has_changes()): printout('Nothing to update.') return errors = [] updates = 0 all_files = get_all_file_names() for mf in all_files: try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if not clear_db: deleted_files = get_files_to_delete_from_db(all_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if errors: raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors)) repo.set_db_latest()
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.RELEASE_NOTES_PATH, settings.RELEASE_NOTES_REPO, branch_name=settings.RELEASE_NOTES_BRANCH) self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No release note updates') return self.output('Loading releases into database') count = ProductRelease.objects.refresh() self.output('%s release notes successfully loaded' % count) repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.CONTENT_CARDS_PATH, settings.CONTENT_CARDS_REPO, branch_name=settings.CONTENT_CARDS_BRANCH, name='Content Cards') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No content card updates') return self.output('Loading content cards into database') count = ContentCard.objects.refresh() self.output('%s content cards successfully loaded' % count) repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): self.quiet = options['quiet'] repo = GitRepo(settings.WWW_CONFIG_PATH, settings.WWW_CONFIG_REPO, branch_name=settings.WWW_CONFIG_BRANCH, name='WWW Config') self.output('Updating git repo') repo.update() if not (options['force'] or repo.has_changes()): self.output('No config updates') return self.output('Loading configs into database') count = refresh_db_values() if count: self.output('%s configs successfully loaded' % count) else: self.output('No configs found. Please try again later.') repo.set_db_latest() self.output('Saved latest git repo state to database') self.output('Done!')
def handle(self, *args, **options): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) repo.update()
class Command(BaseCommand): def __init__(self, stdout=None, stderr=None, no_color=False): self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR) self.db_storage = PDDatabaseStorage() self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH, settings.PROD_DETAILS_JSON_REPO_URI, name='Product Details') super(Command, self).__init__(stdout, stderr, no_color) def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.'), parser.add_argument('--database', default='default', help=('Specifies the database to use, if using a db. ' 'Defaults to "default".')), def handle(self, *args, **options): # don't really care about deleted files. almost never happens in p-d. if not self.update_file_data(): if not options['quiet']: print('Product Details data was already up to date') return try: self.validate_data() except Exception: raise CommandError('Product Details data is invalid') if not options['quiet']: print('Product Details data is valid') if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'): # no need to continue if not using DB backend return self.load_changes(options, self.file_storage.all_json_files()) self.repo.set_db_latest() if not options['quiet']: print('Product Details data update is complete') def load_changes(self, options, modified_files): with transaction.atomic(using=options['database']): for filename in modified_files: self.db_storage.update(filename, self.file_storage.content(filename), self.file_storage.last_modified(filename)) if not options['quiet']: print('Updated ' + filename) self.db_storage.update('/', '', self.file_storage.last_modified('/')) self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/')) def update_file_data(self): self.repo.update() return self.repo.has_changes() def count_builds(self, version_key, min_builds=20): version = self.file_storage.data('firefox_versions.json')[version_key] if not version: if version_key == 'FIREFOX_ESR_NEXT': return builds = len([locale for locale, build in self.file_storage.data('firefox_primary_builds.json').items() if version in build]) if builds < min_builds: raise ValueError('Too few builds for {}'.format(version_key)) def validate_data(self): self.file_storage.clear_cache() for key in FIREFOX_VERSION_KEYS: self.count_builds(key)
def handle_noargs(self, **options): quiet = options['quiet'] force = options['force'] no_git = options['no_git'] clear_db = options['clear_db'] if no_git or clear_db: force = True cloned = False repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH) modified_files = deleted_files = [] def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() if not no_git: printout('Updating repository.') modified_files, deleted_files = repo.update() if modified_files is None: cloned = True else: modified_files = filter_advisory_filenames(modified_files) deleted_files = filter_advisory_filenames(deleted_files) if force or cloned: printout('Reading all files.') modified_files = get_all_mfsa_files() if clear_db: deleted_files = [] else: deleted_files = get_files_to_delete_from_db(modified_files) errors = [] updates = 0 if modified_files: for mf in modified_files: mf = os.path.join(ADVISORIES_PATH, mf) try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if deleted_files: deleted_files = filter_updated_from_deleted(modified_files, deleted_files) delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if not modified_files and not deleted_files: printout('Nothing to update.') if errors: raise CommandError('Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors))
def update_lang_files(self): repo = GitRepo(settings.LOCALES_PATH, settings.LOCALES_REPO) repo.update() self.stdout.write('Updated .lang files')
def handle_noargs(self, **options): quiet = options['quiet'] force = options['force'] no_git = options['no_git'] clear_db = options['clear_db'] if no_git or clear_db: force = True cloned = False repo = GitRepo(ADVISORIES_PATH, ADVISORIES_REPO, branch_name=ADVISORIES_BRANCH) modified_files = deleted_files = [] def printout(msg, ending=None): if not quiet: self.stdout.write(msg, ending=ending) if clear_db: printout('Clearing all security advisories.') SecurityAdvisory.objects.all().delete() Product.objects.all().delete() if not no_git: printout('Updating repository.') modified_files, deleted_files = repo.update() if modified_files is None: cloned = True else: modified_files = filter_advisory_filenames(modified_files) deleted_files = filter_advisory_filenames(deleted_files) if force or cloned: printout('Reading all files.') modified_files = get_all_mfsa_files() if clear_db: deleted_files = [] else: deleted_files = get_files_to_delete_from_db(modified_files) errors = [] updates = 0 if modified_files: for mf in modified_files: mf = os.path.join(ADVISORIES_PATH, mf) try: update_db_from_file(mf) except Exception as e: errors.append('ERROR parsing %s: %s' % (mf, e)) if not quiet: sys.stdout.write('E') sys.stdout.flush() continue if not quiet: sys.stdout.write('.') sys.stdout.flush() updates += 1 printout('\nUpdated {0} files.'.format(updates)) if deleted_files: delete_files(deleted_files) printout('Deleted {0} files.'.format(len(deleted_files))) num_products = delete_orphaned_products() if num_products: printout('Deleted {0} orphaned products.'.format(num_products)) if not modified_files and not deleted_files: printout('Nothing to update.') if errors: sys.stderr.write( 'Encountered {0} errors:\n\n'.format(len(errors)) + '\n==========\n'.join(errors))
class Command(BaseCommand): def __init__(self, stdout=None, stderr=None, no_color=False): self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR) self.db_storage = PDDatabaseStorage() self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH, settings.PROD_DETAILS_JSON_REPO_URI, settings.PROD_DETAILS_JSON_REPO_BRANCH, name='Product Details') # fake last-modified string since the releng repo doesn't store those files # and we rely on git commits for updates self.last_modified = datetime.now().isoformat() super(Command, self).__init__(stdout, stderr, no_color) def add_arguments(self, parser): parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='If no error occurs, swallow all output.'), parser.add_argument('--database', default='default', help=('Specifies the database to use, if using a db. ' 'Defaults to "default".')), parser.add_argument('-f', '--force', action='store_true', dest='force', default=False, help='Load the data even if nothing new from git.'), def handle(self, *args, **options): # don't really care about deleted files. almost never happens in p-d. if not (self.update_file_data() or options['force']): if not options['quiet']: print('Product Details data was already up to date') return try: self.validate_data() except Exception: raise CommandError('Product Details data is invalid') if not options['quiet']: print('Product Details data is valid') if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'): # no need to continue if not using DB backend return self.load_changes(options, self.file_storage.all_json_files()) self.repo.set_db_latest() if not options['quiet']: print('Product Details data update is complete') def load_changes(self, options, modified_files): with transaction.atomic(using=options['database']): for filename in modified_files: # skip the l10n directory for now if filename.startswith('l10n/'): continue self.db_storage.update(filename, self.file_storage.content(filename), self.last_modified) if not options['quiet']: print('Updated ' + filename) self.db_storage.update('/', '', self.last_modified) self.db_storage.update('regions/', '', self.last_modified) def update_file_data(self): self.repo.update() return self.repo.has_changes() def count_builds(self, version_key, min_builds=20): version = self.file_storage.data('firefox_versions.json')[version_key] if not version: if version_key == 'FIREFOX_ESR_NEXT': return builds = len([locale for locale, build in self.file_storage.data('firefox_primary_builds.json').items() if version in build]) if builds < min_builds: raise ValueError('Too few builds for {}'.format(version_key)) def validate_data(self): self.file_storage.clear_cache() for key in FIREFOX_VERSION_KEYS: self.count_builds(key)
def handle(self, *args, **options): if options['quiet']: self.stdout._out = StringIO() self.l10n_repo = GitRepo(settings.FLUENT_L10N_TEAM_REPO_PATH, settings.FLUENT_L10N_TEAM_REPO_URL) self.meao_repo = GitRepo(settings.FLUENT_REPO_PATH, settings.FLUENT_REPO_URL)