def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(
json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI)
super(Command, self).__init__(stdout, stderr, no_color)
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
settings.PROD_DETAILS_JSON_REPO_BRANCH,
name='Product Details')
# fake last-modified string since the releng repo doesn't store those files
# and we rely on git commits for updates
self.last_modified = datetime.now().isoformat()
super(Command, self).__init__(stdout, stderr, no_color)
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
name='Product Details')
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI)
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument('--database', default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not self.update_file_data():
if not options['quiet']:
print('Product Details data was already up to date')
return
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options, modified_files):
with transaction.atomic(using=options['database']):
for filename in modified_files:
self.db_storage.update(filename,
self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '', self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '', self.file_storage.last_modified('regions/'))
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([locale for locale, build in
self.file_storage.data('firefox_primary_builds.json').items()
if version in build])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(
json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
self.repo = GitRepo(settings.PROD_DETAILS_JSON_REPO_PATH,
settings.PROD_DETAILS_JSON_REPO_URI,
settings.PROD_DETAILS_JSON_REPO_BRANCH,
name="Product Details")
# fake last-modified string since the releng repo doesn't store those files
# and we rely on git commits for updates
self.last_modified = datetime.now().isoformat()
super().__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="If no error occurs, swallow all output."),
parser.add_argument(
"--database",
default="default",
help=("Specifies the database to use, if using a db. "
'Defaults to "default".')),
parser.add_argument(
"-f",
"--force",
action="store_true",
dest="force",
default=False,
help="Load the data even if nothing new from git."),
def handle(self, *args, **options):
# don't really care about deleted files. almost never happens in p-d.
if not (self.update_file_data() or options["force"]):
if not options["quiet"]:
print("Product Details data was already up to date")
return
try:
self.validate_data()
except Exception:
raise CommandError("Product Details data is invalid")
if not options["quiet"]:
print("Product Details data is valid")
if not settings.PROD_DETAILS_STORAGE.endswith("PDDatabaseStorage"):
# no need to continue if not using DB backend
return
self.load_changes(options, self.file_storage.all_json_files())
self.repo.set_db_latest()
if not options["quiet"]:
print("Product Details data update is complete")
def load_changes(self, options, modified_files):
with transaction.atomic(using=options["database"]):
for filename in modified_files:
# skip the l10n directory for now
if filename.startswith("l10n/"):
continue
self.db_storage.update(filename,
self.file_storage.content(filename),
self.last_modified)
if not options["quiet"]:
print("Updated " + filename)
self.db_storage.update("/", "", self.last_modified)
self.db_storage.update("regions/", "", self.last_modified)
def update_file_data(self):
self.repo.update()
return self.repo.has_changes()
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data("firefox_versions.json")[version_key]
if not version:
if version_key == "FIREFOX_ESR_NEXT":
return
builds = len([
locale for locale, build in self.file_storage.data(
"firefox_primary_builds.json").items() if version in build
])
if builds < min_builds:
raise ValueError(f"Too few builds for {version_key}")
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)
class Command(BaseCommand):
def __init__(self, stdout=None, stderr=None, no_color=False):
self.file_storage = PDFileStorage(
json_dir=settings.PROD_DETAILS_TEST_DIR)
self.db_storage = PDDatabaseStorage()
super(Command, self).__init__(stdout, stderr, no_color)
def add_arguments(self, parser):
parser.add_argument('-f',
'--force',
action='store_true',
dest='force',
default=False,
help=('Download product details even if they have '
'not been updated since the last fetch.'))
parser.add_argument('-q',
'--quiet',
action='store_true',
dest='quiet',
default=False,
help='If no error occurs, swallow all output.'),
parser.add_argument(
'--database',
default='default',
help=('Specifies the database to use, if using a db. '
'Defaults to "default".')),
def handle(self, *args, **options):
if not settings.PROD_DETAILS_STORAGE.endswith('PDDatabaseStorage'):
raise CommandError(
'Must be setup for database product-details storage to use this'
)
self.update_file_data(options)
try:
self.validate_data()
except Exception:
raise CommandError('Product Details data is invalid')
if not options['quiet']:
print('Product Details data is valid')
self.load_changes(options)
if not options['quiet']:
print('Product Details data update is complete')
def load_changes(self, options):
with transaction.atomic(using=options['database']):
for filename in self.file_storage.all_json_files():
fs_file_mtime = self.file_storage.last_modified_datetime(
filename)
db_file_mtime = self.db_storage.last_modified_datetime(
filename)
if options[
'force'] or not db_file_mtime or fs_file_mtime > db_file_mtime:
self.db_storage.update(
filename, self.file_storage.content(filename),
self.file_storage.last_modified(filename))
if not options['quiet']:
print('Updated ' + filename)
self.db_storage.update('/', '',
self.file_storage.last_modified('/'))
self.db_storage.update('regions/', '',
self.file_storage.last_modified('regions/'))
def update_file_data(self, options):
# json dir set in settings
command = PDCommand()
command._storage = self.file_storage
command.is_db_storage = False
command.handle(**options)
def count_builds(self, version_key, min_builds=20):
version = self.file_storage.data('firefox_versions.json')[version_key]
if not version:
if version_key == 'FIREFOX_ESR_NEXT':
return
builds = len([
locale for locale, build in self.file_storage.data(
'firefox_primary_builds.json').items() if version in build
])
if builds < min_builds:
raise ValueError('Too few builds for {}'.format(version_key))
def validate_data(self):
self.file_storage.clear_cache()
for key in FIREFOX_VERSION_KEYS:
self.count_builds(key)