def get_versions_for_product(product, use_cache=True): """Returns list of recent version strings for specified product This looks at the crash reports submitted for this product over VERSIONS_WINDOW_DAYS days and returns the versions of those crash reports. If SuperSearch returns an error, this returns an empty list. NOTE(willkg): This data is noisy if there are crash reports with junk versions. :arg product: either a product name or Product to query for :arg bool use_cache: whether or not to pull results from cache :returns: list of versions sorted in reverse order or ``[]`` """ if isinstance(product, str): product = productlib.get_product_by_name(product) if use_cache: key = "get_versions_for_product:%s" % product.name.lower().replace( " ", "") ret = cache.get(key) if ret is not None: return ret api = supersearch_models.SuperSearchUnredacted() now = timezone.now() # Find versions for specified product in crash reports reported in the last # VERSIONS_WINDOW_DAYS days and use a big _facets_size so that it picks up versions # that have just been released that don't have many crash reports, yet window = settings.VERSIONS_WINDOW_DAYS params = { "product": product.name, "_results_number": 0, "_facets": "version", "_facets_size": 1000, "date": [ ">=" + (now - datetime.timedelta(days=window)).isoformat(), "<" + now.isoformat(), ], } # Since we're caching the results of the search plus additional work done, # we don't want to cache the fetch ret = api.get(**params, dont_cache=True) if "facets" not in ret or "version" not in ret["facets"]: return [] # Get versions from facet, drop junk, and sort the final list versions = set() for item in ret["facets"]["version"]: if item["count"] < settings.VERSIONS_COUNT_THRESHOLD: continue version = item["term"] # Bug #1622932 is about something submitting crash reports with a version of # 1024 which is clearly junk, but it messes everything up; this is hacky, # but let's just drop those explicitly and push off thinking about a better # way of doing all of this until later if version.startswith("1024"): continue try: # This generates the sort key but also parses the version to make sure it's # a valid looking version versions.add((generate_semver(version), version)) # Add X.Yb to betas set if "b" in version: beta_version = version[:version.find("b") + 1] versions.add((generate_semver(beta_version), beta_version)) except VersionParseError: pass # Sort by sortkey and then drop the sortkey versions = sorted(versions, key=lambda v: v[0], reverse=True) versions = [v[1] for v in versions] if use_cache: # Cache value for an hour plus a fudge factor in seconds cache.set(key, versions, timeout=(60 * 60) + random.randint(60, 120)) return versions
def get_versions_for_product(product='Firefox', use_cache=True): """Returns list of recent version strings for specified product This looks at the crash reports submitted for this product over VERSIONS_WINDOW_DAYS days and returns the versinos of those crash reports. If SuperSearch returns an error, this returns an empty list. NOTE(willkg): This data can be noisy in cases where crash reports return junk versions. We might want to add a "minimum to matter" number. :arg str product: the product to query for :arg bool use_cache: whether or not to pull results from cache :returns: list of versions sorted in reverse order or ``[]`` """ if use_cache: key = 'get_versions_for_product:%s' % product.lower().replace(' ', '') ret = cache.get(key) if ret is not None: return ret api = supersearch_models.SuperSearchUnredacted() now = timezone.now() # Find versions for specified product in crash reports reported in the last # 6 months params = { 'product': product, '_results_number': 0, '_facets': 'version', '_facets_size': 100, 'date': [ '>=' + (now - datetime.timedelta(days=VERSIONS_WINDOW_DAYS)).isoformat(), '<' + now.isoformat() ] } ret = api.get(**params) if 'facets' not in ret or 'version' not in ret['facets']: return [] # Get versions from facet, drop junk, and sort the final list versions = [] for item in ret['facets']['version']: version = item['term'] try: # This generates the sort key but also parses the version to # make sure it's a valid looking version versions.append((generate_version_key(version), version)) except VersionParseError: pass versions.sort(key=lambda v: v[0], reverse=True) versions = [v[1] for v in versions] if use_cache: # Cache value for an hour plus a fudge factor in seconds cache.set(key, versions, (60 * 60) + random.randint(60, 120)) return versions
def get_versions_for_product(product='Firefox', use_cache=True): """Returns list of recent version strings for specified product This looks at the crash reports submitted for this product over VERSIONS_WINDOW_DAYS days and returns the versinos of those crash reports. If SuperSearch returns an error, this returns an empty list. NOTE(willkg): This data is noisy if there are crash reports with junk versions. :arg str product: the product to query for :arg bool use_cache: whether or not to pull results from cache :returns: list of versions sorted in reverse order or ``[]`` """ if use_cache: key = 'get_versions_for_product:%s' % product.lower().replace(' ', '') ret = cache.get(key) if ret is not None: return ret api = supersearch_models.SuperSearchUnredacted() now = timezone.now() # Find versions for specified product in crash reports reported in the last # 6 months and use a big _facets_size so that it picks up versions that # have just been released that don't have many crash reports, yet params = { 'product': product, '_results_number': 0, '_facets': 'version', '_facets_size': 1000, 'date': [ '>=' + (now - datetime.timedelta(days=VERSIONS_WINDOW_DAYS)).isoformat(), '<' + now.isoformat() ] } # Since we're caching the results of the search plus additional work done, # we don't need to cache the fetch ret = api.get(**params, dont_cache=True) if 'facets' not in ret or 'version' not in ret['facets']: return [] # Get versions from facet, drop junk, and sort the final list betas = set() versions = [] for item in ret['facets']['version']: version = item['term'] try: # This generates the sort key but also parses the version to # make sure it's a valid looking version versions.append((generate_version_key(version), version)) # Add X.Yb to betas set if 'b' in version: beta_version = version[:version.find('b') + 1] betas.add(beta_version) except VersionParseError: pass # Add the (sortkey, X.Yb) to the list versions.extend([(generate_version_key(beta), beta) for beta in betas]) # Sort by sortkey and then drop the sortkey versions.sort(key=lambda v: v[0], reverse=True) versions = [v[1] for v in versions] if use_cache: # Cache value for an hour plus a fudge factor in seconds cache.set(key, versions, timeout=(60 * 60) + random.randint(60, 120)) return versions
def get_versions_for_product(product): """Returns recent versions for specified product This looks at the crash reports submitted for this product over VERSIONS_WINDOW_DAYS days and returns the versinos of those crash reports. If SuperSearch returns an error, this returns an empty list. NOTE(willkg): This data can be noisy in cases where crash reports return junk versions. We might want to add a "minimum to matter" number. :arg product: the product to query for :returns: list of versions sorted in reverse order or ``[]`` """ key = 'get_versions_for_product:%s' % product.lower().replace(' ', '') ret = cache.get(key) if ret is not None: return ret api = supersearch_models.SuperSearchUnredacted() now = timezone.now() # Find versions for specified product in crash reports reported in the last # 6 months params = { 'product': product, '_results_number': 0, '_facets': 'version', '_facets_size': 100, 'date': [ '>=' + (now - datetime.timedelta(days=VERSIONS_WINDOW_DAYS)).isoformat(), '<' + now.isoformat() ] } ret = api.get(**params) if 'facets' not in ret or 'version' not in ret['facets']: return [] versions = sorted_versions([item['term'] for item in ret['facets']['version']]) # Set of X.Yb to add betas = set() # Map of major version (int) -> list of versions (str) so we can # get the most recent version of the last three majors which # we'll assume are "featured versions" major_to_versions = {} for version in versions: try: major = int(version.split('.', 1)[0]) major_to_versions.setdefault(major, []).append(version) if 'b' in version: # Add X.Yb to the betas set betas.add(version[:version.find('b') + 1]) except ValueError: # If the first thing in the major version isn't an int, then skip # it continue # The featured versions is the most recent 3 of the list of recent versions # for each major version. Since versions were sorted when we went through # them, the most recent one is in index 0. featured_versions = sorted_versions([values[0] for values in major_to_versions.values()]) featured_versions = featured_versions[:3] # Add the beta versions and then resort the versions versions.extend(betas) versions = sorted_versions(versions) # Generate the version data the context needs ret = [ { 'product': product, 'version': version, 'is_featured': version in featured_versions, 'has_builds': False } for version in versions ] # Cache value for an hour plus a fudge factor in seconds cache.set(key, ret, (60 * 60) + random.randint(60, 120)) return ret