Ejemplo n.º 1
0
def _progress():
    """Returns unreviewed apps progress.

    Return the number of apps still unreviewed for a given period of time and
    the percentage.
    """

    queues_helper = ReviewersQueuesHelper()

    base_filters = {
        'pending': (queues_helper.get_pending_queue(),
                    'nomination'),
        'homescreen': (queues_helper.get_homescreen_queue(),
                       'nomination'),
        'rereview': (queues_helper.get_rereview_queue(),
                     'created'),
        'escalated': (queues_helper.get_escalated_queue(),
                      'created'),
        'updates': (queues_helper.get_updates_queue(),
                    'nomination')
    }

    operators_and_values = {
        'new': ('gt', days_ago(5)),
        'med': ('range', (days_ago(10), days_ago(5))),
        'old': ('lt', days_ago(10)),
        'week': ('gte', days_ago(7))
    }

    types = base_filters.keys()
    progress = {}

    for t in types:
        tmp = {}
        base_query, field = base_filters[t]
        for k in operators_and_values.keys():
            operator, value = operators_and_values[k]
            filter_ = {}
            filter_['%s__%s' % (field, operator)] = value
            tmp[k] = base_query.filter(**filter_).count()
        progress[t] = tmp

    def pct(p, t):
        # Return the percent of (p)rogress out of (t)otal.
        return (p / float(t)) * 100 if p > 0 else 0

    percentage = {}
    for t in types:
        total = progress[t]['new'] + progress[t]['med'] + progress[t]['old']
        percentage[t] = {}
        for duration in ('new', 'med', 'old'):
            percentage[t][duration] = pct(progress[t][duration], total)

    return (progress, percentage)
Ejemplo n.º 2
0
def _progress():
    """Returns unreviewed apps progress.

    Return the number of apps still unreviewed for a given period of time and
    the percentage.
    """

    queues_helper = ReviewersQueuesHelper()

    base_filters = {
        'pending': (queues_helper.get_pending_queue(),
                    'nomination'),
        'rereview': (queues_helper.get_rereview_queue(),
                     'created'),
        'escalated': (queues_helper.get_escalated_queue(),
                      'created'),
        'updates': (queues_helper.get_updates_queue(),
                    'nomination')
    }

    operators_and_values = {
        'new': ('gt', days_ago(5)),
        'med': ('range', (days_ago(10), days_ago(5))),
        'old': ('lt', days_ago(10)),
        'week': ('gte', days_ago(7))
    }

    types = base_filters.keys()
    progress = {}

    for t in types:
        tmp = {}
        base_query, field = base_filters[t]
        for k in operators_and_values.keys():
            operator, value = operators_and_values[k]
            filter_ = {}
            filter_['%s__%s' % (field, operator)] = value
            tmp[k] = base_query.filter(**filter_).count()
        progress[t] = tmp

    def pct(p, t):
        # Return the percent of (p)rogress out of (t)otal.
        return (p / float(t)) * 100 if p > 0 else 0

    percentage = {}
    for t in types:
        total = progress[t]['new'] + progress[t]['med'] + progress[t]['old']
        percentage[t] = {}
        for duration in ('new', 'med', 'old'):
            percentage[t][duration] = pct(progress[t][duration], total)

    return (progress, percentage)
Ejemplo n.º 3
0
def mkt_gc(**kw):
    """Site-wide garbage collections."""
    log.info('Collecting data to delete')
    logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude(
        action__in=mkt.LOG_KEEP).values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        chunk.sort()
        log.info('Deleting log entries: %s' % str(chunk))
        delete_logs.delay(chunk)

    # Clear oauth nonce rows. These expire after 10 minutes but we're just
    # clearing those that are more than 1 day old.
    Nonce.objects.filter(created__lt=days_ago(1)).delete()

    # Delete the dump apps over 30 days.
    _remove_stale_files(os.path.join(settings.DUMPED_APPS_PATH, 'tarballs'),
                        settings.DUMPED_APPS_DAYS_DELETE,
                        'Deleting old tarball: {0}',
                        storage=public_storage)

    # Delete the dumped user installs over 30 days. Those are using private
    # storage.
    _remove_stale_files(os.path.join(settings.DUMPED_USERS_PATH, 'tarballs'),
                        settings.DUMPED_USERS_DAYS_DELETE,
                        'Deleting old tarball: {0}',
                        storage=private_storage)

    # Delete old files in select directories under TMP_PATH.
    _remove_stale_files(os.path.join(settings.TMP_PATH, 'preview'),
                        settings.TMP_PATH_DAYS_DELETE,
                        'Deleting TMP_PATH file: {0}',
                        storage=private_storage)
    _remove_stale_files(os.path.join(settings.TMP_PATH, 'icon'),
                        settings.TMP_PATH_DAYS_DELETE,
                        'Deleting TMP_PATH file: {0}',
                        storage=private_storage)

    # Delete stale FileUploads.
    for fu in FileUpload.objects.filter(created__lte=days_ago(90)):
        log.debug(u'[FileUpload:{uuid}] Removing file: {path}'.format(
            uuid=fu.uuid, path=fu.path))
        if fu.path:
            try:
                private_storage.delete(fu.path)
            except OSError:
                pass
        fu.delete()
Ejemplo n.º 4
0
def mkt_gc(**kw):
    """Site-wide garbage collections."""
    log.info('Collecting data to delete')
    logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
            .exclude(action__in=mkt.LOG_KEEP).values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        chunk.sort()
        log.info('Deleting log entries: %s' % str(chunk))
        delete_logs.delay(chunk)

    # Clear oauth nonce rows. These expire after 10 minutes but we're just
    # clearing those that are more than 1 day old.
    Nonce.objects.filter(created__lt=days_ago(1)).delete()

    # Delete the dump apps over 30 days.
    _remove_stale_files(os.path.join(settings.DUMPED_APPS_PATH, 'tarballs'),
                        settings.DUMPED_APPS_DAYS_DELETE,
                        'Deleting old tarball: {0}',
                        storage=public_storage)

    # Delete the dumped user installs over 30 days. Those are using private
    # storage.
    _remove_stale_files(os.path.join(settings.DUMPED_USERS_PATH, 'tarballs'),
                        settings.DUMPED_USERS_DAYS_DELETE,
                        'Deleting old tarball: {0}',
                        storage=private_storage)

    # Delete old files in select directories under TMP_PATH.
    _remove_stale_files(os.path.join(settings.TMP_PATH, 'preview'),
                        settings.TMP_PATH_DAYS_DELETE,
                        'Deleting TMP_PATH file: {0}',
                        storage=private_storage)
    _remove_stale_files(os.path.join(settings.TMP_PATH, 'icon'),
                        settings.TMP_PATH_DAYS_DELETE,
                        'Deleting TMP_PATH file: {0}',
                        storage=private_storage)

    # Delete stale FileUploads.
    for fu in FileUpload.objects.filter(created__lte=days_ago(90)):
        log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
                  .format(uuid=fu.uuid, path=fu.path))
        if fu.path:
            try:
                private_storage.delete(fu.path)
            except OSError:
                pass
        fu.delete()
Ejemplo n.º 5
0
def _get_trending(app_id):
    """
    Calculate trending for app for all regions and per region.

    a = installs from 8 days ago to 1 day ago
    b = installs from 29 days ago to 9 days ago, averaged per week
    trending = (a - b) / b if a > 100 and b > 1 else 0

    Returns value in the format of::

        {'all': <global trending score>,
         <region_slug>: <regional trending score>,
         ...}

    """
    # How many app installs are required in the prior week to be considered
    # "trending". Adjust this as total Marketplace app installs increases.
    #
    # Note: AMO uses 1000.0 for add-ons.
    PRIOR_WEEK_INSTALL_THRESHOLD = 100.0

    client = get_monolith_client()

    week1 = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(8).date().isoformat(),
                    'lte': days_ago(1).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }
    week3 = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(29).date().isoformat(),
                    'lte': days_ago(9).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }

    query = {
        'query': {
            'filtered': {
                'query': {'match_all': {}},
                'filter': {'term': {'app-id': app_id}}
            }
        },
        'aggregations': {
            'week1': week1,
            'week3': week3,
            'region': {
                'terms': {
                    'field': 'region',
                    # Add size so we get all regions, not just the top 10.
                    'size': len(mkt.regions.ALL_REGIONS)
                },
                'aggregations': {
                    'week1': week1,
                    'week3': week3
                }
            }
        },
        'size': 0
    }

    try:
        res = client.raw(query)
    except ValueError as e:
        task_log.error('Error response from Monolith: {0}'.format(e))
        return {}

    if 'aggregations' not in res:
        task_log.error('No installs for app {}'.format(app_id))
        return {}

    def _score(week1, week3):
        # If last week app installs are < 100, this app isn't trending.
        if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
            return 0.0

        score = 0.0
        if week3 > 1.0:
            score = (week1 - week3) / week3
        if score < 0.0:
            score = 0.0
        return score

    # Global trending score.
    week1 = res['aggregations']['week1']['total_installs']['value']
    week3 = res['aggregations']['week3']['total_installs']['value'] / 3.0

    if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
        # If global installs over the last week aren't over 100, we
        # short-circuit and return a zero-like value as this is not a trending
        # app by definition. Since global installs aren't above 100, per-region
        # installs won't be either.
        return {}

    results = {
        'all': _score(week1, week3)
    }

    if 'region' in res['aggregations']:
        for regional_res in res['aggregations']['region']['buckets']:
            region_slug = regional_res['key']
            week1 = regional_res['week1']['total_installs']['value']
            week3 = regional_res['week3']['total_installs']['value'] / 3.0
            results[region_slug] = _score(week1, week3)

    return results
Ejemplo n.º 6
0
def _get_installs(app_id):
    """
    Calculate popularity of app for all regions and per region.

    Returns value in the format of::

        {'all': <global installs>,
         <region_slug>: <regional installs>,
         ...}

    """
    # How many days back do we include when calculating popularity.
    POPULARITY_PERIOD = 90

    client = get_monolith_client()

    popular = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(POPULARITY_PERIOD).date().isoformat(),
                    'lte': days_ago(1).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }

    query = {
        'query': {
            'filtered': {
                'query': {'match_all': {}},
                'filter': {'term': {'app-id': app_id}}
            }
        },
        'aggregations': {
            'popular': popular,
            'region': {
                'terms': {
                    'field': 'region',
                    # Add size so we get all regions, not just the top 10.
                    'size': len(mkt.regions.ALL_REGIONS)
                },
                'aggregations': {
                    'popular': popular
                }
            }
        },
        'size': 0
    }

    try:
        res = client.raw(query)
    except ValueError as e:
        task_log.error('Error response from Monolith: {0}'.format(e))
        return {}

    if 'aggregations' not in res:
        task_log.error('No installs for app {}'.format(app_id))
        return {}

    results = {
        'all': res['aggregations']['popular']['total_installs']['value']
    }

    if 'region' in res['aggregations']:
        for regional_res in res['aggregations']['region']['buckets']:
            region_slug = regional_res['key']
            popular = regional_res['popular']['total_installs']['value']
            results[region_slug] = popular

    return results
Ejemplo n.º 7
0
def _get_trending(app_id):
    """
    Calculate trending for app for all regions and per region.

    a = installs from 8 days ago to 1 day ago
    b = installs from 29 days ago to 9 days ago, averaged per week
    trending = (a - b) / b if a > 100 and b > 1 else 0

    Returns value in the format of::

        {'all': <global trending score>,
         <region_slug>: <regional trending score>,
         ...}

    """
    # How many app installs are required in the prior week to be considered
    # "trending". Adjust this as total Marketplace app installs increases.
    #
    # Note: AMO uses 1000.0 for add-ons.
    PRIOR_WEEK_INSTALL_THRESHOLD = 100.0

    client = get_monolith_client()

    week1 = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(8).date().isoformat(),
                    'lte': days_ago(1).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }
    week3 = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(29).date().isoformat(),
                    'lte': days_ago(9).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }

    query = {
        'query': {
            'filtered': {
                'query': {
                    'match_all': {}
                },
                'filter': {
                    'term': {
                        'app-id': app_id
                    }
                }
            }
        },
        'aggregations': {
            'week1': week1,
            'week3': week3,
            'region': {
                'terms': {
                    'field': 'region',
                    # Add size so we get all regions, not just the top 10.
                    'size': len(mkt.regions.ALL_REGIONS)
                },
                'aggregations': {
                    'week1': week1,
                    'week3': week3
                }
            }
        },
        'size': 0
    }

    try:
        res = client.raw(query)
    except ValueError as e:
        task_log.error('Error response from Monolith: {0}'.format(e))
        return {}

    if 'aggregations' not in res:
        task_log.error('No installs for app {}'.format(app_id))
        return {}

    def _score(week1, week3):
        # If last week app installs are < 100, this app isn't trending.
        if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
            return 0.0

        score = 0.0
        if week3 > 1.0:
            score = (week1 - week3) / week3
        if score < 0.0:
            score = 0.0
        return score

    # Global trending score.
    week1 = res['aggregations']['week1']['total_installs']['value']
    week3 = res['aggregations']['week3']['total_installs']['value'] / 3.0

    if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
        # If global installs over the last week aren't over 100, we
        # short-circuit and return a zero-like value as this is not a trending
        # app by definition. Since global installs aren't above 100, per-region
        # installs won't be either.
        return {}

    results = {'all': _score(week1, week3)}

    if 'region' in res['aggregations']:
        for regional_res in res['aggregations']['region']['buckets']:
            region_slug = regional_res['key']
            week1 = regional_res['week1']['total_installs']['value']
            week3 = regional_res['week3']['total_installs']['value'] / 3.0
            results[region_slug] = _score(week1, week3)

    return results
Ejemplo n.º 8
0
def _get_installs(app_id):
    """
    Calculate popularity of app for all regions and per region.

    Returns value in the format of::

        {'all': <global installs>,
         <region_slug>: <regional installs>,
         ...}

    """
    # How many days back do we include when calculating popularity.
    POPULARITY_PERIOD = 90

    client = get_monolith_client()

    popular = {
        'filter': {
            'range': {
                'date': {
                    'gte': days_ago(POPULARITY_PERIOD).date().isoformat(),
                    'lte': days_ago(1).date().isoformat()
                }
            }
        },
        'aggs': {
            'total_installs': {
                'sum': {
                    'field': 'app_installs'
                }
            }
        }
    }

    query = {
        'query': {
            'filtered': {
                'query': {
                    'match_all': {}
                },
                'filter': {
                    'term': {
                        'app-id': app_id
                    }
                }
            }
        },
        'aggregations': {
            'popular': popular,
            'region': {
                'terms': {
                    'field': 'region',
                    # Add size so we get all regions, not just the top 10.
                    'size': len(mkt.regions.ALL_REGIONS)
                },
                'aggregations': {
                    'popular': popular
                }
            }
        },
        'size': 0
    }

    try:
        res = client.raw(query)
    except ValueError as e:
        task_log.error('Error response from Monolith: {0}'.format(e))
        return {}

    if 'aggregations' not in res:
        task_log.error('No installs for app {}'.format(app_id))
        return {}

    results = {
        'all': res['aggregations']['popular']['total_installs']['value']
    }

    if 'region' in res['aggregations']:
        for regional_res in res['aggregations']['region']['buckets']:
            region_slug = regional_res['key']
            popular = regional_res['popular']['total_installs']['value']
            results[region_slug] = popular

    return results