def handle(self, *args, **options):
start = datetime.now() # Measure the time it takes to run the script.
day = options['date']
if not day:
day = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
folder = args[0] if args else 'hive_results'
folder = path.join(settings.TMP_PATH, folder, day)
sep = options['separator']
groups = ('version', 'status', 'app', 'os', 'locale')
group_filepaths = []
# Make sure we're not trying to update with mismatched data.
for group in groups:
filepath = path.join(folder, 'update_counts_by_%s.hive' % group)
if get_date_from_file(filepath, sep) != day:
raise CommandError('%s file contains data for another day' %
filepath)
group_filepaths.append((group, filepath))
# First, make sure we don't have any existing counts for the same day,
# or it would just increment again the same data.
UpdateCount.objects.filter(date=day).delete()
# Memoize the addons and the UpdateCounts.
update_counts = {}
# Perf: preload all the addons once and for all.
# This builds a dict where each key (the addon guid we get from the
# hive query) has the addon_id as value.
guids_to_addon = (dict(Addon.objects.exclude(guid__isnull=True)
.filter(type=amo.ADDON_EXTENSION)
.values_list('guid', 'id')))
index = -1
for group, filepath in group_filepaths:
with open(filepath) as results_file:
for line in results_file:
index += 1
if index and (index % 1000000) == 0:
log.info('Processed %s lines' % index)
splitted = line[:-1].split(sep)
if ((group == 'app' and len(splitted) != 6)
or (group != 'app' and len(splitted) != 5)):
log.debug('Badly formatted row: %s' % line)
continue
if group == 'app':
day, addon_guid, app_id, app_ver, count, \
update_type = splitted
else:
day, addon_guid, data, count, update_type = splitted
if update_type:
update_type.strip()
# Old versions of Firefox don't provide the update type.
# All the following are "empty-like" values.
if update_type in ['0', 'NULL', 'None', '', '\N',
'%UPDATE_TYPE%']:
update_type = None
try:
count = int(count)
if update_type:
update_type = int(update_type)
except ValueError: # Badly formatted? Drop.
continue
# The following is magic that I don't understand. I've just
# been told that this is the way we can make sure a request
# is valid:
# > the lower bits for updateType (eg 112) should add to
# > 16, if not, ignore the request.
# > udpateType & 31 == 16 == valid request.
if update_type and update_type & 31 != 16:
log.debug("Update type doesn't add to 16: %s" %
update_type)
continue
# Does this addon exit?
if addon_guid.strip() and addon_guid in guids_to_addon:
addon_id = guids_to_addon[addon_guid]
else:
continue
# Memoize the UpdateCount.
if addon_guid in update_counts:
uc = update_counts[addon_guid]
else:
uc = UpdateCount(date=day, addon_id=addon_id, count=0)
update_counts[addon_guid] = uc
# We can now fill the UpdateCount object.
if group == 'version':
# Take this count as the global number of daily users.
uc.count += count
uc.versions = update_inc(uc.versions, data, count)
elif group == 'status':
uc.statuses = update_inc(uc.statuses, data, count)
elif group == 'app':
# Applications is a dict of dicts, eg:
# {"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}":
# {"10.0": 2, "21.0": 1, ....},
# "some other application guid": ...
# }
if uc.applications is None:
uc.applications = {}
app = uc.applications.get(app_id, {})
# Now overwrite this application's dict with
# incremented counts for its versions.
uc.applications.update(
{app_id: update_inc(app, app_ver, count)})
elif group == 'os':
uc.oses = update_inc(uc.oses, data, count)
elif group == 'locale':
# Drop incorrect locales sizes.
if len(data) > 10:
continue
# Collapse locales to `xx_yy` if possible.
data = data.strip().lower().replace('-', '_')
uc.locales = update_inc(uc.locales, data, count)
# Create in bulk: this is much faster.
UpdateCount.objects.bulk_create(update_counts.values(), 100)
log.info('Processed a total of %s lines' % (index + 1))
log.debug('Total processing time: %s' % (datetime.now() - start))
# Clean up files.
for _, filepath in group_filepaths:
log.debug('Deleting {path}'.format(path=filepath))
unlink(filepath)
def handle(self, *args, **options):
start = datetime.now() # Measure the time it takes to run the script.
day = options['date']
if not day:
day = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
folder = args[0] if args else 'hive_results'
folder = path.join(settings.NETAPP_STORAGE, 'tmp', folder, day)
sep = options['separator']
groups = ('version', 'status', 'app', 'os', 'locale')
# Make sure we're not trying to update with mismatched data.
for group in groups:
filepath = path.join(folder, 'update_counts_by_%s.hive' % group)
if get_date_from_file(filepath, sep) != day:
raise CommandError('%s file contains data for another day' %
filepath)
# First, make sure we don't have any existing counts for the same day,
# or it would just increment again the same data.
UpdateCount.objects.filter(date=day).delete()
# Memoize the addons and the UpdateCounts.
update_counts = {}
# Perf: preload all the addons once and for all.
# This builds a dict where each key (the addon guid we get from the
# hive query) has the addon_id as value.
guids_to_addon = (dict(
Addon.objects.exclude(guid__isnull=True).filter(
type=amo.ADDON_EXTENSION).values_list('guid', 'id')))
index = -1
for group in groups:
filepath = path.join(folder, 'update_counts_by_%s.hive' % group)
with open(filepath) as results_file:
for line in results_file:
index += 1
if index and (index % 1000000) == 0:
log.info('Processed %s lines' % index)
splitted = line[:-1].split(sep)
if ((group == 'app' and len(splitted) != 6)
or (group != 'app' and len(splitted) != 5)):
log.debug('Badly formatted row: %s' % line)
continue
if group == 'app':
day, addon_guid, app_id, app_ver, count, \
update_type = splitted
else:
day, addon_guid, data, count, update_type = splitted
try:
count, update_type = int(count), int(update_type)
except ValueError: # Badly formatted? Drop.
continue
# The following is magic that I don't understand. I've just
# been told that this is the way we can make sure a request
# is valid:
# > the lower bits for updateType (eg 112) should add to
# > 16, if not, ignore the request.
# > udpateType & 31 == 16 == valid request.
if update_type & 31 != 16:
log.debug("Update type doesn't add to 16: %s" %
update_type)
continue
# Does this addon exit?
if addon_guid.strip() and addon_guid in guids_to_addon:
addon_id = guids_to_addon[addon_guid]
else:
continue
# Memoize the UpdateCount.
if addon_guid in update_counts:
uc = update_counts[addon_guid]
else:
uc = UpdateCount(date=day, addon_id=addon_id, count=0)
update_counts[addon_guid] = uc
# We can now fill the UpdateCount object.
if group == 'version':
# Take this count as the global number of daily users.
uc.count += count
uc.versions = update_inc(uc.versions, data, count)
elif group == 'status':
uc.statuses = update_inc(uc.statuses, data, count)
elif group == 'app':
# Applications is a dict of dicts, eg:
# {"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}":
# {"10.0": 2, "21.0": 1, ....},
# "some other application guid": ...
# }
if uc.applications is None:
uc.applications = {}
app = uc.applications.get(app_id, {})
# Now overwrite this application's dict with
# incremented counts for its versions.
uc.applications.update(
{app_id: update_inc(app, app_ver, count)})
elif group == 'os':
uc.oses = update_inc(uc.oses, data, count)
elif group == 'locale':
# Drop incorrect locales sizes.
if len(data) > 10:
continue
# Collapse locales to `xx_yy` if possible.
data = data.strip().lower().replace('-', '_')
uc.locales = update_inc(uc.locales, data, count)
# Create in bulk: this is much faster.
UpdateCount.objects.bulk_create(update_counts.values(), 100)
log.info('Processed a total of %s lines' % (index + 1))
log.debug('Total processing time: %s' % (datetime.now() - start))
def handle(self, *args, **options):
start = datetime.now() # Measure the time it takes to run the script.
day = options['date']
if not day:
raise CommandError('You must specify a --date parameter in the '
' YYYY-MM-DD format.')
sep = options['separator']
filename = args[0]
# First, make sure we don't have any existing counts for the same day,
# or it would just increment again the same data.
UpdateCount.objects.filter(date=day).delete()
# Memoize the addons and the UpdateCounts.
update_counts = {}
# Perf: preload all the addons once and for all.
# This builds a dict where each key (the addon guid we get from the
# hive query) has the addon_id as value.
guids_to_addon = dict(Addon.objects.values_list('guid', 'id'))
with open(filename) as count_file:
for index, line in enumerate(count_file):
if index and (index % 10000) == 0:
log.info('Processed %s lines' % index)
splitted = line[:-1].split(sep)
if len(splitted) != 8:
log.debug('Badly formatted row: %s' % line)
continue
counter, addon_guid, version, status, app_id, version, \
app_os, locale, update_type = splitted
try:
counter = int(counter)
except ValueError: # Badly formatted? Drop.
continue
# The following is magic that I don't understand. I've just
# been told that this is the way we can make sure a request is
# valid:
# > the lower bits for updateType=112 should add to 16, if not,
# > ignore the request. udpateType & 31 == 16 == valid request.
# The 8th column is the updateType this quote is talking about.
try:
if int(update_type) & 31 == 16:
continue
except:
continue
# We may have several statuses in the same field.
statuses = status.split(',')
# Does this addon exit?
if addon_guid in guids_to_addon:
addon_id = guids_to_addon[addon_guid]
else:
log.info('Addon with guid: %s not found' % addon_guid)
continue
# Memoize the UpdateCount.
if addon_guid in update_counts:
uc = update_counts[addon_guid]
else:
uc = UpdateCount(date=day, addon_id=addon_id, count=0)
update_counts[addon_guid] = uc
# We can now fill the UpdateCount object.
uc.count += counter
uc.versions = update_inc(uc.versions, version, counter)
# Applications is a dict of dicts, eg:
# {"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}": # Firefox.
# {"10.0": 2, "21.0": 1, ....},
# "some other application guid": ...
# }
if uc.applications is None:
uc.applications = {}
app = uc.applications.get(app_id, {})
# Now overwrite this application's dict with incremented
# counts for its versions.
uc.applications.update(
{app_id: update_inc(app, version, counter)})
uc.oses = update_inc(uc.oses, app_os, counter)
uc.locales = update_inc(uc.locales, locale, counter)
# We may have received a list of more than one status.
for status in statuses:
uc.statuses = update_inc(uc.statuses, status, counter)
# Create in bulk: this is much faster.
UpdateCount.objects.bulk_create(update_counts.values(), 100)
log.info('Processed a total of %s lines' % (index + 1))
log.debug('Total processing time: %s' % (datetime.now() - start))